Skip to content

Commit 16938ea

Browse files
koenvanderveenkoenvanderveen
authored
Merge pull request #9346 from OpenMined/testing-k8s-migrations
add k8s migration tests (not in CI)
2 parents 50800b7 + 185d8c3 commit 16938ea

File tree

17 files changed

+1143
-36
lines changed

17 files changed

+1143
-36
lines changed

‎notebooks/scenarios/bigquery/upgradability/0.9.1_helpers/apis/live/schema.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from ..rate_limiter import is_within_rate_limit
1010

1111

12-
def make_schema(settings: dict, worker_pool_name: str) -> Callable:
12+
def make_schema(settings: dict, worker_pool: str) -> Callable:
1313
updated_settings = {
1414
"calls_per_min": 5,
1515
"rate_limiter_enabled": True,
@@ -28,7 +28,7 @@ def make_schema(settings: dict, worker_pool_name: str) -> Callable:
2828
helper_functions=[
2929
is_within_rate_limit
3030
], # Adds ratelimit as this is also a method available to data scientists
31-
worker_pool_name=worker_pool_name,
31+
worker_pool=worker_pool,
3232
)
3333
def live_schema(
3434
context,

‎notebooks/scenarios/bigquery/upgradability/0.9.1_helpers/apis/mock/schema.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from .data import schema_dict
1010

1111

12-
def make_schema(settings, worker_pool_name) -> Callable:
12+
def make_schema(settings, worker_pool) -> Callable:
1313
updated_settings = {
1414
"calls_per_min": 5,
1515
"rate_limiter_enabled": True,
@@ -21,7 +21,7 @@ def make_schema(settings, worker_pool_name) -> Callable:
2121
description="This endpoint allows for visualising the metadata of tables available in BigQuery.",
2222
settings=updated_settings,
2323
helper_functions=[is_within_rate_limit],
24-
worker_pool_name=worker_pool_name,
24+
worker_pool=worker_pool,
2525
)
2626
def mock_schema(
2727
context,

‎notebooks/scenarios/bigquery/upgradability/0.9.1_helpers/apis/submit_query.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22
import syft as sy
33

44

5-
def make_submit_query(settings, worker_pool_name):
6-
updated_settings = {"user_code_worker": worker_pool_name} | settings
5+
def make_submit_query(settings, worker_pool):
6+
updated_settings = {"user_code_worker": worker_pool} | settings
77

88
@sy.api_endpoint(
99
path="bigquery.submit_query",
1010
description="API endpoint that allows you to submit SQL queries to run on the private data.",
11-
worker_pool_name=worker_pool_name,
11+
worker_pool=worker_pool,
1212
settings=updated_settings,
1313
)
1414
def submit_query(

‎notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/011-users-emails-passwords.ipynb‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -657,11 +657,6 @@
657657
}
658658
],
659659
"metadata": {
660-
"kernelspec": {
661-
"display_name": "syft_3.12",
662-
"language": "python",
663-
"name": "python3"
664-
},
665660
"language_info": {
666661
"codemirror_mode": {
667662
"name": "ipython",
@@ -672,7 +667,7 @@
672667
"name": "python",
673668
"nbconvert_exporter": "python",
674669
"pygments_lexer": "ipython3",
675-
"version": "3.12.4"
670+
"version": "3.12.5"
676671
}
677672
},
678673
"nbformat": 4,

‎notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/020-configure-api.ipynb‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@
216216
" description=\"This endpoint allows to query Bigquery storage via SQL queries.\",\n",
217217
" private_function=private_func,\n",
218218
" mock_function=mock_func,\n",
219-
" worker_pool_name=this_worker_pool_name,\n",
219+
" worker_pool=this_worker_pool_name,\n",
220220
")\n",
221221
"\n",
222222
"high_client.custom_api.add(endpoint=new_endpoint)"
@@ -407,7 +407,7 @@
407407
" settings={\n",
408408
" \"calls_per_min\": 5,\n",
409409
" },\n",
410-
" worker_pool_name=this_worker_pool_name,\n",
410+
" worker_pool=this_worker_pool_name,\n",
411411
")"
412412
]
413413
},
@@ -462,7 +462,7 @@
462462
"outputs": [],
463463
"source": [
464464
"submit_query_function = make_submit_query(\n",
465-
" settings={}, worker_pool_name=this_worker_pool_name\n",
465+
" settings={}, worker_pool=this_worker_pool_name\n",
466466
")"
467467
]
468468
},
@@ -640,11 +640,6 @@
640640
}
641641
],
642642
"metadata": {
643-
"kernelspec": {
644-
"display_name": "syft_3.12",
645-
"language": "python",
646-
"name": "python3"
647-
},
648643
"language_info": {
649644
"codemirror_mode": {
650645
"name": "ipython",

‎notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/emails_k8s.json‎

Lines changed: 107 additions & 0 deletions
Large diffs are not rendered by default.

‎notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/jobs_k8s.json‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
{
2+
"richard-porter-fake@openmined.org": [
3+
{
4+
"user_email": "richard-porter-fake@openmined.org",
5+
"func_name": "simple_query_61dd0e",
6+
"query": "SELECT table_id, AVG(colname) AS average_score\n FROM data_10gb.subreddits\n GROUP BY table_id\n LIMIT 713078",
7+
"job_type": "simple_query",
8+
"settings": {
9+
"dataset": "data_10gb",
10+
"table": "subreddits",
11+
"groupby_col": "table_id",
12+
"score_col": "colname",
13+
"limit": 713078
14+
},
15+
"should_succeed": true,
16+
"should_submit": true,
17+
"code_path": "simple_query_61dd0e",
18+
"admin_reviewed": true,
19+
"result_as_expected": true
20+
},
21+
{
22+
"user_email": "richard-porter-fake@openmined.org",
23+
"func_name": "simple_query_7e0b14",
24+
"query": "SELECT subreddit_id, AVG(score) AS average_score\n FROM data_10gb.comments\n GROUP BY subreddit_id\n LIMIT 770329",
25+
"job_type": "simple_query",
26+
"settings": {
27+
"dataset": "data_10gb",
28+
"table": "comments",
29+
"groupby_col": "subreddit_id",
30+
"score_col": "score",
31+
"limit": 770329
32+
},
33+
"should_succeed": true,
34+
"should_submit": true,
35+
"code_path": "simple_query_7e0b14",
36+
"admin_reviewed": true,
37+
"result_as_expected": true
38+
}
39+
],
40+
"christine-peterson-fake@openmined.org": [
41+
{
42+
"user_email": "christine-peterson-fake@openmined.org",
43+
"func_name": "simple_query_6e409e",
44+
"query": "SELECT table_id, AVG(colname) AS average_score\n FROM test_1gb.subreddits\n GROUP BY table_id\n LIMIT 969621",
45+
"job_type": "simple_query",
46+
"settings": {
47+
"dataset": "test_1gb",
48+
"table": "subreddits",
49+
"groupby_col": "table_id",
50+
"score_col": "colname",
51+
"limit": 969621
52+
},
53+
"should_succeed": true,
54+
"should_submit": true,
55+
"code_path": "simple_query_6e409e",
56+
"admin_reviewed": true,
57+
"result_as_expected": true
58+
},
59+
{
60+
"user_email": "christine-peterson-fake@openmined.org",
61+
"func_name": "wrong_syntax_query_201be7",
62+
"query": "SELECT * FROM table INCORRECT SYNTAX",
63+
"job_type": "wrong_syntax_query",
64+
"settings": {},
65+
"should_succeed": false,
66+
"should_submit": true,
67+
"code_path": "wrong_syntax_query_201be7",
68+
"admin_reviewed": true,
69+
"result_as_expected": true
70+
}
71+
],
72+
"jon-wilson-fake@openmined.org": [
73+
{
74+
"user_email": "jon-wilson-fake@openmined.org",
75+
"func_name": "simple_query_cac747",
76+
"query": "SELECT table_id, AVG(colname) AS average_score\n FROM data_10gb.subreddits\n GROUP BY table_id\n LIMIT 865657",
77+
"job_type": "simple_query",
78+
"settings": {
79+
"dataset": "data_10gb",
80+
"table": "subreddits",
81+
"groupby_col": "table_id",
82+
"score_col": "colname",
83+
"limit": 865657
84+
},
85+
"should_succeed": true,
86+
"should_submit": true,
87+
"code_path": "simple_query_cac747",
88+
"admin_reviewed": true,
89+
"result_as_expected": true
90+
},
91+
{
92+
"user_email": "jon-wilson-fake@openmined.org",
93+
"func_name": "job_too_much_text_7d0b7e",
94+
"query": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
95+
"job_type": "job_too_much_text",
96+
"settings": {},
97+
"should_succeed": false,
98+
"should_submit": true,
99+
"code_path": "job_too_much_text_7d0b7e",
100+
"admin_reviewed": true,
101+
"result_as_expected": true
102+
}
103+
],
104+
"tyrone-flores-fake@openmined.org": [
105+
{
106+
"user_email": "tyrone-flores-fake@openmined.org",
107+
"func_name": "simple_query_9ab2f5",
108+
"query": "SELECT table_id, AVG(colname) AS average_score\n FROM test_1gb.subreddits\n GROUP BY table_id\n LIMIT 897138",
109+
"job_type": "simple_query",
110+
"settings": {
111+
"dataset": "test_1gb",
112+
"table": "subreddits",
113+
"groupby_col": "table_id",
114+
"score_col": "colname",
115+
"limit": 897138
116+
},
117+
"should_succeed": true,
118+
"should_submit": true,
119+
"code_path": "simple_query_9ab2f5",
120+
"admin_reviewed": true,
121+
"result_as_expected": true
122+
},
123+
{
124+
"user_email": "tyrone-flores-fake@openmined.org",
125+
"func_name": "job_long_name_d6277aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
126+
"query": "SELECT subreddit_id, AVG(score) AS average_score\n FROM data_10gb.comments\n GROUP BY subreddit_id\n LIMIT 591345",
127+
"job_type": "job_long_name",
128+
"settings": {
129+
"dataset": "data_10gb",
130+
"table": "comments",
131+
"groupby_col": "subreddit_id",
132+
"score_col": "score",
133+
"limit": 591345
134+
},
135+
"should_succeed": true,
136+
"should_submit": true,
137+
"code_path": "job_long_name_d6277aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
138+
"admin_reviewed": true,
139+
"result_as_expected": true
140+
}
141+
],
142+
"brian-bradford-fake@openmined.org": [
143+
{
144+
"user_email": "brian-bradford-fake@openmined.org",
145+
"func_name": "simple_query_edeb0b",
146+
"query": "SELECT subreddit_id, AVG(score) AS average_score\n FROM data_10gb.comments\n GROUP BY subreddit_id\n LIMIT 133654",
147+
"job_type": "simple_query",
148+
"settings": {
149+
"dataset": "data_10gb",
150+
"table": "comments",
151+
"groupby_col": "subreddit_id",
152+
"score_col": "score",
153+
"limit": 133654
154+
},
155+
"should_succeed": true,
156+
"should_submit": true,
157+
"code_path": "simple_query_edeb0b",
158+
"admin_reviewed": true,
159+
"result_as_expected": true
160+
},
161+
{
162+
"user_email": "brian-bradford-fake@openmined.org",
163+
"func_name": "job_funcname_xss_338b43<script>alert('XSS in funcname')</script>",
164+
"query": "SELECT table_id, AVG(colname) AS average_score\n FROM test_1gb.subreddits\n GROUP BY table_id\n LIMIT 64464",
165+
"job_type": "job_funcname_xss",
166+
"settings": {
167+
"dataset": "test_1gb",
168+
"table": "subreddits",
169+
"groupby_col": "table_id",
170+
"score_col": "colname",
171+
"limit": 64464
172+
},
173+
"should_succeed": true,
174+
"should_submit": false,
175+
"code_path": null,
176+
"admin_reviewed": false,
177+
"result_as_expected": null
178+
}
179+
]
180+
}

‎notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/users_k8s.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
[
2+
{
3+
"name": "Richard Porter",
4+
"email": "richard-porter-fake@openmined.org",
5+
"password": "Fmo4!MxcZ(",
6+
"role": "ServiceRole.DATA_SCIENTIST",
7+
"new_password": null,
8+
"email_disabled": true,
9+
"reset_password": false,
10+
"reset_token": null
11+
},
12+
{
13+
"name": "Christine Peterson",
14+
"email": "christine-peterson-fake@openmined.org",
15+
"password": "4J13O(x&!G",
16+
"role": "ServiceRole.DATA_SCIENTIST",
17+
"new_password": null,
18+
"email_disabled": false,
19+
"reset_password": true,
20+
"reset_token": "DgSgUZ2gOpvm"
21+
},
22+
{
23+
"name": "Jon Wilson",
24+
"email": "jon-wilson-fake@openmined.org",
25+
"password": "h5TEQw#t%4",
26+
"role": "ServiceRole.DATA_SCIENTIST",
27+
"new_password": null,
28+
"email_disabled": false,
29+
"reset_password": false,
30+
"reset_token": null
31+
},
32+
{
33+
"name": "Tyrone Flores",
34+
"email": "tyrone-flores-fake@openmined.org",
35+
"password": "&#CJFvxb3_",
36+
"role": "ServiceRole.DATA_SCIENTIST",
37+
"new_password": null,
38+
"email_disabled": false,
39+
"reset_password": false,
40+
"reset_token": null
41+
},
42+
{
43+
"name": "Brian Bradford",
44+
"email": "brian-bradford-fake@openmined.org",
45+
"password": "crZ#4Heh6e",
46+
"role": "ServiceRole.DATA_SCIENTIST",
47+
"new_password": null,
48+
"email_disabled": false,
49+
"reset_password": false,
50+
"reset_token": null
51+
}
52+
]

0 commit comments

Comments
 (0)