Skip to content

Commit 2442f06

Browse files
olevskiolevski
committed
fix: migrate old session launchers (#513)
1 parent 39f981c commit 2442f06

6 files changed

Lines changed: 283 additions & 38 deletions

File tree

‎.devcontainer/docker-compose.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,12 @@ services:
3636
restart: unless-stopped
3737
volumes:
3838
- postgres-data:/var/lib/postgresql/data
39+
- type: bind
40+
source: ./generate_ulid_func.sql
41+
target: /docker-entrypoint-initdb.d/generate_ulid_func.sql
3942
environment:
4043
POSTGRES_USER: renku
41-
POSTGRES_DB: renku
44+
POSTGRES_DB: postgres
4245
POSTGRES_PASSWORD: renku
4346
ports:
4447
- "8000:8000"

‎.devcontainer/generate_ulid_func.sql‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
CREATE DATABASE renku_template;
2+
ALTER DATABASE renku_template WITH is_template TRUE;
3+
\c renku_template;
4+
5+
-- From https://github.com/geckoboard/pgulid/blob/master/pgulid.sql
6+
-- Taken at commit sha b265253
7+
-- pgulid is based on OK Log's Go implementation of the ULID spec
8+
--
9+
-- https://github.com/oklog/ulid
10+
-- https://github.com/ulid/spec
11+
--
12+
-- Copyright 2016 The Oklog Authors
13+
-- Licensed under the Apache License, Version 2.0 (the "License");
14+
-- you may not use this file except in compliance with the License.
15+
-- You may obtain a copy of the License at
16+
--
17+
-- http://www.apache.org/licenses/LICENSE-2.0
18+
--
19+
-- Unless required by applicable law or agreed to in writing, software
20+
-- distributed under the License is distributed on an "AS IS" BASIS,
21+
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22+
-- See the License for the specific language governing permissions and
23+
-- limitations under the License.
24+
25+
CREATE EXTENSION IF NOT EXISTS pgcrypto;
26+
27+
-- NOTE: REPLACE will error if you change the name, args or return type of the function
28+
-- There is no CREATE IF EXISTS, this is the closest thing that gives similar functionality
29+
CREATE OR REPLACE FUNCTION generate_ulid()
30+
RETURNS TEXT
31+
AS $$
32+
DECLARE
33+
-- Crockford's Base32
34+
encoding BYTEA = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
35+
timestamp BYTEA = E'\\000\\000\\000\\000\\000\\000';
36+
output TEXT = '';
37+
38+
unix_time BIGINT;
39+
ulid BYTEA;
40+
BEGIN
41+
-- 6 timestamp bytes
42+
unix_time = (EXTRACT(EPOCH FROM CLOCK_TIMESTAMP()) * 1000)::BIGINT;
43+
timestamp = SET_BYTE(timestamp, 0, (unix_time >> 40)::BIT(8)::INTEGER);
44+
timestamp = SET_BYTE(timestamp, 1, (unix_time >> 32)::BIT(8)::INTEGER);
45+
timestamp = SET_BYTE(timestamp, 2, (unix_time >> 24)::BIT(8)::INTEGER);
46+
timestamp = SET_BYTE(timestamp, 3, (unix_time >> 16)::BIT(8)::INTEGER);
47+
timestamp = SET_BYTE(timestamp, 4, (unix_time >> 8)::BIT(8)::INTEGER);
48+
timestamp = SET_BYTE(timestamp, 5, unix_time::BIT(8)::INTEGER);
49+
50+
-- 10 entropy bytes
51+
ulid = timestamp || gen_random_bytes(10);
52+
53+
-- Encode the timestamp
54+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 224) >> 5));
55+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 31)));
56+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 1) & 248) >> 3));
57+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 1) & 7) << 2) | ((GET_BYTE(ulid, 2) & 192) >> 6)));
58+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 2) & 62) >> 1));
59+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 2) & 1) << 4) | ((GET_BYTE(ulid, 3) & 240) >> 4)));
60+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 3) & 15) << 1) | ((GET_BYTE(ulid, 4) & 128) >> 7)));
61+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 4) & 124) >> 2));
62+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 4) & 3) << 3) | ((GET_BYTE(ulid, 5) & 224) >> 5)));
63+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 5) & 31)));
64+
65+
-- Encode the entropy
66+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 6) & 248) >> 3));
67+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 6) & 7) << 2) | ((GET_BYTE(ulid, 7) & 192) >> 6)));
68+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 7) & 62) >> 1));
69+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 7) & 1) << 4) | ((GET_BYTE(ulid, 8) & 240) >> 4)));
70+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 8) & 15) << 1) | ((GET_BYTE(ulid, 9) & 128) >> 7)));
71+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 9) & 124) >> 2));
72+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 9) & 3) << 3) | ((GET_BYTE(ulid, 10) & 224) >> 5)));
73+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 10) & 31)));
74+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 11) & 248) >> 3));
75+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 11) & 7) << 2) | ((GET_BYTE(ulid, 12) & 192) >> 6)));
76+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 12) & 62) >> 1));
77+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 12) & 1) << 4) | ((GET_BYTE(ulid, 13) & 240) >> 4)));
78+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 13) & 15) << 1) | ((GET_BYTE(ulid, 14) & 128) >> 7)));
79+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 14) & 124) >> 2));
80+
output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 14) & 3) << 3) | ((GET_BYTE(ulid, 15) & 224) >> 5)));
81+
output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 15) & 31)));
82+
83+
RETURN output;
84+
END
85+
$$
86+
LANGUAGE plpgsql
87+
VOLATILE;
88+
89+
CREATE DATABASE renku TEMPLATE renku_template;

‎components/renku_data_services/migrations/versions/1ef98b967767_add_command_and_args_to_environment.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,13 @@
1616
branch_labels = None
1717
depends_on = None
1818

19+
default_command = ["sh", "-c"]
20+
default_args = [
21+
"jupyter server --ServerApp.ip=0.0.0.0 --ServerApp.port=8888 --ServerApp.base_url=$RENKU_BASE_URL_PATH "
22+
'--ServerApp.token="" --ServerApp.password="" --ServerApp.allow_remote_access=true '
23+
"--ContentsManager.allow_hidden=true --ServerApp.allow_origin=*",
24+
]
25+
1926

2027
def upgrade() -> None:
2128
# ### commands auto generated by Alembic - please adjust! ###
@@ -31,6 +38,12 @@ def upgrade() -> None:
3138
),
3239
schema="sessions",
3340
)
41+
op.execute(
42+
sa.text("UPDATE sessions.environments SET command=:command, args=:args").bindparams(
43+
sa.bindparam("command", value=default_command, type_=sa.JSON),
44+
sa.bindparam("args", value=default_args, type_=sa.JSON),
45+
)
46+
)
3447
# ### end Alembic commands ###
3548

3649

‎components/renku_data_services/migrations/versions/584598f3b769_expand_and_separate_environments_from_.py‎

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,48 @@
2626

2727
def upgrade() -> None:
2828
# ### commands auto generated by Alembic - please adjust! ###
29-
op.execute("DELETE FROM sessions.launchers")
30-
op.drop_column("launchers", "default_url", schema="sessions")
31-
op.drop_column("launchers", "environment_kind", schema="sessions")
32-
op.drop_column("launchers", "container_image", schema="sessions")
29+
# Migrate session launchers
3330
op.execute("DROP TYPE environmentkind CASCADE")
3431
op.execute("CREATE TYPE environmentkind AS ENUM ('GLOBAL', 'CUSTOM')")
35-
op.add_column("environments", sa.Column("port", sa.Integer(), nullable=True), schema="sessions")
36-
op.add_column("environments", sa.Column("working_directory", sa.String(), nullable=True), schema="sessions")
37-
op.add_column("environments", sa.Column("mount_directory", sa.String(), nullable=True), schema="sessions")
38-
op.add_column("environments", sa.Column("uid", sa.Integer(), nullable=True), schema="sessions")
39-
op.add_column("environments", sa.Column("gid", sa.Integer(), nullable=True), schema="sessions")
4032
op.add_column(
4133
"environments",
4234
sa.Column("environment_kind", sa.Enum("GLOBAL", "CUSTOM", name="environmentkind"), nullable=True),
4335
schema="sessions",
4436
)
37+
op.execute("UPDATE sessions.environments SET environment_kind = 'GLOBAL' WHERE environment_kind is NULL")
38+
# NOTE: When the session launcher has environment_id set to null then it is a custom environment
39+
# NOTE: We populate the name column in the new environment with the session launcher id
40+
# This way we can join the newly add environments with their launchers and update the foreign key in the launchers
41+
# with the newly created environments.
42+
# NOTE: Since postgres cannot autogenerate ulids we use the session launcher ulid when creating the environment
43+
op.execute(
44+
"INSERT INTO sessions.environments(id, name, created_by_id, creation_date, container_image, default_url, environment_kind) "
45+
"SELECT generate_ulid(), id, created_by_id, creation_date, container_image, default_url, 'CUSTOM' "
46+
"FROM sessions.launchers "
47+
"WHERE environment_id IS NULL"
48+
)
49+
op.execute(
50+
"UPDATE sessions.launchers "
51+
"SET environment_id = sessions.environments.id "
52+
"FROM sessions.environments "
53+
"WHERE sessions.environments.name = sessions.launchers.id"
54+
)
55+
# NOTE: Make the environment name human readable when the join and udpate is done
56+
op.execute(
57+
"UPDATE sessions.environments "
58+
"SET name = CONCAT('Custom environment for session launcher ID ', name) "
59+
"WHERE environment_kind = 'CUSTOM'"
60+
)
61+
# Drop unused fields from session launchers
62+
op.drop_column("launchers", "default_url", schema="sessions")
63+
# op.drop_column("launchers", "environment_kind", schema="sessions")
64+
op.drop_column("launchers", "container_image", schema="sessions")
65+
# Add new fields to environment
66+
op.add_column("environments", sa.Column("port", sa.Integer(), nullable=True), schema="sessions")
67+
op.add_column("environments", sa.Column("working_directory", sa.String(), nullable=True), schema="sessions")
68+
op.add_column("environments", sa.Column("mount_directory", sa.String(), nullable=True), schema="sessions")
69+
op.add_column("environments", sa.Column("uid", sa.Integer(), nullable=True), schema="sessions")
70+
op.add_column("environments", sa.Column("gid", sa.Integer(), nullable=True), schema="sessions")
4571
op.execute(sa.text("UPDATE sessions.environments SET port = :port WHERE port is NULL").bindparams(port=port))
4672
op.execute(
4773
sa.text(
@@ -55,12 +81,12 @@ def upgrade() -> None:
5581
)
5682
op.execute(sa.text("UPDATE sessions.environments SET uid = :uid WHERE uid is NULL").bindparams(uid=uid))
5783
op.execute(sa.text("UPDATE sessions.environments SET gid = :gid WHERE gid is NULL").bindparams(gid=gid))
58-
op.execute("UPDATE sessions.environments SET environment_kind = 'GLOBAL' WHERE environment_kind is NULL")
5984
op.execute(
6085
sa.text("UPDATE sessions.environments SET default_url = :default_url WHERE default_url is NULL").bindparams(
6186
default_url=default_url
6287
)
6388
)
89+
# Set proper nullable constraints
6490
op.alter_column("environments", "port", nullable=False, schema="sessions")
6591
op.alter_column("environments", "working_directory", nullable=False, schema="sessions")
6692
op.alter_column("environments", "mount_directory", nullable=False, schema="sessions")
@@ -75,13 +101,7 @@ def upgrade() -> None:
75101

76102
def downgrade() -> None:
77103
# ### commands auto generated by Alembic - please adjust! ###
78-
op.drop_column("environments", "environment_kind", schema="sessions")
79-
op.drop_column("environments", "gid", schema="sessions")
80-
op.drop_column("environments", "uid", schema="sessions")
81-
op.drop_column("environments", "mount_directory", schema="sessions")
82-
op.drop_column("environments", "working_directory", schema="sessions")
83-
op.drop_column("environments", "port", schema="sessions")
84-
op.execute("DROP TYPE environmentkind")
104+
op.execute("ALTER TYPE environmentkind RENAME TO environmentkind_old;")
85105
op.execute("CREATE TYPE environmentkind AS ENUM ('global_environment', 'container_image')")
86106
op.add_column(
87107
"launchers",
@@ -94,7 +114,7 @@ def downgrade() -> None:
94114
"environment_kind",
95115
postgresql.ENUM("global_environment", "container_image", name="environmentkind"),
96116
autoincrement=False,
97-
nullable=False,
117+
nullable=True,
98118
),
99119
schema="sessions",
100120
)
@@ -103,7 +123,26 @@ def downgrade() -> None:
103123
sa.Column("default_url", sa.VARCHAR(length=200), autoincrement=False, nullable=True),
104124
schema="sessions",
105125
)
126+
# Move the custom environments spec back into the session launcher table
127+
op.execute(
128+
"UPDATE sessions.launchers "
129+
"SET default_url = environments.default_url, "
130+
"container_image = environments.container_image, "
131+
"environment_kind = 'container_image' "
132+
"FROM sessions.environments "
133+
"WHERE launchers.environment_id = environments.id AND "
134+
"environments.environment_kind = 'CUSTOM'"
135+
)
136+
op.execute("UPDATE sessions.launchers SET environment_kind = 'global_environment' WHERE environment_kind IS NULL")
137+
op.alter_column("launchers", "environment_kind", nullable=False, schema="sessions")
106138
op.alter_column(
107139
"environments", "default_url", existing_type=sa.VARCHAR(length=200), nullable=True, schema="sessions"
108140
)
141+
op.drop_column("environments", "environment_kind", schema="sessions")
142+
op.drop_column("environments", "gid", schema="sessions")
143+
op.drop_column("environments", "uid", schema="sessions")
144+
op.drop_column("environments", "mount_directory", schema="sessions")
145+
op.drop_column("environments", "working_directory", schema="sessions")
146+
op.drop_column("environments", "port", schema="sessions")
147+
op.execute("DROP TYPE environmentkind_old CASCADE")
109148
# ### end Alembic commands ###

0 commit comments

Comments
 (0)