Skip to content

Commit dcc7d03

Browse files
olevskiolevski
committed
squashme: notebooks changes
This includes major edits to the notebooks code to work with the data service.
1 parent 6f3b3c0 commit dcc7d03

182 files changed

Lines changed: 11906 additions & 3866 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.devcontainer/.poetry_cache/.keep‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎.devcontainer/devcontainer.json‎

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,42 @@
1-
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
2-
// README at: https://github.com/devcontainers/templates/tree/main/src/postgres
31
{
4-
"name": "Python 3 & PostgreSQL",
2+
"name": "Data services devcontainer",
53
"dockerComposeFile": "docker-compose.yml",
64
"service": "data_service",
7-
"remoteUser": "renku",
8-
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
9-
// Features to add to the dev container. More info: https://containers.dev/features.
5+
"containerUser": "root",
6+
"workspaceFolder": "/workspace",
7+
"shutdownAction": "stopCompose",
108
"features": {
119
"ghcr.io/devcontainers-contrib/features/poetry:2": {},
12-
"ghcr.io/devcontainers/features/git:1": {},
13-
"ghcr.io/jungaretti/features/make:1": {},
14-
"ghcr.io/rocker-org/devcontainer-features/apt-packages:1": {
15-
"packages": "procps,gcc,python3-dev,curl,ca-certificates,gpg"
16-
},
1710
"ghcr.io/devcontainers-contrib/features/bash-command:1": {
18-
"command": "apt update && apt install -y curl ca-certificates gpg && curl https://apt.fury.io/authzed/gpg.key | apt-key add - && echo 'deb https://apt.fury.io/authzed/ * *' > /etc/apt/sources.list.d/fury.list && apt update && apt install -y zed && curl https://pkg.authzed.com/apt/gpg.key | apt-key add - && echo 'deb https://pkg.authzed.com/apt/ * *' > /etc/apt/sources.list.d/authzed.list && apt update && apt install -y spicedb lsof"
19-
}
11+
"command": "poetry self add poetry-polylith-plugin"
12+
},
13+
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
14+
"ghcr.io/mpriscella/features/kind:1": {},
15+
"ghcr.io/devcontainers-contrib/features/gh-release:1": {
16+
"repo": "authzed/zed",
17+
"binaryNames": "zed"
18+
},
19+
"ghcr.io/devcontainers-contrib/features/spicedb:1": {},
20+
"ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {
21+
"minikube": "none"
22+
},
23+
"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {
24+
"jqVersion": "latest",
25+
"yqVersion": "latest"
26+
},
27+
"ghcr.io/dhoeric/features/k9s:1": {},
28+
"ghcr.io/EliiseS/devcontainer-features/bash-profile:1": {
29+
"command": "alias k=kubectl"
30+
},
31+
"ghcr.io/devcontainers-contrib/features/rclone:1": {}
2032
},
21-
// Use 'forwardPorts' to make a list of ports inside the container available locally.
22-
// This can be used to network with other containers or the host.
23-
//"forwardPorts": [8000, 8080, 5432],
24-
// Use 'postCreateCommand' to run commands after the container is created.
25-
"postCreateCommand": "poetry self add poetry-polylith-plugin && poetry install --with dev && echo 'cd /workspaces/renku-data-services/'>~/.bashrc",
26-
// Configure tool-specific properties.
33+
"overrideFeatureInstallOrder": [
34+
"ghcr.io/devcontainers-contrib/features/poetry",
35+
"ghcr.io/devcontainers-contrib/features/bash-command"
36+
],
37+
"postCreateCommand": "poetry install --with dev",
2738
"customizations": {
28-
// Configure properties specific to VS Code.
2939
"vscode": {
30-
// Add the IDs of extensions you want installed when the container is created.
3140
"extensions": [
3241
"ms-python.python",
3342
"ms-python.flake8",
@@ -39,6 +48,4 @@
3948
]
4049
}
4150
}
42-
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
43-
//"remoteUser": "root"
4451
}

‎.devcontainer/docker-compose.yml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,29 @@ version: '3.8'
22

33
services:
44
data_service:
5-
build:
6-
context: ..
7-
dockerfile: projects/renku_data_service/Dockerfile
8-
args:
9-
- USER_UID=${USER_UID:-1000}
10-
- USER_GID=${USER_GID:-1000}
5+
image: "mcr.microsoft.com/devcontainers/python:3.12-bookworm"
116
volumes:
12-
- ../..:/workspaces:cached
7+
- ../:/workspace:cached
8+
- .poetry_cache/:/poetry_cache:delegated
139
# Overrides default command so things don't shut down after the process ends.
1410
entrypoint: sleep infinity
1511
environment:
1612
DB_USER: renku
1713
DB_NAME: renku
1814
DB_PASSWORD: renku
19-
DB_HOST: 127.0.0.1
15+
DB_HOST: 127.0.0.1
2016
CORS_ALLOW_ALL_ORIGINS: "true"
21-
ALEMBIC_CONFIG: /workspaces/renku-data-services/components/renku_data_services/migrations/alembic.ini
22-
AUTHZ_DB_HOST: 127.0.0.1
23-
AUTHZ_DB_GRPC_PORT: "50051"
24-
AUTHZ_DB_KEY: renku
17+
ALEMBIC_CONFIG: /workspace/components/renku_data_services/migrations/alembic.ini
18+
AUTHZ_DB_HOST: 127.0.0.1
19+
AUTHZ_DB_GRPC_PORT: "50051"
20+
AUTHZ_DB_KEY: renku
2521
AUTHZ_DB_NO_TLS_CONNECTION: "true"
2622
ZED_ENDPOINT: 127.0.0.1:50051
2723
ZED_TOKEN: renku
2824
ZED_INSECURE: "true"
25+
POETRY_CACHE_DIR: "/poetry_cache"
26+
NB_SERVER_OPTIONS__DEFAULTS_PATH: /workspace/server_defaults.json
27+
NB_SERVER_OPTIONS__UI_CHOICES_PATH: /workspace/server_options.json
2928
network_mode: service:db
3029
depends_on:
3130
- db
@@ -46,14 +45,15 @@ services:
4645
- "8080:8080"
4746
- "5678:5678"
4847
- "50051:50051"
48+
- "8888:80"
4949

5050
swagger:
5151
image: swaggerapi/swagger-ui
5252
environment:
5353
SWAGGER_JSON_URL: http://localhost:8000/api/data/spec.json
5454
PORT: "8080"
5555
network_mode: service:db
56-
56+
5757
authz:
5858
image: authzed/spicedb:latest-debug
5959
restart: unless-stopped

‎.github/dependabot.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,8 @@ updates:
1414
poetry:
1515
patterns:
1616
- "*"
17+
- package-ecosystem: "github-actions"
18+
directory: "/"
19+
schedule:
20+
interval: "weekly"
21+
day: "sunday"

‎.github/workflows/acceptance-tests.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@ jobs:
2525
renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
2626
renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
2727
renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
28+
amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}}
29+
amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
2830
test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
2931
test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}}
3032
persist: ${{ steps.deploy-comment.outputs.persist}}
3133
extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
3234
steps:
3335
- id: deploy-comment
34-
uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.11.3
36+
uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.1
3537
with:
3638
string: /deploy
3739
pr_ref: ${{ github.event.number }}
@@ -44,7 +46,7 @@ jobs:
4446
name: renku-ci-ds-${{ github.event.number }}
4547
url: https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
4648
steps:
47-
- uses: actions/checkout@v3
49+
- uses: actions/checkout@v4
4850
- name: Find deplyoment url
4951
uses: peter-evans/find-comment@v1
5052
id: deploymentUrlMessage
@@ -61,7 +63,7 @@ jobs:
6163
body: |
6264
You can access the deployment of this PR at https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
6365
- name: Build and deploy
64-
uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.11.3
66+
uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.1
6567
env:
6668
RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }}
6769
DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
@@ -84,14 +86,16 @@ jobs:
8486
renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
8587
renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
8688
renku_data_services: "@${{ github.head_ref }}"
89+
amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
90+
amalthea-sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}"
8791
extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
8892

8993
selenium-acceptance-tests:
9094
needs: [check-deploy, deploy-pr]
9195
if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'
9296
runs-on: ubuntu-22.04
9397
steps:
94-
- uses: SwissDataScienceCenter/renku-actions/test-renku@v1.11.3
98+
- uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1
9599
with:
96100
kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
97101
renku-release: renku-ci-ds-${{ github.event.number }}
@@ -118,7 +122,7 @@ jobs:
118122
steps:
119123
- name: Extract Renku repository reference
120124
run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV
121-
- uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.11.3
125+
- uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.1
122126
with:
123127
e2e-target: ${{ matrix.tests }}
124128
renku-reference: ${{ env.RENKU_REFERENCE }}
@@ -146,7 +150,7 @@ jobs:
146150
body: |
147151
Tearing down the temporary RenkuLab deplyoment for this PR.
148152
- name: renku teardown
149-
uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.11.3
153+
uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1
150154
env:
151155
HELM_RELEASE_REGEX: "^renku-ci-ds-${{ github.event.number }}$"
152156
GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}

‎.github/workflows/save_cache.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: Create cache from commits on main
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- chore-add-kind
8+
workflow_dispatch:
9+
10+
11+
jobs:
12+
save-poetry-cache:
13+
runs-on: ubuntu-latest
14+
env:
15+
CACHE_KEY: main-branch-poetry-cache-ubuntu
16+
CACHE_PATH: .devcontainer/.poetry_cache
17+
DEVCONTAINER_IMAGE_CACHE: ghcr.io/swissdatasciencecenter/renku-data-services/devcontainer
18+
19+
steps:
20+
- uses: actions/checkout@v3
21+
with:
22+
fetch-depth: 0
23+
- name: Login to Docker Hub
24+
uses: docker/login-action@v2
25+
with:
26+
registry: ghcr.io
27+
username: ${{ github.actor }}
28+
password: ${{ secrets.GITHUB_TOKEN }}
29+
- name: Install python deps
30+
uses: devcontainers/ci@v0.3
31+
with:
32+
runCmd: poetry install --with dev
33+
push: always
34+
skipContainerUserIdUpdate: false
35+
imageName: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
36+
cacheFrom: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
37+
- uses: actions/cache/save@v3
38+
name: Create cache
39+
with:
40+
path: ${{ env.CACHE_PATH }}
41+
key: ${{ env.CACHE_KEY }}

0 commit comments

Comments
 (0)