Skip to content

Commit 612d4ec

Browse files
author
DataLens Team
committed
Initial commit
0 parents  commit 612d4ec

2,414 files changed

Lines changed: 226599 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build_action_runner_image_base.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: build_gh_actions_image_base
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "04 05 * * 6"
7+
8+
jobs:
9+
build:
10+
runs-on: [ self-hosted, linux ]
11+
permissions:
12+
packages: write
13+
steps:
14+
- name: 'Make tmp dir'
15+
run: mkdir -p build_context
16+
- run: |
17+
echo "
18+
FROM debian:bookworm
19+
RUN export DEBIAN_FRONTEND=noninteractive && \
20+
apt-get update && \
21+
apt-get install --yes curl gpg ca-certificates && \
22+
install -m 0755 -d /etc/apt/keyrings && \
23+
curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
24+
chmod a+r /etc/apt/keyrings/docker.gpg && \
25+
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian/ bookworm stable" > /etc/apt/sources.list.d/docker.list && \
26+
apt-get update && \
27+
apt-get install --yes docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && \
28+
apt-get clean && \
29+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
30+
" > build_context/Dockerfile
31+
- name: Login to GitHub Container Registry
32+
uses: docker/login-action@v2
33+
with:
34+
registry: ghcr.io
35+
username: ${{ github.actor }}
36+
password: ${{ secrets.GITHUB_TOKEN }}
37+
- name: Build and push
38+
uses: docker/build-push-action@v4
39+
with:
40+
context: build_context/
41+
push: true
42+
tags: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest"
43+
- name: cleanup
44+
if: always()
45+
run: rm -rf build_context

‎.github/workflows/build_apps.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: build_apps
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
bake_applications:
8+
runs-on: [ self-hosted, linux ]
9+
permissions:
10+
packages: write
11+
contents: read
12+
container:
13+
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest"
14+
options: -v /var/run/docker.sock:/var/run/docker.sock
15+
credentials:
16+
username: ${{ github.actor }}
17+
password: ${{ secrets.GITHUB_TOKEN }}
18+
env:
19+
CR_TAG_BASE: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}"
20+
IMAGE_NAME_CONTROL_API: "dl-control-api"
21+
IMAGE_NAME_DATA_API: "dl-data-api"
22+
BAKE_TARGET_CONTROL_API_APP: "dl_control_api"
23+
BAKE_TARGET_DATA_API_APP: "dl_data_api"
24+
steps:
25+
- name: Log in to the Container registry
26+
uses: docker/login-action@v2
27+
with:
28+
registry: "ghcr.io"
29+
username: ${{ github.actor }}
30+
password: ${{ secrets.GITHUB_TOKEN }}
31+
- name: 'Cleanup build folder'
32+
run: |
33+
rm -rf ./* || true
34+
rm -rf ./.??* || true
35+
- name: Checkout code
36+
uses: actions/checkout@v4
37+
- run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
38+
- run: |
39+
cd docker_build
40+
./run-project-bake "${BAKE_TARGET_CONTROL_API_APP}" --push --set "${BAKE_TARGET_CONTROL_API_APP}.tags=${CR_TAG_BASE}/${IMAGE_NAME_CONTROL_API}:latest" --set "${BAKE_TARGET_CONTROL_API_APP}.tags=${CR_TAG_BASE}/${IMAGE_NAME_CONTROL_API}:$(git rev-parse HEAD)"
41+
./run-project-bake "${BAKE_TARGET_DATA_API_APP}" --push --set "${BAKE_TARGET_DATA_API_APP}.tags=${CR_TAG_BASE}/${IMAGE_NAME_DATA_API}:latest" --set "${BAKE_TARGET_DATA_API_APP}.tags=${CR_TAG_BASE}/${IMAGE_NAME_DATA_API}:$(git rev-parse HEAD)"

‎.github/workflows/main.yml‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
name: main
2+
3+
concurrency:
4+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
5+
cancel-in-progress: true
6+
7+
on:
8+
pull_request:
9+
workflow_dispatch:
10+
inputs:
11+
test_targets:
12+
default: ""
13+
description: "Comma separated list to run pytest on, e.g.: `lib/dl_api_lib,lib/dl_core`. "
14+
run_mypy_only:
15+
type: boolean
16+
default: false
17+
description: "Check to only run mypy"
18+
pytest_timeout_minutes:
19+
type: number
20+
default: 10
21+
description: "Timeout for pytest JOB in minutes"
22+
23+
jobs:
24+
gh_build_image:
25+
runs-on: [ self-hosted, linux ]
26+
permissions:
27+
packages: write
28+
contents: read
29+
container:
30+
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/debian_docker:latest"
31+
options: -v /var/run/docker.sock:/var/run/docker.sock
32+
credentials:
33+
username: ${{ github.actor }}
34+
password: ${{ secrets.GITHUB_TOKEN }}
35+
env:
36+
REPO_OWNER: ${{ github.repository_owner }}
37+
REPO_NAME: ${{ github.event.repository.name }}
38+
GIT_SHA: "${{ github.sha }}"
39+
steps:
40+
- name: Log in to the Container registry
41+
uses: docker/login-action@v2
42+
with:
43+
registry: "ghcr.io"
44+
username: ${{ github.actor }}
45+
password: ${{ secrets.GITHUB_TOKEN }}
46+
- name: 'Cleanup build folder'
47+
run: |
48+
rm -rf ./* || true
49+
rm -rf ./.??* || true
50+
- name: Checkout code
51+
uses: actions/checkout@v4
52+
- run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
53+
- run: |
54+
export ROOT_DIR="$(realpath .)"
55+
/bin/bash ci/build_naive.sh
56+
57+
router:
58+
runs-on: [self-hosted, linux, light]
59+
needs: gh_build_image
60+
container:
61+
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
62+
credentials:
63+
username: ${{ github.actor }}
64+
password: ${{ secrets.GITHUB_TOKEN }}
65+
outputs:
66+
affected: ${{ steps.get_affected.outputs.affected }}
67+
steps:
68+
- name: 'Cleanup build folder'
69+
run: |
70+
rm -rf ./* || true
71+
rm -rf ./.??* || true
72+
- name: Checkout code
73+
uses: actions/checkout@v4
74+
with:
75+
fetch-depth: 0
76+
- run: git config --global --add safe.directory .
77+
- run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
78+
- run: git fetch origin main
79+
- name: Get packages affected by changes in the current commit
80+
id: get_affected
81+
run: |
82+
TARGET_DATA=$(bash /src/ci/gh_list_changes.sh ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})
83+
echo "$TARGET_DATA" > /tmp/targets.json
84+
cat /tmp/targets.json
85+
. /venv/bin/activate && detect-affected-packages \
86+
--repo=/src \
87+
--changes_file="/tmp/targets.json" \
88+
--root_pkgs="lib,app,terrarium" \
89+
--fallback_pkg="terrarium/bi_ci" | tee -a >> "$GITHUB_OUTPUT"
90+
env:
91+
TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }}
92+
93+
pytest_split:
94+
runs-on: [ self-hosted, linux ]
95+
needs: router
96+
# if: ${{ github.event.inputs.run_mypy_only != 'true' }}
97+
container:
98+
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
99+
credentials:
100+
username: ${{ github.actor }}
101+
password: ${{ secrets.GITHUB_TOKEN }}
102+
outputs:
103+
split_base: ${{ steps.get_split.outputs.split_base }}
104+
split_fat: ${{ steps.get_split.outputs.split_fat }}
105+
steps:
106+
- name: 'Cleanup build folder'
107+
run: |
108+
rm -rf ./* || true
109+
rm -rf ./.??* || true
110+
- name: Checkout code
111+
uses: actions/checkout@v4
112+
with:
113+
fetch-depth: 1
114+
- run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
115+
- name: Run python script to split job for general and fat runners
116+
id: get_split
117+
run: |
118+
. /venv/bin/activate && echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json
119+
split-pytest-tasks base /src /tmp/dl_test_targets.json | tee -a >> "$GITHUB_OUTPUT"
120+
split-pytest-tasks fat /src /tmp/dl_test_targets.json | tee -a >> "$GITHUB_OUTPUT"
121+
122+
run_tests_base:
123+
runs-on: [ self-hosted, linux, light ]
124+
name: "🐍[pytest]${{ matrix.value }}"
125+
needs: pytest_split
126+
if: ${{ needs.pytest_split.outputs.split_base != '[]' }}
127+
permissions:
128+
packages: write
129+
contents: read
130+
container:
131+
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
132+
options: -v /var/run/docker.sock:/var/run/docker.sock
133+
credentials:
134+
username: ${{ github.actor }}
135+
password: ${{ secrets.GITHUB_TOKEN }}
136+
volumes:
137+
- /var/run/docker.sock:/var/run/docker.sock
138+
strategy:
139+
fail-fast: false
140+
matrix:
141+
value: ${{fromJson(needs.pytest_split.outputs.split_base)}}
142+
steps:
143+
- run: echo "Going to run tests for ${{ matrix.value }}"
144+
- run: mkdir /report
145+
- run: echo "Running py tests for ${{ matrix.value }}"
146+
- name: Log in to the Container registry
147+
uses: docker/login-action@v2
148+
with:
149+
registry: "ghcr.io"
150+
username: ${{ github.actor }}
151+
password: ${{ secrets.GITHUB_TOKEN }}
152+
- run: echo compose_path="/src/$(echo ${{ matrix.value }} | cut -d ":" -f1)/" >> "$GITHUB_ENV"
153+
- run: cd "${{ env.compose_path }}" && echo compose_prj="$(basename "$PWD")_$(shuf -i 1000000-1000000000 -n 1)" >> "$GITHUB_ENV"
154+
# We need to set custom compose project name to ensure "unique" container names in the host docker env
155+
- name: run bash script with all logic for starting compose and running tests
156+
run: |
157+
bash /src/ci/execute_test_with_docker_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}" \
158+
WE_ARE_IN_CI=1
159+
- name: Stop compose if provided
160+
# We could not put this into bash script, since job could be cancelled by user request
161+
if: always() # yes! always
162+
run: bash /src/ci/stop_compose.sh "${{ matrix.value }}" "${{ job.container.network }}" "${{ env.compose_prj }}"
163+
- uses: actions/upload-artifact@v3
164+
if: "!cancelled()"
165+
with:
166+
name: "pytest_reports_${{ env.compose_prj }}"
167+
path: /report/
168+
retention-days: 1
169+
170+
mypy:
171+
runs-on: [ self-hosted, linux, light ]
172+
needs: router
173+
container:
174+
# until https://github.com/github/docs/issues/25520 is resolved, using vars
175+
image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
176+
credentials:
177+
username: ${{ github.actor }}
178+
password: ${{ secrets.GITHUB_TOKEN }}
179+
volumes:
180+
- /var/run/docker.sock:/var/run/docker.sock
181+
name: "🐲 mypy"
182+
timeout-minutes: ${{ inputs.pytest_timeout_minutes && fromJSON(inputs.pytest_timeout_minutes ) || 10 }}
183+
steps:
184+
- name: "Cleanup folder"
185+
run: |
186+
rm -rf ./* || true
187+
rm -rf ./.??* || true
188+
- name: Checkout code, just to get access to .github/execute_mypy...
189+
uses: actions/checkout@v4
190+
with:
191+
fetch-depth: 1
192+
- run: echo '${{ needs.router.outputs.affected }}' >> /tmp/dl_test_targets.json
193+
- name: run mypy
194+
run: |
195+
. /venv/bin/activate
196+
execute-mypy-multi /src /tmp/dl_test_targets.json
197+
env:
198+
PYTHONUNBUFFERED: "1"
199+
200+
publish-result:
201+
runs-on: [self-hosted, linux ]
202+
needs: [ "run_tests_base" ] # , "run_tests_fat" ]
203+
if: "!cancelled() && ${{ github.event.inputs.run_mypy_only != 'true' }}"
204+
permissions:
205+
contents: read
206+
issues: read
207+
checks: write
208+
pull-requests: write
209+
container: docker:latest
210+
steps:
211+
- uses: actions/download-artifact@v3
212+
with:
213+
path: ./report/
214+
- run: ls -lah ./report
215+
- name: Publish Test Results
216+
uses: datalens-tech/publish-unit-test-result-action@55478522536e0c60b0a4ff0c2bb8ab110d7a0f33
217+
with:
218+
files: |
219+
./report/**/*.xml
220+
event_name: ${{ github.event.workflow_run.event }}
221+
report_individual_runs: "true"

‎.gitignore‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
*.egg-info
2+
.venv
3+
build
4+
.task
5+
.tox
6+
.env
7+
.tmp.env*
8+
.coverage
9+
.python-version
10+
*.ipynb
11+
.ruff_cache
12+
.idea
13+
__pycache__
14+
ci_artifacts
15+
.DS_Store
16+
artifacts
17+
Taskfile.yml

‎CODE_OF_CONDUCT.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
We welcome everyone to contribute to our product, see [CONTRIBUTING.md](CONTRIBUTING.md).

‎CONTRIBUTING.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Notice to external contributors
2+
3+
## General info
4+
5+
Hello! In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA can be found here https://yandex.ru/legal/cla/?lang=en
6+
7+
By adopting the CLA, you state the following:
8+
9+
- You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
10+
- You have read the terms and conditions of the CLA and agree with them in full,
11+
- You are legally able to provide and license your contributions as stated,
12+
- We may use your contributions for our open source projects and for any other our project too,
13+
- We rely on your assurances concerning the rights of third parties in relation to your contributions.
14+
15+
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
16+
17+
## Provide contributions
18+
19+
If you have already adopted terms and conditions of the CLA, you are able to provide your contributions. When you submit your pull request, please add the following information into it:
20+
21+
```
22+
I hereby agree to the terms of the CLA available at: [link].
23+
```
24+
25+
Replace the bracketed text as follows:
26+
27+
- [link] is the link to the current version of the CLA: https://yandex.ru/legal/cla/?lang=en.
28+
29+
It is enough to provide us such notification once.
30+
31+
## Other questions
32+
33+
If you have any questions, please mail us at datalens-opensource@yandex-team.ru.

0 commit comments

Comments
 (0)