Skip to content

Commit 1ad2b1e

Browse files
antoninkrizantoninkriz
committed
Attempt at new semi-manual data loading method
1 parent a188408 commit 1ad2b1e

9 files changed

Lines changed: 1296 additions & 630 deletions

File tree

‎.github/actions/setup-python-poetry/action.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
name: Setup Python and Poetry
2+
description: "Set up Python and Poetry environment"
23

34
inputs:
45
python-version:

‎.github/workflows/ci-python.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ jobs:
2222
- name: Setup
2323
uses: ./.github/actions/setup-python-poetry
2424
with:
25-
python-version: 3.13.1
26-
poetry-version: 2.0.1
25+
python-version: 3.14.2
26+
poetry-version: 2.3.1
2727
path: kos-loader
2828
install-dev: true
2929

@@ -46,8 +46,8 @@ jobs:
4646
- name: Setup
4747
uses: ./.github/actions/setup-python-poetry
4848
with:
49-
python-version: 3.13.1
50-
poetry-version: 2.0.1
49+
python-version: 3.14.2
50+
poetry-version: 2.3.1
5151
path: kos-loader
5252
install-dev: true
5353

@@ -65,8 +65,8 @@ jobs:
6565
- name: Setup
6666
uses: ./.github/actions/setup-python-poetry
6767
with:
68-
python-version: 3.13.1
69-
poetry-version: 2.0.1
68+
python-version: 3.14.2
69+
poetry-version: 2.3.1
7070
path: kos-loader
7171
install-dev: true
7272

‎.github/workflows/fetch-data.yml‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,25 @@
11
name: Fetch data
22

3-
# on:
4-
# workflow_dispatch:
5-
# schedule:
6-
# - cron: '0 */6 * * *'
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
jsessionid_enc:
7+
description: "Encrypted base64 cookie value for JSESSIONID"
8+
required: true
9+
type: string
10+
xsrf_token_enc:
11+
description: "Encrypted base64 cookie value for XSRF-TOKEN"
12+
required: true
13+
type: string
14+
bigipserverfarm_is_stud_newkos_https_enc:
15+
description: "Encrypted base64 cookie value for BIGipServerFARM-IS-STUD-NEWKOS-HTTPS"
16+
required: false
17+
type: string
18+
msal_cache_encryption_enc:
19+
description: "Encrypted base64 cookie value for msal.cache.encryption"
20+
required: false
21+
type: string
22+
723

824
jobs:
925
fetch-data:
@@ -13,6 +29,9 @@ jobs:
1329
contents: write
1430
strategy:
1531
fail-fast: true
32+
matrix:
33+
python-version: [3.14.2]
34+
poetry-version: [2.3.1]
1635

1736
steps:
1837
- name: Check out repository
@@ -21,16 +40,19 @@ jobs:
2140
- name: Setup
2241
uses: ./.github/actions/setup-python-poetry
2342
with:
24-
python-version: 3.13.1
25-
poetry-version: 2.0.1
43+
python-version: ${{ matrix.python-version }}
44+
poetry-version: ${{ matrix.poetry-version }}
2645
path: kos-loader
2746

2847
- name: Fetch data
2948
working-directory: kos-loader
3049
shell: bash
3150
env:
32-
KOS_USERNAME: ${{ secrets.KOS_USERNAME }}
33-
KOS_PASSWORD: ${{ secrets.KOS_PASSWORD }}
51+
COOKIES_PRIVATE_KEY: ${{ secrets.COOKIES_PRIVATE_KEY }}
52+
JSESSIONID_ENC: ${{ github.event.inputs.jsessionid_enc }}
53+
XSRF_TOKEN_ENC: ${{ github.event.inputs.xsrf_token_enc }}
54+
BIGIPSERVERFARM_IS_STUD_NEWKOS_HTTPS_ENC: ${{ github.event.inputs.bigipserverfarm_is_stud_newkos_https_enc }}
55+
MSAL_CACHE_ENCRYPTION_ENC: ${{ github.event.inputs.msal_cache_encryption_enc }}
3456
LOGLEVEL: info
3557
run: poetry run python3 -m kos_loader.kos_loader build/data.json
3658

‎kos-loader/README.md‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,39 @@ This tool downloads data from the undocumented private KOS API using a using tra
1515

1616
## Usage
1717

18+
Generate public and private keys.
19+
20+
```bash
21+
openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out private_key.pem
22+
openssl rsa -in private_key.pem -pubout -out public_key.pem
23+
```
24+
25+
Update the `PUB_KEY_PEM` variable in `encrypt_cookie.js`.
26+
27+
Get cookies from browser:
28+
* 1. Open kos.cvut.cz in browser
29+
* 2. Login
30+
* 3. Open developer console (F12)
31+
* 4. Open Application tab -> Cookies -> https://kos.cvut.cz
32+
* 5. Open Console
33+
* 6. Paste this code in console
34+
* 7. Run `await encryptCookie('<YOUR_COOKIE_VALUE>')` with copy-pasted cookie values for cookies:
35+
* - JSESSIONID
36+
* - XSRF-TOKEN
37+
* 8. Copy the output values and use them as encrypted cookies in kos-loader
38+
39+
1840
```bash
19-
KOS_USERNAME=your_username
20-
KOS_PASSWORD=********
21-
LOGLEVEL=info
22-
python3 -m kos_loader.kos_loader
41+
export COOKIES_PRIVATE_KEY=private key in PEM format
42+
43+
export JSESSIONID_ENC=encrypted and base64 encoded cookie value
44+
export XSRF_TOKEN_ENC=...
45+
export BIGIPSERVERFARM_IS_STUD_NEWKOS_HTTPS_ENC=...
46+
export MSAL_CACHE_ENCRYPTION_ENC=...
47+
48+
export LOGLEVEL=info
49+
50+
python3 -m kos_loader.kos_loader
2351
```
2452

2553

‎kos-loader/encrypt_cookie.js‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/**
2+
* Cookie encryption from KOS frontend
3+
*
4+
* How to use:
5+
* 1. Open kos.cvut.cz in browser
6+
* 2. Login
7+
* 3. Open developer console (F12)
8+
* 4. Open Application tab -> Cookies -> https://kos.cvut.cz
9+
* 5. Open Console
10+
* 6. Paste this code in console
11+
* 7. Run `await encryptCookie('<YOUR_COOKIE_VALUE>')` with copy-pasted cookie values for cookies:
12+
* - JSESSIONID
13+
* - XSRF-TOKEN
14+
* 8. Copy the output values and use them as encrypted cookies in kos-loader
15+
*/
16+
17+
/**
18+
* RSA Public Key in PEM format
19+
*/
20+
const PUB_KEY_PEM = `
21+
-----BEGIN PUBLIC KEY-----
22+
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2BGOAQqMQ0Uda3jweNM0
23+
uMiWVHDy0vipzSyvYiScmv0eU5bLVMClH532r2AeCk8rDfk6t3mUrtsL89P8TVqF
24+
gB6a8xnojybOeVuMjKCHgXB+pTwfEzgROK35oXgXD158ClZ9aEtEtoUXrTSYa2Z6
25+
iqXh/pzhNC16z+Z2O4mZffBt5X6Fhd+uF/NDQJijh8mnLLBaT20E/MNxkehzL6QO
26+
ycQUEEemP1FG0v7Ipn9z1V0zz82Mqjdm5UEfaDKm3PbIB9Y+em2abnwmOqcvAl82
27+
fmW18zjW9882EwFcSpj9auT5GC4fdnEITJjc+8lo702EPrZcKXytNR/lDmFHOvN5
28+
5wIDAQAB
29+
-----END PUBLIC KEY-----
30+
`
31+
32+
/**
33+
* Encrypt plaintext with an RSA public key (PEM) using RSA-OAEP SHA-256.
34+
*
35+
* @param {string} plaintext - String to encrypt
36+
* @returns {Promise<string>} Base64-encoded ciphertext
37+
*/
38+
async function encryptCookie(plaintext) {
39+
const pemBody = PUB_KEY_PEM
40+
.trim()
41+
.replace(/-----BEGIN PUBLIC KEY-----/, "")
42+
.replace(/-----END PUBLIC KEY-----/, "")
43+
.replace(/\s+/g, "")
44+
45+
const keyDer = Uint8Array.fromBase64(pemBody)
46+
47+
const publicKey = await crypto.subtle.importKey(
48+
"spki",
49+
keyDer,
50+
{
51+
name: "RSA-OAEP",
52+
hash: "SHA-256",
53+
},
54+
false,
55+
["encrypt"],
56+
)
57+
58+
const encoded = new TextEncoder().encode(plaintext)
59+
60+
const ciphertext = await crypto.subtle.encrypt(
61+
{ name: "RSA-OAEP" },
62+
publicKey,
63+
encoded,
64+
)
65+
66+
return new Uint8Array(ciphertext).toBase64()
67+
}

‎kos-loader/kos_loader/consts.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
"""Constants"""
22

33
KOS_HOST = "kos.cvut.cz"
4-
KOS_URL = f"https://{KOS_HOST}"
5-
KOS_REST = f"{KOS_URL}/rest"
6-
KOS_API = f"{KOS_REST}/api"
4+
KOS_API = f"https://{KOS_HOST}/rest/api"
75

86
PAGINATION = 1000

0 commit comments

Comments
 (0)