Skip to content

Commit d868b94

Browse files
RawannadaRawannada
authored
Add files via upload
1 parent 4389e38 commit d868b94

1 file changed

Lines changed: 301 additions & 0 deletions

File tree

‎examples/ingest-from-rest-api-example/VDK_Ingest_from_REST_API_Example.ipynb‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
{
2+
"nbformat": 4,
3+
"nbformat_minor": 0,
4+
"metadata": {
5+
"colab": {
6+
"provenance": []
7+
},
8+
"kernelspec": {
9+
"name": "python3",
10+
"display_name": "Python 3"
11+
},
12+
"language_info": {
13+
"name": "python"
14+
}
15+
},
16+
"cells": [
17+
{
18+
"cell_type": "markdown",
19+
"source": [
20+
"**Install VDK and the SQLite plugin**"
21+
],
22+
"metadata": {
23+
"id": "LUK5BrjelQcX"
24+
}
25+
},
26+
{
27+
"cell_type": "code",
28+
"execution_count": 1,
29+
"metadata": {
30+
"colab": {
31+
"base_uri": "https://localhost:8080/"
32+
},
33+
"id": "1mAnQLHEk4uz",
34+
"outputId": "8b841e80-8931-4849-d73a-6af5f01e5c12"
35+
},
36+
"outputs": [
37+
{
38+
"output_type": "stream",
39+
"name": "stdout",
40+
"text": [
41+
"Collecting vdk-core\n",
42+
" Downloading vdk_core-0.3.1466514302.tar.gz (119 kB)\n",
43+
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.1/119.1 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
44+
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
45+
"Collecting vdk-sqlite\n",
46+
" Downloading vdk_sqlite-0.1.1431637373.tar.gz (7.4 kB)\n",
47+
" Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
48+
"Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (2.32.4)\n",
49+
"Requirement already satisfied: click==8.* in /usr/local/lib/python3.12/dist-packages (from vdk-core) (8.3.1)\n",
50+
"Requirement already satisfied: pluggy in /usr/local/lib/python3.12/dist-packages (from vdk-core) (1.6.0)\n",
51+
"Collecting click_log (from vdk-core)\n",
52+
" Downloading click_log-0.4.0-py2.py3-none-any.whl.metadata (1.2 kB)\n",
53+
"Requirement already satisfied: click-plugins in /usr/local/lib/python3.12/dist-packages (from vdk-core) (1.1.1.2)\n",
54+
"Requirement already satisfied: tenacity in /usr/local/lib/python3.12/dist-packages (from vdk-core) (9.1.2)\n",
55+
"Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from vdk-core) (75.2.0)\n",
56+
"Requirement already satisfied: tabulate in /usr/local/lib/python3.12/dist-packages (from vdk-sqlite) (0.9.0)\n",
57+
"Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests) (3.4.4)\n",
58+
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests) (3.11)\n",
59+
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests) (2.5.0)\n",
60+
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests) (2025.11.12)\n",
61+
"Downloading click_log-0.4.0-py2.py3-none-any.whl (4.3 kB)\n",
62+
"Building wheels for collected packages: vdk-core, vdk-sqlite\n",
63+
" Building wheel for vdk-core (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
64+
" Created wheel for vdk-core: filename=vdk_core-0.3.1466514302-py2.py3-none-any.whl size=163460 sha256=4c11c764993c7f81827f3ccc2de2f74f50cd4e6ab68b291f881a99b2060eedba\n",
65+
" Stored in directory: /root/.cache/pip/wheels/ee/3f/c8/010983f3969c74948165dbfb7ab7621cdf4ff619cc0aca83c2\n",
66+
" Building wheel for vdk-sqlite (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
67+
" Created wheel for vdk-sqlite: filename=vdk_sqlite-0.1.1431637373-py3-none-any.whl size=7065 sha256=c33784b1cbd3b9ba9e8dc4df163d952762180c7a24c8e9fe6934514da0257c90\n",
68+
" Stored in directory: /root/.cache/pip/wheels/1e/fa/b3/11d78000303a2c009ad8ca4679a5b8b210a9074ae78956f045\n",
69+
"Successfully built vdk-core vdk-sqlite\n",
70+
"Installing collected packages: click_log, vdk-core, vdk-sqlite\n",
71+
"Successfully installed click_log-0.4.0 vdk-core-0.3.1466514302 vdk-sqlite-0.1.1431637373\n"
72+
]
73+
}
74+
],
75+
"source": [
76+
"!pip install vdk-core vdk-sqlite requests"
77+
]
78+
},
79+
{
80+
"cell_type": "markdown",
81+
"source": [
82+
"**Set Environment Variables for SQLite Ingestion**"
83+
],
84+
"metadata": {
85+
"id": "sjbcK6rol8ia"
86+
}
87+
},
88+
{
89+
"cell_type": "code",
90+
"source": [
91+
"import os\n",
92+
"os.environ[\"VDK_DB_DEFAULT_TYPE\"] = \"SQLITE\"\n",
93+
"os.environ[\"VDK_INGEST_METHOD_DEFAULT\"] = \"SQLITE\""
94+
],
95+
"metadata": {
96+
"id": "Lh7kxenylKy2"
97+
},
98+
"execution_count": 2,
99+
"outputs": []
100+
},
101+
{
102+
"cell_type": "code",
103+
"source": [
104+
"%%writefile 10_delete_table.sql\n",
105+
"DROP TABLE IF EXISTS rest_target_table;"
106+
],
107+
"metadata": {
108+
"colab": {
109+
"base_uri": "https://localhost:8080/"
110+
},
111+
"id": "olUlK83QlK2i",
112+
"outputId": "79849252-933e-4c5b-f174-7fd1d2553a67"
113+
},
114+
"execution_count": 3,
115+
"outputs": [
116+
{
117+
"output_type": "stream",
118+
"name": "stdout",
119+
"text": [
120+
"Writing 10_delete_table.sql\n"
121+
]
122+
}
123+
]
124+
},
125+
{
126+
"cell_type": "code",
127+
"source": [
128+
"%%writefile 20_create_table.sql\n",
129+
"CREATE TABLE rest_target_table (userId, id, title, completed);"
130+
],
131+
"metadata": {
132+
"colab": {
133+
"base_uri": "https://localhost:8080/"
134+
},
135+
"id": "CZ7Q07FumJb9",
136+
"outputId": "0f85ce65-94f5-454b-aeea-31779cf1d8b6"
137+
},
138+
"execution_count": 4,
139+
"outputs": [
140+
{
141+
"output_type": "stream",
142+
"name": "stdout",
143+
"text": [
144+
"Writing 20_create_table.sql\n"
145+
]
146+
}
147+
]
148+
},
149+
{
150+
"cell_type": "code",
151+
"source": [
152+
"%%writefile 30_rest_ingest.py\n",
153+
"import requests\n",
154+
"\n",
155+
"def run(job_input):\n",
156+
" # Fetching from the API\n",
157+
" response = requests.get(\"https://jsonplaceholder.typicode.com/todos/1\")\n",
158+
" response.raise_for_status()\n",
159+
" payload = response.json()\n",
160+
"\n",
161+
" # Sending data to the target database\n",
162+
" job_input.send_object_for_ingestion(\n",
163+
" payload=payload,\n",
164+
" destination_table=\"rest_target_table\"\n",
165+
" )"
166+
],
167+
"metadata": {
168+
"colab": {
169+
"base_uri": "https://localhost:8080/"
170+
},
171+
"id": "ToVbdX5DmMqT",
172+
"outputId": "2177fd96-9ce2-442a-8796-cf9a5509baf1"
173+
},
174+
"execution_count": 5,
175+
"outputs": [
176+
{
177+
"output_type": "stream",
178+
"name": "stdout",
179+
"text": [
180+
"Writing 30_rest_ingest.py\n"
181+
]
182+
}
183+
]
184+
},
185+
{
186+
"cell_type": "code",
187+
"source": [
188+
"!vdk run ."
189+
],
190+
"metadata": {
191+
"colab": {
192+
"base_uri": "https://localhost:8080/"
193+
},
194+
"id": "xIaKrG5rmQ50",
195+
"outputId": "1b0a6882-7eff-49e5-bab9-b431676e1b6c"
196+
},
197+
"execution_count": 6,
198+
"outputs": [
199+
{
200+
"output_type": "stream",
201+
"name": "stdout",
202+
"text": [
203+
"\n",
204+
"Versatile Data Kit (VDK)\n",
205+
"Version: 0.3.1466514302\n",
206+
"Build details: RELEASE_VERSION=0.3.1466514302, BUILD_DATE=Tue Sep 24 08:50:56 UTC 2024, BUILD_MACHINE_INFO=Linux runner--azerasq-project-28359933-concurrent-0 5.15.154+ #1 SMP Sat May 4 12:14:42 UTC 2024 x86_64 GNU/Linux, GITLAB_CI_JOB_ID=7902585266, GIT_COMMIT_SHA=880df089916daa16d3fe5fbe789b81dd6099911f, GIT_BRANCH=main\n",
207+
"Python version: 3.12.12 64bit (/usr/bin/python3)\n",
208+
"\n",
209+
"Installed plugins:\n",
210+
"vdk-sqlite (from package vdk-sqlite, version 0.1.1431637373)\n",
211+
"--------------------------------------------------------------------------------\n",
212+
"Run job with directory /content\n",
213+
"Missing config.ini file.\n",
214+
"2026-01-10 01:50:42,039 [VDK] content [INFO ] vdk.plugin.sqlite.sqlite_conne sqlite_connection.py:29 new_connection [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Creating new connection against local file database located at: /tmp/vdk-sqlite.db\n",
215+
"2026-01-10 01:50:42,039 [VDK] content [INFO ] vdk.plugin.sqlite.sqlite_conne sqlite_connection.py:29 new_connection [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Creating new connection against local file database located at: /tmp/vdk-sqlite.db\n",
216+
"2026-01-10 01:50:42,040 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:201 _execute_operati[id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Executing query:\n",
217+
"-- job_name: content\n",
218+
"-- op_id: 317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841\n",
219+
"DROP TABLE IF EXISTS rest_target_table;\n",
220+
"\n",
221+
"2026-01-10 01:50:42,041 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:103 execute [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Executing query SUCCEEDED. Query duration 00h:00m:00s\n",
222+
"2026-01-10 01:50:42,041 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_connection_b:135 execute_query [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Fetching query result...\n",
223+
"2026-01-10 01:50:42,041 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:239 fetchall [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Fetching all results from query ...\n",
224+
"2026-01-10 01:50:42,041 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:242 fetchall [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Fetching all results from query SUCCEEDED.\n",
225+
"2026-01-10 01:50:42,041 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:249 close [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Closing DB cursor ...\n",
226+
"2026-01-10 01:50:42,041 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:251 close [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Closing DB cursor SUCCEEDED.\n",
227+
"2026-01-10 01:50:42,042 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:201 _execute_operati[id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Executing query:\n",
228+
"-- job_name: content\n",
229+
"-- op_id: 317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841\n",
230+
" select 1 -- Testing if connection is alive.\n",
231+
"2026-01-10 01:50:42,042 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:103 execute [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Executing query SUCCEEDED. Query duration 00h:00m:00s\n",
232+
"2026-01-10 01:50:42,042 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:201 _execute_operati[id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Executing query:\n",
233+
"-- job_name: content\n",
234+
"-- op_id: 317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841\n",
235+
"CREATE TABLE rest_target_table (userId, id, title, completed);\n",
236+
"\n",
237+
"2026-01-10 01:50:42,055 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:103 execute [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Executing query SUCCEEDED. Query duration 00h:00m:00s\n",
238+
"2026-01-10 01:50:42,055 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_connection_b:135 execute_query [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Fetching query result...\n",
239+
"2026-01-10 01:50:42,055 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:239 fetchall [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Fetching all results from query ...\n",
240+
"2026-01-10 01:50:42,055 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:242 fetchall [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Fetching all results from query SUCCEEDED.\n",
241+
"2026-01-10 01:50:42,055 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:249 close [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Closing DB cursor ...\n",
242+
"2026-01-10 01:50:42,055 [VDK] content [INFO ] vdk.internal.builtin_plugins.c managed_cursor.py:251 close [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Closing DB cursor SUCCEEDED.\n",
243+
"2026-01-10 01:50:42,262 [VDK] content [INFO ] vdk.internal.builtin_plugins.r file_based_step.py:119 run_python_step [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Entering 30_rest_ingest.py#run(...) ...\n",
244+
"2026-01-10 01:50:42,377 [VDK] content [INFO ] vdk.internal.builtin_plugins.i ingester_router.py:64 send_object_for_[id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Sending object for ingestion with method: sqlite and target: None\n",
245+
"2026-01-10 01:50:42,867 [VDK] content [INFO ] numexpr.utils utils.py:164 _init_num_thread[id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- NumExpr defaulting to 2 threads.\n",
246+
"2026-01-10 01:50:43,154 [VDK] content [INFO ] vdk.internal.builtin_plugins.r file_based_step.py:125 run_python_step [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Exiting 30_rest_ingest.py#run(...) SUCCESS\n",
247+
"2026-01-10 01:50:45,155 [VDK] content [INFO ] vdk.plugin.sqlite.ingest_to_sq ingest_to_sqlite.py:76 ingest_payload [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Ingesting payloads for target: /tmp/vdk-sqlite.db; collection_id: content|317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841\n",
248+
"2026-01-10 01:50:45,156 [VDK] content [INFO ] vdk.plugin.sqlite.sqlite_conne sqlite_connection.py:29 new_connection [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Creating new connection against local file database located at: /tmp/vdk-sqlite.db\n",
249+
"2026-01-10 01:50:45,165 [VDK] content [INFO ] vdk.internal.builtin_plugins.i ingester_base.py:584 close_now [id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Ingester statistics: \n",
250+
"\t\tSuccessful uploads: 1\n",
251+
"\t\tFailed uploads: 0\n",
252+
"\t\tIngesting plugin errors: None\n",
253+
"\t\t\n",
254+
"2026-01-10 01:50:45,166 [VDK] content [INFO ] vdk.internal.builtin_plugins.r cli_run.py:170 __log_short_exec[id:317624ed-7ba7-4632-af5c-9d04c87281d1-1768009841-51ceb]- Job execution result: SUCCESS\n",
255+
"Step results:\n",
256+
"10_delete_table.sql - SUCCESS\n",
257+
"20_create_table.sql - SUCCESS\n",
258+
"30_rest_ingest.py - SUCCESS\n",
259+
"\n"
260+
]
261+
}
262+
]
263+
},
264+
{
265+
"cell_type": "code",
266+
"source": [
267+
"!vdk sqlite-query -q 'SELECT * FROM rest_target_table'"
268+
],
269+
"metadata": {
270+
"colab": {
271+
"base_uri": "https://localhost:8080/"
272+
},
273+
"id": "OdonSoTtmTco",
274+
"outputId": "978f69eb-c39e-4a4c-da8d-c22543c6f5ad"
275+
},
276+
"execution_count": 7,
277+
"outputs": [
278+
{
279+
"output_type": "stream",
280+
"name": "stdout",
281+
"text": [
282+
"\n",
283+
"Creating new connection against local file database located at: /tmp/vdk-sqlite.db\n",
284+
" userId id title completed\n",
285+
"-------- ---- ------------------ -----------\n",
286+
" 1 1 delectus aut autem 0\n"
287+
]
288+
}
289+
]
290+
},
291+
{
292+
"cell_type": "code",
293+
"source": [],
294+
"metadata": {
295+
"id": "9NUHJHASmZKl"
296+
},
297+
"execution_count": null,
298+
"outputs": []
299+
}
300+
]
301+
}

0 commit comments

Comments
 (0)