Skip to content

Commit 841a7b3

Browse files
iosyediosyed
authored
Merge pull request #2276 from oracle-devrel/assaf-jobs-pipelines
initial commit for jobs and pipelines examples
2 parents c8ada6b + 431f3d4 commit 841a7b3

13 files changed

Lines changed: 2489 additions & 0 deletions

File tree

‎data-platform/data-science/oracle-data-science/jobs-pipelines/LICENSE‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
Copyright (c) 2025 Oracle and/or its affiliates.
2+
3+
The Universal Permissive License (UPL), Version 1.0
4+
5+
Subject to the condition set forth below, permission is hereby granted to any
6+
person obtaining a copy of this software, associated documentation and/or data
7+
(collectively the "Software"), free of charge and under any and all copyright
8+
rights in the Software, and any and all patent rights owned or freely
9+
licensable by each licensor hereunder covering either (i) the unmodified
10+
Software as contributed to or provided by such licensor, or (ii) the Larger
11+
Works (as defined below), to deal in both
12+
13+
(a) the Software, and
14+
(b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
15+
one is included with the Software (each a "Larger Work" to which the Software
16+
is contributed by such licensors),
17+
18+
without restriction, including without limitation the rights to copy, create
19+
derivative works of, display, perform, and distribute the Software and make,
20+
use, sell, offer for sale, import, export, have made, and have sold the
21+
Software and the Larger Work(s), and to sublicense the foregoing rights on
22+
either these or other terms.
23+
24+
This license is subject to the following condition:
25+
The above copyright notice and either this complete permission notice or at
26+
a minimum a reference to the UPL must be included in all copies or
27+
substantial portions of the Software.
28+
29+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
32+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
33+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
34+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35+
SOFTWARE.

‎data-platform/data-science/oracle-data-science/jobs-pipelines/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Overview
2+
3+
This package provides examples for creating and managing OCI Data Science Jobs and Pipelines using the ADS SDK. It includes three use cases:
4+
5+
1. Basic Job Creation – simple job using a single script
6+
2. Advanced Job – job using a source folder, custom variables, and flexible configuration
7+
3. Pipeline Creation – multi-step workflows using jobs as pipeline steps
8+
9+
# Prerequisites
10+
11+
To use this package, you need:
12+
13+
- Basic Python knowledge
14+
- Access to the OCI Data Science Platform with appropriate Resource Principal permissions and IAM policies
15+
- An OCI Object Storage bucket
16+
17+
# Environment
18+
19+
Run the examples in a Jupyter Notebook inside an OCI Data Science Notebook Session using: generaml_p311_cpu_x86_64_v1

‎data-platform/data-science/oracle-data-science/jobs-pipelines/files/1. basic job creation/create_job.ipynb‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "cafbd039-d653-4dfc-9cd3-aa389c58f92a",
6+
"metadata": {},
7+
"source": [
8+
"### OCI Data Science - Useful Tips\n",
9+
"<details>\n",
10+
"<summary><font size=\"2\">Check for Public Internet Access</font></summary>\n",
11+
"\n",
12+
"```python\n",
13+
"import requests\n",
14+
"response = requests.get(\"https://oracle.com\")\n",
15+
"assert response.status_code==200, \"Internet connection failed\"\n",
16+
"```\n",
17+
"</details>\n",
18+
"<details>\n",
19+
"<summary><font size=\"2\">Helpful Documentation </font></summary>\n",
20+
"<ul><li><a href=\"https://docs.cloud.oracle.com/en-us/iaas/data-science/using/data-science.htm\">Data Science Service Documentation</a></li>\n",
21+
"<li><a href=\"https://docs.cloud.oracle.com/iaas/tools/ads-sdk/latest/index.html\">ADS documentation</a></li>\n",
22+
"</ul>\n",
23+
"</details>\n",
24+
"<details>\n",
25+
"<summary><font size=\"2\">Typical Cell Imports and Settings for ADS</font></summary>\n",
26+
"\n",
27+
"```python\n",
28+
"%load_ext autoreload\n",
29+
"%autoreload 2\n",
30+
"%matplotlib inline\n",
31+
"\n",
32+
"import warnings\n",
33+
"warnings.filterwarnings('ignore')\n",
34+
"\n",
35+
"import logging\n",
36+
"logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.ERROR)\n",
37+
"\n",
38+
"import ads\n",
39+
"from ads.dataset.factory import DatasetFactory\n",
40+
"from ads.automl.provider import OracleAutoMLProvider\n",
41+
"from ads.automl.driver import AutoML\n",
42+
"from ads.evaluations.evaluator import ADSEvaluator\n",
43+
"from ads.common.data import ADSData\n",
44+
"from ads.explanations.explainer import ADSExplainer\n",
45+
"from ads.explanations.mlx_global_explainer import MLXGlobalExplainer\n",
46+
"from ads.explanations.mlx_local_explainer import MLXLocalExplainer\n",
47+
"from ads.catalog.model import ModelCatalog\n",
48+
"from ads.common.model_artifact import ModelArtifact\n",
49+
"```\n",
50+
"</details>\n",
51+
"<details>\n",
52+
"<summary><font size=\"2\">Useful Environment Variables</font></summary>\n",
53+
"\n",
54+
"```python\n",
55+
"import os\n",
56+
"print(os.environ[\"NB_SESSION_COMPARTMENT_OCID\"])\n",
57+
"print(os.environ[\"PROJECT_OCID\"])\n",
58+
"print(os.environ[\"USER_OCID\"])\n",
59+
"print(os.environ[\"TENANCY_OCID\"])\n",
60+
"print(os.environ[\"NB_REGION\"])\n",
61+
"```\n",
62+
"</details>"
63+
]
64+
},
65+
{
66+
"cell_type": "markdown",
67+
"id": "30e3ce44-ed71-4c23-88ad-a81ca9b94bb9",
68+
"metadata": {},
69+
"source": [
70+
"# Import Packages"
71+
]
72+
},
73+
{
74+
"cell_type": "code",
75+
"execution_count": null,
76+
"id": "3a951454-9a00-4d9b-99dd-77f8cac53f1c",
77+
"metadata": {},
78+
"outputs": [],
79+
"source": [
80+
"from ads.jobs import Job, DataScienceJob, PythonRuntime\n",
81+
"from ads import set_auth\n",
82+
"\n",
83+
"set_auth(auth='resource_principal') # your dynamic group should have matching rules for: data science, job, and schudeler."
84+
]
85+
},
86+
{
87+
"cell_type": "markdown",
88+
"id": "15b25cd9-9d6c-48ae-95d9-20ba7a9b0811",
89+
"metadata": {},
90+
"source": [
91+
"# Creating the Job"
92+
]
93+
},
94+
{
95+
"cell_type": "code",
96+
"execution_count": 5,
97+
"id": "8da9ac43-c07e-46f9-936d-e3a4a59e3ab3",
98+
"metadata": {
99+
"tags": []
100+
},
101+
"outputs": [],
102+
"source": [
103+
"job = (\n",
104+
" Job(name=\"adult_income_model_training_basic\")\n",
105+
" .with_infrastructure(\n",
106+
" DataScienceJob()\n",
107+
" .with_log_group_id('<your_log_group_ocid>')\n",
108+
" .with_shape_name(\"VM.Standard.E4.Flex\")\n",
109+
" .with_shape_config_details(memory_in_gbs=4, ocpus=1)\n",
110+
" .with_block_storage_size(50) # minimus is 50\n",
111+
" )\n",
112+
" .with_runtime(\n",
113+
" PythonRuntime()\n",
114+
" .with_service_conda(\"generalml_p38_cpu_v1\")\n",
115+
" .with_source(\"<yout script>\") # can be a .ipynb, as well as py.\n",
116+
" )\n",
117+
")"
118+
]
119+
},
120+
{
121+
"cell_type": "code",
122+
"execution_count": null,
123+
"id": "dbf5548e-a679-4087-9c80-40a4b2a1a982",
124+
"metadata": {
125+
"tags": []
126+
},
127+
"outputs": [],
128+
"source": [
129+
"job.create()"
130+
]
131+
},
132+
{
133+
"cell_type": "markdown",
134+
"id": "6d0c4f22-ddae-42da-9bdd-43b8ac11dca0",
135+
"metadata": {},
136+
"source": [
137+
"# Running a Job Run"
138+
]
139+
},
140+
{
141+
"cell_type": "code",
142+
"execution_count": null,
143+
"id": "829803aa-e686-4015-b834-085fba8212b8",
144+
"metadata": {
145+
"tags": []
146+
},
147+
"outputs": [],
148+
"source": [
149+
"run = job.run() # run the job\n",
150+
"run.watch() # print the log below"
151+
]
152+
}
153+
],
154+
"metadata": {
155+
"kernelspec": {
156+
"display_name": "Python [conda env:generalml_p311_cpu_x86_64_v1]",
157+
"language": "python",
158+
"name": "conda-env-generalml_p311_cpu_x86_64_v1-py"
159+
},
160+
"language_info": {
161+
"codemirror_mode": {
162+
"name": "ipython",
163+
"version": 3
164+
},
165+
"file_extension": ".py",
166+
"mimetype": "text/x-python",
167+
"name": "python",
168+
"nbconvert_exporter": "python",
169+
"pygments_lexer": "ipython3",
170+
"version": "3.11.9"
171+
}
172+
},
173+
"nbformat": 4,
174+
"nbformat_minor": 5
175+
}

0 commit comments

Comments
 (0)