Skip to content

Commit c5bce85

Browse files
committed
TutorTask696: Add input handling, date time formatting and integrity checking of dataset at ingestion
Pre-commit checks: All checks passed ✅
1 parent b93e93b commit c5bce85

7 files changed

Lines changed: 917 additions & 0 deletions

File tree

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# OS files
2+
.DS_Store
3+
4+
# Python cache/build artifacts
5+
__pycache__/
6+
*.py[cod]
7+
*.pyo
8+
*.pyd
9+
10+
# Secrets and local environment files
11+
.env
12+
*.env
13+
config/.env
14+
*.secret
15+
*secret*
16+
*.key
17+
*.pem
Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import os
2+
import dataclasses
3+
import functools
4+
import pydantic
5+
6+
import dotenv
7+
import langchain_openai
8+
import langchain_anthropic # ChatAnthropic
9+
import langchain_google_genai # ChatGoogleGenerativeAI
10+
# import langchain_groq # ChatGroq
11+
# import langchain_mistralai # ChatMistralAI
12+
# import langchain_ollama # ChatOllama
13+
14+
15+
dataclass = dataclasses.dataclass
16+
lru_cache = functools.lru_cache
17+
ChatOpenAI = langchain_openai.ChatOpenAI
18+
ChatAnthropic = langchain_anthropic.ChatAnthropic
19+
ChatGoogleGenerativeAI = langchain_google_genai.ChatGoogleGenerativeAI
20+
# ChatGroq = langchain_groq.ChatGroq
21+
# ChatMistralAI = langchain_mistralai.ChatMistralAI
22+
# ChatOllama = langchain_ollama.ChatOllama
23+
SecretStr = pydantic.SecretStr
24+
25+
# Load Variables
26+
dotenv.load_dotenv()
27+
28+
29+
# Immutable data class
30+
@dataclass(frozen=True)
31+
class Settings:
32+
provider: str
33+
model: str
34+
temperature: float
35+
timeout: float
36+
max_retries: int
37+
38+
def _need(name:str) -> str:
39+
v = os.getenv(name)
40+
if v is None or v == "":
41+
raise RuntimeError(f"Missing required environment variable: {name}")
42+
return v
43+
44+
@lru_cache(maxsize=1)
45+
def get_settings() -> Settings:
46+
return Settings(
47+
provider=os.getenv("LLM_PROVIDER", "openai"),
48+
model=os.getenv("LLM_MODEL", "gpt-5-nano"),
49+
temperature=float(os.getenv("LLM_TEMP", 0.2)),
50+
timeout=float(os.getenv("LLM_TIMEOUT", 60)),
51+
max_retries=int(os.getenv("LLM_MAX_RETRIES", 2)),
52+
53+
)
54+
55+
@lru_cache(maxsize=1)
56+
def get_chat_model(model=get_settings().model):
57+
s = get_settings()
58+
59+
# OpenAI-adjacent
60+
61+
if s.provider == "openai":
62+
63+
# READ API KEY.
64+
_need("OPENAI_API_KEY")
65+
66+
# Return the chatmodel
67+
68+
return ChatOpenAI(
69+
model=s.model,
70+
temperature=s.temperature,
71+
timeout=s.timeout,
72+
max_retries=s.max_retries,
73+
)
74+
75+
if s.provider == "openai_compatible":
76+
77+
# Secrets.
78+
base_url = _need("OPENAI_COMPAT_BASE_URL")
79+
api_key = _need("OPENAI_COMPAT_API_KEY")
80+
return ChatOpenAI(
81+
model=model,
82+
base_url=base_url,
83+
api_key=SecretStr(api_key),
84+
temperature=s.temperature,
85+
timeout=s.timeout,
86+
max_retries=s.max_retries,
87+
88+
)
89+
90+
if s.provider == "azure_openai_v1":
91+
92+
# Secrets.
93+
azure_base = _need("AZURE_OPENAI_BASE_URL")
94+
azure_key = SecretStr(_need("AZURE_OPENAI_API_KEY"))
95+
96+
return ChatOpenAI(
97+
model=s.model,
98+
base_url=azure_base,
99+
api_key=azure_key,
100+
temperature=s.temperature,
101+
timeout=s.timeout,
102+
max_retries=s.max_retries,
103+
104+
)
105+
106+
# Anthropic
107+
108+
if s.provider == "anthropic":
109+
110+
# Secrets.
111+
_need("ANTHROPIC_API_KEY")
112+
return ChatAnthropic(
113+
model_name=s.model,
114+
temperature=s.temperature,
115+
timeout=s.timeout,
116+
max_retries=s.max_retries,
117+
stop=None
118+
)
119+
120+
# Google
121+
if s.provider in ("google", "gemini", "google_genai"):
122+
# Secrets.
123+
_need("GOOGLE_API_KEY")
124+
return ChatGoogleGenerativeAI(
125+
model=s.model,
126+
temperature=s.temperature,
127+
)
128+
129+
130+
131+
132+
133+
raise ValueError("TODO(*): expand support!")
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
Date/Time,LV ActivePower (kW),Wind Speed (m/s),Theoretical_Power_Curve (KWh),Wind Direction (°)
2+
01 01 2018 00:00,380.047790527343,5.31133604049682,416.328907824861,259.994903564453
3+
01 01 2018 00:10,453.76919555664,5.67216682434082,519.917511061494,268.64111328125
4+
01 01 2018 00:20,306.376586914062,5.21603679656982,390.900015810951,272.564788818359
5+
01 01 2018 00:30,419.645904541015,5.65967416763305,516.127568975674,271.258087158203
6+
01 01 2018 00:40,380.650695800781,5.57794094085693,491.702971953588,265.674285888671
7+
01 01 2018 00:50,402.391998291015,5.60405206680297,499.436385024805,264.57861328125
8+
01 01 2018 01:00,447.605712890625,5.79300785064697,557.372363290225,266.163604736328
9+
01 01 2018 01:10,387.2421875,5.30604982376098,414.898178826186,257.949493408203
10+
01 01 2018 01:20,463.651214599609,5.58462905883789,493.677652137077,253.480697631835
11+
01 01 2018 01:30,439.725708007812,5.52322816848754,475.706782818068,258.72378540039
12+
01 01 2018 01:40,498.181701660156,5.72411584854125,535.841397042263,251.850997924804
13+
01 01 2018 01:50,526.816223144531,5.93419885635375,603.014076510633,265.504699707031
14+
01 01 2018 02:00,710.587280273437,6.54741382598876,824.662513585882,274.23291015625
15+
01 01 2018 02:10,655.194274902343,6.19974613189697,693.472641075637,266.733184814453
16+
01 01 2018 02:20,754.762512207031,6.50538301467895,808.098138482693,266.76040649414
17+
01 01 2018 02:30,790.173278808593,6.63411617279052,859.459020788565,270.493194580078
18+
01 01 2018 02:40,742.985290527343,6.37891292572021,759.434536596592,266.593292236328
19+
01 01 2018 02:50,748.229614257812,6.4466528892517,785.28100987646,265.571807861328
20+
01 01 2018 03:00,736.647827148437,6.41508293151855,773.172863451736,261.15869140625
21+
01 01 2018 03:10,787.246215820312,6.43753099441528,781.7712157188,257.56021118164
22+
01 01 2018 03:20,722.864074707031,6.22002410888671,700.764699868076,255.926498413085
23+
01 01 2018 03:30,935.033386230468,6.89802598953247,970.736626881787,250.012893676757
24+
01 01 2018 03:40,1220.60900878906,7.60971117019653,1315.04892785216,255.985702514648
25+
01 01 2018 03:50,1053.77197265625,7.28835582733154,1151.26574355584,255.444595336914
26+
01 01 2018 04:00,1493.80798339843,7.94310188293457,1497.58372354361,256.407409667968
27+
01 01 2018 04:10,1724.48803710937,8.37616157531738,1752.19966204818,252.41259765625
28+
01 01 2018 04:20,1636.93505859375,8.23695755004882,1668.47070685152,247.979400634765
29+
01 01 2018 04:30,1385.48803710937,7.87959098815917,1461.81579081391,238.609603881835
30+
01 01 2018 04:40,1098.93200683593,7.10137605667114,1062.28503444311,245.095596313476
31+
01 01 2018 04:50,1021.4580078125,6.95530700683593,995.995854606612,245.410202026367
32+
01 01 2018 05:00,1164.89294433593,7.09829807281494,1060.85971215544,235.227905273437
33+
01 01 2018 05:10,1073.33203125,6.95363092422485,995.250960801046,242.872695922851
34+
01 01 2018 05:20,1165.30798339843,7.24957799911499,1132.4168612641,244.835693359375
35+
01 01 2018 05:30,1177.98999023437,7.29469108581542,1154.36530469206,242.48159790039
36+
01 01 2018 05:40,1170.53601074218,7.37636995315551,1194.8430985043,247.97720336914
37+
01 01 2018 05:50,1145.53601074218,7.44855403900146,1231.43070603717,249.682998657226
38+
01 01 2018 06:00,1114.02697753906,7.2392520904541,1127.43320551345,248.401000976562
39+
01 01 2018 06:10,1153.18505859375,7.32921123504638,1171.35504358957,244.621704101562
40+
01 01 2018 06:20,1125.3310546875,7.13970518112182,1080.13908466205,244.631805419921
41+
01 01 2018 06:30,1228.73205566406,7.47422885894775,1244.63353439737,245.785995483398
42+
01 01 2018 06:40,1021.79302978515,7.03317403793334,1030.99268581181,248.652206420898
43+
01 01 2018 06:50,957.378173828125,6.88645505905151,965.683334443832,244.611694335937
44+
01 01 2018 07:00,909.887817382812,6.88782119750976,966.279104864065,235.84829711914
45+
01 01 2018 07:10,1000.95397949218,7.21643209457397,1116.4718990154,232.842697143554
46+
01 01 2018 07:20,1024.47802734375,7.0685977935791,1047.17023059277,229.933197021484
47+
01 01 2018 07:30,1009.53399658203,6.93829584121704,988.451940715539,230.13670349121
48+
01 01 2018 07:40,899.492980957031,6.53668785095214,820.416658585943,234.933807373046
49+
01 01 2018 07:50,725.110107421875,6.18062496185302,686.636942163399,232.837905883789
50+
01 01 2018 08:00,585.259399414062,5.81682586669921,564.927659543473,240.328796386718
51+
01 01 2018 08:10,443.913909912109,5.45015096664428,454.773587146918,238.12629699707
52+
01 01 2018 08:20,565.253784179687,5.81814908981323,565.349093224668,235.80029296875
53+
01 01 2018 08:30,644.037780761718,6.13027286529541,668.823569309414,224.958694458007
54+
01 01 2018 08:40,712.058898925781,6.34707784652709,747.460673422601,216.803894042968
55+
01 01 2018 08:50,737.394775390625,6.34743690490722,747.595109122642,205.785293579101
56+
01 01 2018 09:00,725.868103027343,6.19436883926391,691.546334303948,199.848495483398
57+
01 01 2018 09:10,408.997406005859,4.97719812393188,330.417630427964,207.997802734375
58+
01 01 2018 09:20,628.436828613281,5.95911121368408,611.283836510667,210.954895019531
59+
01 01 2018 09:30,716.1005859375,6.21137619018554,697.649474372052,215.69400024414
60+
01 01 2018 09:40,711.49560546875,6.11145305633544,662.235163012206,220.84260559082
61+
01 01 2018 09:50,838.151916503906,6.45632219314575,789.011422412419,237.065307617187
62+
01 01 2018 10:00,881.062072753906,6.66665792465209,872.739625855708,235.667495727539
63+
01 01 2018 10:10,663.703125,6.16287899017333,680.327891653483,229.329696655273
64+
01 01 2018 10:20,578.261596679687,6.01316785812377,628.442560754699,234.900604248046
65+
01 01 2018 10:30,465.620086669921,5.56120300292968,486.779567601972,230.422805786132
66+
01 01 2018 10:40,311.050903320312,4.96073198318481,326.411025380213,229.537506103515
67+
01 01 2018 10:50,230.05549621582,4.60387516021728,244.31624421611,231.79849243164
68+
01 01 2018 11:00,233.990600585937,4.55453395843505,233.632780531927,234.105606079101
69+
01 01 2018 11:10,175.592193603515,4.26362895965576,173.573663122312,228.776702880859
70+
01 01 2018 11:20,118.133102416992,3.89413905143737,108.571221110423,227.938995361328
71+
01 01 2018 11:30,142.202499389648,4.03876113891601,130.229989593698,224.46499633789
72+
01 01 2018 11:40,212.566192626953,4.50565099716186,223.196784083793,224.950500488281
73+
01 01 2018 11:50,222.610000610351,4.54339790344238,231.242507343633,229.12759399414
74+
01 01 2018 12:00,194.181198120117,4.32376098632812,185.598479588255,227.039993286132
75+
01 01 2018 12:10,82.6407470703125,3.63443708419799,68.5028197987886,230.31460571289
76+
01 01 2018 12:20,75.8952178955078,3.70551204681396,78.3961653540173,233.953292846679
77+
01 01 2018 12:30,41.9472389221191,3.25396800041198,29.2869556318446,233.06590270996
78+
01 01 2018 12:40,118.534599304199,3.77513694763183,88.8713653309387,227.753494262695
79+
01 01 2018 12:50,250.755905151367,4.69350099563598,264.119257409418,229.896606445312
80+
01 01 2018 13:00,346.86441040039,5.00293922424316,336.721998240131,235.279495239257
81+
01 01 2018 13:10,416.417907714843,5.36474990844726,430.92108895689,235.585296630859
82+
01 01 2018 13:20,331.941497802734,5.01618194580078,339.984940156412,229.942901611328
83+
01 01 2018 13:30,583.479919433593,5.97040796279907,615.05563084927,235.69529724121
84+
01 01 2018 13:40,776.552673339843,6.6555209159851,868.180844867276,241.457397460937
85+
01 01 2018 13:50,752.726379394531,6.60090398788452,846.029409522117,242.782104492187
86+
01 01 2018 14:00,589.073120117187,5.98137807846069,618.731442665699,234.984405517578
87+
01 01 2018 14:10,1109.12805175781,7.42459392547607,1219.19978672882,235.14729309082
88+
01 01 2018 14:20,1482.4599609375,8.18645191192626,1638.50890923271,238.479095458984
89+
01 01 2018 14:30,1523.43005371093,8.27493000030517,1691.1470390233,237.033203125
90+
01 01 2018 14:40,1572.17004394531,8.44920253753662,1796.76309010091,238.332397460937
91+
01 01 2018 14:50,1698.93994140625,8.5759744644165,1875.04719734159,235.641403198242
92+
01 01 2018 15:00,1616.84594726562,8.28225994110107,1695.53877696245,236.461395263671
93+
01 01 2018 15:10,1796.82397460937,8.73455238342285,1974.47580025242,234.354797363281
94+
01 01 2018 15:20,1885.86096191406,8.76410388946533,1993.17071186444,231.001602172851
95+
01 01 2018 15:30,2327.51196289062,9.66943168640136,2568.82712862015,227.60009765625
96+
01 01 2018 15:40,2499.162109375,10.1410903930664,2876.75361614448,227.73159790039
97+
01 01 2018 15:50,2820.51293945312,10.7724199295043,3186.02988321436,225.276397705078
98+
01 01 2018 16:00,2812.27905273437,10.6475200653076,3133.25922420184,224.680603027343
99+
01 01 2018 16:10,2530.44702148437,9.98266124725341,2781.27404078649,225.519500732421
100+
01 01 2018 16:20,2399.12109375,9.87438583374023,2711.49245838958,227.273803710937
101+
01 01 2018 16:30,2335.587890625,9.78547954559326,2651.34100928894,229.255493164062

0 commit comments

Comments
 (0)