update 02 notebook

Author: yohan-markose

notebooks/02_raw_to_harmonized/02_raw_to_harmonized.ipynb

@@ -1,4 +1,20 @@
 {
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Streamlit Notebook",
+   "name": "streamlit"
+  },
+  "lastEditStatus": {
+   "notebookId": "6ppanuxguyb2g2embnsa",
+   "authorId": "6690310009356",
+   "authorName": "YOHANMARKOSE",
+   "authorEmail": "markose.y@northeastern.edu",
+   "sessionId": "96206498-35e6-4bcf-b1a7-a81dbdb91cc5",
+   "lastEditTime": 1740709418860
+  }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
  "cells": [
   {
    "cell_type": "code",
@@ -48,38 +64,7 @@
     "name": "cell3"
    },
    "outputs": [],
-   "source": [
-    "import snowflake.snowpark.functions as F\n",
-    "\n",
-    "env = schema_name[:3]\n",
-    "def create_fred_view(session):\n",
-    "    raw_fred_10Y = session.table(f\"{database_name}.{env}_RAW_FRED.FRED_DGS10Y\").select(\n",
-    "                                                                                        F.to_date(F.col('\"date\"')).alias(\"OBSERVATION_DATE\"),\n",
-    "                                                                                        F.col('\"value\"').cast(\"float\").alias(\"10Y_YIELD\")\n",
-    "                                                                                    )\n",
-    "    \n",
-    "    raw_fred_2Y = session.table(f\"{database_name}.{env}_RAW_FRED.FRED_DGS2Y\").select(\n",
-    "                                                                                        F.to_date(F.col('\"date\"')).alias(\"OBSERVATION_DATE\"),\n",
-    "                                                                                        F.col('\"value\"').cast(\"float\").alias(\"2Y_YIELD\")\n",
-    "                                                                                    )\n",
-    "\n",
-    "    harmonized_data = raw_fred_10Y.join(raw_fred_2Y, raw_fred_10Y[\"OBSERVATION_DATE\"] == raw_fred_2Y[\"OBSERVATION_DATE\"]).select(\n",
-    "                                                                                                                                    raw_fred_10Y[\"OBSERVATION_DATE\"].alias(\"OBSERVATION_DATE\"),\n",
-    "                                                                                                                                    F.col(\"10Y_YIELD\"),\n",
-    "                                                                                                                                    F.col(\"2Y_YIELD\")\n",
-    "                                                                                                                                )\n",
-    "\n",
-    "\n",
-    "    \n",
-    "    session.use_schema(f\"{database_name}.{schema_name}\")\n",
-    "    harmonized_data.write.mode(\"overwrite\").save_as_table('FRED_FLATTENED')\n",
-    "\n",
-    "def create_fred_view_stream(session):\n",
-    "    _ = session.sql('CREATE OR REPLACE STREAM FRED_STREAM \\\n",
-    "                        ON TABLE FRED_FLATTENED \\\n",
-    "                        SHOW_INITIAL_ROWS = TRUE').collect()\n",
-    "    \n"
-   ]
+   "source": "import snowflake.snowpark.functions as F\n\nenv = schema_name[:3]\ndef create_fred_table(session):\n    raw_fred_10Y = session.table(f\"{database_name}.{env}_RAW_FRED.FRED_DGS10Y\").select(\n                                                                                        F.to_date(F.col('\"date\"')).alias(\"OBSERVATION_DATE\"),\n                                                                                        F.col('\"value\"').cast(\"float\").alias(\"10Y_YIELD\")\n                                                                                    )\n    \n    raw_fred_2Y = session.table(f\"{database_name}.{env}_RAW_FRED.FRED_DGS2Y\").select(\n                                                                                        F.to_date(F.col('\"date\"')).alias(\"OBSERVATION_DATE\"),\n                                                                                        F.col('\"value\"').cast(\"float\").alias(\"2Y_YIELD\")\n                                                                                    )\n\n    harmonized_data = raw_fred_10Y.join(raw_fred_2Y, raw_fred_10Y[\"OBSERVATION_DATE\"] == raw_fred_2Y[\"OBSERVATION_DATE\"]).select(\n                                                                                                                                    raw_fred_10Y[\"OBSERVATION_DATE\"].alias(\"OBSERVATION_DATE\"),\n                                                                                                                                    F.col(\"10Y_YIELD\"),\n                                                                                                                                    F.col(\"2Y_YIELD\")\n                                                                                                                                )\n\n\n    \n    session.use_schema(f\"{database_name}.{schema_name}\")\n    harmonized_data.write.mode(\"overwrite\").save_as_table('FRED_FLATTENED')\n\ndef create_fred_table_stream(session):\n    _ = session.sql('CREATE OR REPLACE STREAM FRED_STREAM \\\n                        ON TABLE FRED_FLATTENED \\\n                        SHOW_INITIAL_ROWS = TRUE').collect()\n    \n"
   },
   {
    "cell_type": "code",
@@ -90,27 +75,7 @@
     "name": "cell4"
    },
    "outputs": [],
-   "source": [
-    "\n",
-    "create_fred_view(session)\n",
-    "create_fred_view_stream(session)"
-   ]
+   "source": "\ncreate_fred_table(session)\ncreate_fred_table_stream(session)"
   }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Streamlit Notebook",
-   "name": "streamlit"
-  },
-  "lastEditStatus": {
-   "authorEmail": "markose.y@northeastern.edu",
-   "authorId": "6690310009356",
-   "authorName": "YOHANMARKOSE",
-   "lastEditTime": 1740634222219,
-   "notebookId": "gjmtbtbofcart6nlju5h",
-   "sessionId": "d73d2d9f-2f54-4d4b-96d0-fe0c676a14f9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
+ ]
+}