updates the data preparation data

Author: WuorBhang

.gitignore

@@ -17,3 +17,7 @@ venv/
 Thumbs.db
 __pycache__/
 *.pyc
+
+# Except these files
+. 1_datasets\processed\tedsd_puf_2023_cleaned.csv
+. 1_datasets\raw\tedsd_puf_2023.csv

2_data_preparation/cleaning_teds_d.ipynb

@@ -20,7 +20,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 31,
    "id": "596bf3e0",
    "metadata": {},
    "outputs": [],
@@ -29,6 +29,8 @@
     "import numpy as np\n",
     "from tqdm import tqdm\n",
     "import warnings\n",
+    "import os\n",
+    "os.getcwd()\n",
     "\n",
     "warnings.filterwarnings(\"ignore\")"
    ]
@@ -876,15 +878,18 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9ba487dc",
+   "id": "dfbecb3a",
    "metadata": {},
    "outputs": [],
    "source": [
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
     "if __name__ == \"__main__\":\n",
     "    INPUT_FILE = \"1_datasets/raw/tedsd_puf_2023.csv\"\n",
     "    OUTPUT_FILE = \"1_datasets/processed/teds_d_2023_cleaned.csv\"\n",
     "\n",
-    "    clean_teds_d_data(INPUT_FILE, OUTPUT_FILE)"
+    "    clean_teds_d_data(INPUT_FILE, OUTPUT_FILE)\n"
    ]
   },
   {
@@ -899,21 +904,21 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "b16a295c",
+   "id": "76f5cdd5",
    "metadata": {},
    "outputs": [],
    "source": [
     "df = pd.read_csv(\"1_datasets/processed/teds_d_2023_cleaned.csv\")\n",
     "\n",
     "df_sample = df.sample(1000, random_state=42)\n",
     "\n",
-    "df_sample.to_csv(\"1_datasets/sample/tedsd_sample.csv\", index=False)"
+    "df_sample.to_csv(\"1_datasets/sample/tedsd_sample.csv\", index=False)\n"
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },
@@ -927,7 +932,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.10"
+   "version": "3.14.0"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -0,0 +1,117 @@
+anyio==4.11.0
+argon2-cffi==25.1.0
+argon2-cffi-bindings==25.1.0
+arrow==1.4.0
+astroid==4.0.1
+asttokens==3.0.0
+async-lru==2.0.5
+attrs==25.4.0
+babel==2.17.0
+beautifulsoup4==4.14.2
+bleach==6.3.0
+certifi==2025.10.5
+cffi==2.0.0
+charset-normalizer==3.4.4
+colorama==0.4.6
+comm==0.2.3
+contourpy==1.3.3
+cycler==0.12.1
+debugpy==1.8.17
+decorator==5.2.1
+defusedxml==0.7.1
+dill==0.4.0
+executing==2.2.1
+fastjsonschema==2.21.2
+fonttools==4.60.1
+fqdn==1.5.1
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+idna==3.11
+ipykernel==7.1.0
+ipython==9.7.0
+ipython_pygments_lexers==1.1.1
+ipywidgets==8.1.8
+isoduration==20.11.0
+isort==7.0.0
+jedi==0.19.2
+Jinja2==3.1.6
+json5==0.12.1
+jsonpointer==3.0.0
+jsonschema==4.25.1
+jsonschema-specifications==2025.9.1
+jupyter==1.1.1
+jupyter-console==6.6.3
+jupyter-events==0.12.0
+jupyter-lsp==2.3.0
+jupyter_client==8.6.3
+jupyter_core==5.9.1
+jupyter_server==2.17.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.4.10
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.28.0
+jupyterlab_widgets==3.0.16
+kiwisolver==1.4.9
+lark==1.3.1
+MarkupSafe==3.0.3
+matplotlib==3.10.7
+matplotlib-inline==0.2.1
+mccabe==0.7.0
+mistune==3.1.4
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+nodeenv==1.9.1
+notebook==7.4.7
+notebook_shim==0.2.4
+numpy==2.3.4
+packaging==25.0
+pandas==2.3.3
+pandocfilters==1.5.1
+parso==0.8.5
+pillow==12.0.0
+platformdirs==4.5.0
+prometheus_client==0.23.1
+prompt_toolkit==3.0.52
+psutil==7.1.3
+pure_eval==0.2.3
+pycparser==2.23
+Pygments==2.19.2
+pylint==4.0.2
+pyparsing==3.2.5
+pyright==1.1.407
+python-dateutil==2.9.0.post0
+python-json-logger==4.0.0
+pytz==2025.2
+pywinpty==3.0.2
+PyYAML==6.0.3
+pyzmq==27.1.0
+referencing==0.37.0
+requests==2.32.5
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rfc3987-syntax==1.1.0
+rpds-py==0.28.0
+seaborn==0.13.2
+Send2Trash==1.8.3
+setuptools==80.9.0
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.8
+stack-data==0.6.3
+terminado==0.18.1
+tinycss2==1.4.0
+tomlkit==0.13.3
+tornado==6.5.2
+traitlets==5.14.3
+typing_extensions==4.15.0
+tzdata==2025.2
+uri-template==1.3.0
+urllib3==2.5.0
+wcwidth==0.2.14
+webcolors==25.10.0
+webencodings==0.5.1
+websocket-client==1.9.0
+widgetsnbextension==4.0.15