Fix memory spilling and OOM by capping polars threads; add pipeline performance stress test #21

	name: CI - Code Quality & Tests

	on:
	pull_request:
	branches: [ main ]
	push:
	branches: [ main ]

	jobs:
	test:
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.11"

	- name: Install Dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r dev-requirements.txt

	- name: Set PYTHONPATH
	run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV

	- name: Check Code Formatting
	run: black --check .

	- name: Ruff Linting
	run: ruff check .

	- name: Run Tests with Coverage
	run: \|
	pytest \
	--cov=data_pipeline \
	--cov-report=term-missing \
	--cov-fail-under=85

Provide feedback