multi-cloud-data-pipeline/requirements.txt at main · Helios-coder/multi-cloud-data-pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# Core dependencies
pyspark==3.5.0
py4j==0.10.9.7

# Azure SDK
azure-storage-blob==12.19.0
azure-identity==1.15.0
azure-mgmt-storage==21.1.0
azure-mgmt-datafactory==4.0.0
azure-eventhub==5.11.6

# GCP SDK
google-cloud-bigquery==3.14.1
google-cloud-storage==2.14.0
google-cloud-pubsub==2.19.0
google-cloud-dataflow==0.7.5

# Data Quality
great-expectations==0.18.8

# Orchestration
apache-airflow==2.8.1
prefect==2.14.16

# Data Processing
pandas==2.1.4
numpy==1.26.3
pyarrow==14.0.2

# Configuration
pyyaml==6.0.1
python-dotenv==1.0.0

# Logging and Monitoring
structlog==24.1.0

# Testing
pytest==7.4.3
pytest-cov==4.1.0
pytest-mock==3.12.0
pytest-asyncio==0.23.3

# Code Quality
black==23.12.1
flake8==7.0.0
pylint==3.0.3
isort==5.13.2
mypy==1.8.0

# Security
safety==3.0.1
bandit==1.7.6

# Documentation
sphinx==7.2.6
sphinx-rtd-theme==2.0.0
sphinxcontrib-napoleon==0.7