Skip to content

Commit e9b7f47

Browse files
alokemajumderalokemajumder
authored
Merge pull request #17 from rendiffdev/update1507
updatev2
2 parents 08a0418 + b402b8d commit e9b7f47

75 files changed

Lines changed: 4513 additions & 16434 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci-cd.yml‎

Lines changed: 394 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,394 @@
1+
name: CI/CD Pipeline
2+
3+
on:
4+
push:
5+
branches: [ main, develop ]
6+
pull_request:
7+
branches: [ main ]
8+
workflow_dispatch:
9+
10+
env:
11+
DOCKER_BUILDKIT: 1
12+
COMPOSE_DOCKER_CLI_BUILD: 1
13+
PYTHON_VERSION: 3.12.7
14+
15+
jobs:
16+
test:
17+
name: Test Suite
18+
runs-on: ubuntu-latest
19+
20+
services:
21+
postgres:
22+
image: postgres:15-alpine
23+
env:
24+
POSTGRES_PASSWORD: test_password
25+
POSTGRES_DB: test_db
26+
options: >-
27+
--health-cmd pg_isready
28+
--health-interval 10s
29+
--health-timeout 5s
30+
--health-retries 5
31+
ports:
32+
- 5432:5432
33+
34+
redis:
35+
image: redis:7-alpine
36+
options: >-
37+
--health-cmd "redis-cli ping"
38+
--health-interval 10s
39+
--health-timeout 5s
40+
--health-retries 5
41+
ports:
42+
- 6379:6379
43+
44+
steps:
45+
- name: Checkout code
46+
uses: actions/checkout@v4
47+
48+
- name: Set up Python
49+
uses: actions/setup-python@v4
50+
with:
51+
python-version: ${{ env.PYTHON_VERSION }}
52+
53+
- name: Cache pip dependencies
54+
uses: actions/cache@v3
55+
with:
56+
path: ~/.cache/pip
57+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
58+
restore-keys: |
59+
${{ runner.os }}-pip-
60+
61+
- name: Install dependencies
62+
run: |
63+
python -m pip install --upgrade pip
64+
pip install -r requirements.txt
65+
pip install -r requirements-dev.txt
66+
pip install pytest pytest-asyncio pytest-cov
67+
68+
- name: Create test environment file
69+
run: |
70+
cat > .env.test << EOF
71+
DATABASE_URL=postgresql://postgres:test_password@localhost:5432/test_db
72+
REDIS_URL=redis://localhost:6379
73+
SECRET_KEY=test_secret_key_for_testing_only
74+
ENABLE_API_KEYS=true
75+
LOG_LEVEL=INFO
76+
TESTING=true
77+
EOF
78+
79+
- name: Run database migrations
80+
run: |
81+
export $(cat .env.test | xargs)
82+
alembic upgrade head
83+
84+
- name: Run tests with coverage
85+
run: |
86+
export $(cat .env.test | xargs)
87+
pytest --cov=api --cov-report=xml --cov-report=html --cov-report=term-missing -v
88+
89+
- name: Upload coverage to Codecov
90+
uses: codecov/codecov-action@v3
91+
with:
92+
file: ./coverage.xml
93+
flags: unittests
94+
name: codecov-umbrella
95+
fail_ci_if_error: false
96+
97+
- name: Generate coverage report
98+
run: |
99+
echo "## Test Coverage Report" >> $GITHUB_STEP_SUMMARY
100+
echo "$(coverage report)" >> $GITHUB_STEP_SUMMARY
101+
102+
- name: Archive test results
103+
uses: actions/upload-artifact@v3
104+
if: always()
105+
with:
106+
name: test-results
107+
path: |
108+
htmlcov/
109+
coverage.xml
110+
pytest-report.xml
111+
112+
lint:
113+
name: Code Quality
114+
runs-on: ubuntu-latest
115+
116+
steps:
117+
- name: Checkout code
118+
uses: actions/checkout@v4
119+
120+
- name: Set up Python
121+
uses: actions/setup-python@v4
122+
with:
123+
python-version: ${{ env.PYTHON_VERSION }}
124+
125+
- name: Install linting tools
126+
run: |
127+
python -m pip install --upgrade pip
128+
pip install black flake8 mypy isort bandit safety
129+
130+
- name: Run Black (code formatting)
131+
run: black --check --diff api/ tests/
132+
133+
- name: Run isort (import sorting)
134+
run: isort --check-only --diff api/ tests/
135+
136+
- name: Run flake8 (linting)
137+
run: flake8 api/ tests/
138+
139+
- name: Run mypy (type checking)
140+
run: mypy api/
141+
142+
- name: Run bandit (security)
143+
run: bandit -r api/
144+
145+
- name: Run safety (dependency security)
146+
run: safety check
147+
148+
build:
149+
name: Build Docker Images
150+
runs-on: ubuntu-latest
151+
needs: [test, lint]
152+
153+
strategy:
154+
matrix:
155+
component: [api, worker-cpu, worker-gpu]
156+
include:
157+
- component: api
158+
dockerfile: docker/api/Dockerfile
159+
context: .
160+
build_args: |
161+
PYTHON_VERSION=${{ env.PYTHON_VERSION }}
162+
- component: worker-cpu
163+
dockerfile: docker/worker/Dockerfile
164+
context: .
165+
build_args: |
166+
PYTHON_VERSION=${{ env.PYTHON_VERSION }}
167+
WORKER_TYPE=cpu
168+
- component: worker-gpu
169+
dockerfile: docker/worker/Dockerfile
170+
context: .
171+
build_args: |
172+
PYTHON_VERSION=${{ env.PYTHON_VERSION }}
173+
WORKER_TYPE=gpu
174+
175+
steps:
176+
- name: Checkout code
177+
uses: actions/checkout@v4
178+
179+
- name: Set up Docker Buildx
180+
uses: docker/setup-buildx-action@v3
181+
182+
- name: Login to Container Registry
183+
if: github.ref == 'refs/heads/main'
184+
uses: docker/login-action@v3
185+
with:
186+
registry: ghcr.io
187+
username: ${{ github.actor }}
188+
password: ${{ secrets.GITHUB_TOKEN }}
189+
190+
- name: Extract metadata
191+
id: meta
192+
uses: docker/metadata-action@v5
193+
with:
194+
images: ghcr.io/${{ github.repository }}/${{ matrix.component }}
195+
tags: |
196+
type=ref,event=branch
197+
type=ref,event=pr
198+
type=sha,prefix={{branch}}-
199+
type=raw,value=latest,enable={{is_default_branch}}
200+
201+
- name: Build and push
202+
uses: docker/build-push-action@v5
203+
with:
204+
context: ${{ matrix.context }}
205+
file: ${{ matrix.dockerfile }}
206+
build-args: ${{ matrix.build_args }}
207+
push: ${{ github.ref == 'refs/heads/main' }}
208+
tags: ${{ steps.meta.outputs.tags }}
209+
labels: ${{ steps.meta.outputs.labels }}
210+
cache-from: type=gha
211+
cache-to: type=gha,mode=max
212+
platforms: linux/amd64,linux/arm64
213+
214+
security-scan:
215+
name: Security Scan
216+
runs-on: ubuntu-latest
217+
needs: build
218+
if: github.ref == 'refs/heads/main'
219+
220+
steps:
221+
- name: Checkout code
222+
uses: actions/checkout@v4
223+
224+
- name: Run Trivy vulnerability scanner
225+
uses: aquasecurity/trivy-action@master
226+
with:
227+
image-ref: ghcr.io/${{ github.repository }}/api:latest
228+
format: 'sarif'
229+
output: 'trivy-results.sarif'
230+
231+
- name: Upload Trivy scan results
232+
uses: github/codeql-action/upload-sarif@v2
233+
with:
234+
sarif_file: 'trivy-results.sarif'
235+
236+
integration-test:
237+
name: Integration Tests
238+
runs-on: ubuntu-latest
239+
needs: build
240+
241+
steps:
242+
- name: Checkout code
243+
uses: actions/checkout@v4
244+
245+
- name: Set up Docker Buildx
246+
uses: docker/setup-buildx-action@v3
247+
248+
- name: Build test environment
249+
run: |
250+
docker-compose -f docker-compose.yml -f docker-compose.test.yml build
251+
252+
- name: Run integration tests
253+
run: |
254+
docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d
255+
sleep 30
256+
257+
# Run API health check
258+
curl -f http://localhost:8000/api/v1/health || exit 1
259+
260+
# Run basic API tests
261+
python -m pytest tests/integration/ -v
262+
263+
- name: Cleanup
264+
if: always()
265+
run: |
266+
docker-compose -f docker-compose.yml -f docker-compose.test.yml down -v
267+
268+
deploy-staging:
269+
name: Deploy to Staging
270+
runs-on: ubuntu-latest
271+
needs: [test, lint, build, integration-test]
272+
if: github.ref == 'refs/heads/develop'
273+
environment: staging
274+
275+
steps:
276+
- name: Checkout code
277+
uses: actions/checkout@v4
278+
279+
- name: Deploy to staging
280+
run: |
281+
echo "Deploying to staging environment..."
282+
# Add deployment commands here
283+
# Example: kubectl apply -f k8s/staging/
284+
285+
- name: Run staging tests
286+
run: |
287+
echo "Running staging tests..."
288+
# Add staging test commands here
289+
290+
- name: Notify deployment
291+
if: always()
292+
run: |
293+
echo "Staging deployment completed"
294+
295+
deploy-production:
296+
name: Deploy to Production
297+
runs-on: ubuntu-latest
298+
needs: [test, lint, build, integration-test, security-scan]
299+
if: github.ref == 'refs/heads/main'
300+
environment: production
301+
302+
steps:
303+
- name: Checkout code
304+
uses: actions/checkout@v4
305+
306+
- name: Deploy to production
307+
run: |
308+
echo "Deploying to production environment..."
309+
# Add production deployment commands here
310+
# Example: kubectl apply -f k8s/production/
311+
312+
- name: Run production smoke tests
313+
run: |
314+
echo "Running production smoke tests..."
315+
# Add production smoke test commands here
316+
317+
- name: Create deployment issue
318+
if: failure()
319+
uses: actions/github-script@v6
320+
with:
321+
script: |
322+
github.rest.issues.create({
323+
owner: context.repo.owner,
324+
repo: context.repo.repo,
325+
title: 'Production deployment failed',
326+
body: 'Production deployment failed. Please check the logs and take necessary action.',
327+
labels: ['bug', 'production', 'deployment']
328+
})
329+
330+
- name: Notify deployment
331+
if: always()
332+
run: |
333+
echo "Production deployment completed"
334+
335+
backup-database:
336+
name: Database Backup
337+
runs-on: ubuntu-latest
338+
if: github.ref == 'refs/heads/main'
339+
340+
steps:
341+
- name: Checkout code
342+
uses: actions/checkout@v4
343+
344+
- name: Run database backup
345+
run: |
346+
echo "Running database backup..."
347+
# Add database backup commands here
348+
# Example: ./scripts/backup-database.sh
349+
350+
- name: Upload backup artifacts
351+
uses: actions/upload-artifact@v3
352+
with:
353+
name: database-backup
354+
path: backups/
355+
retention-days: 7
356+
357+
performance-test:
358+
name: Performance Tests
359+
runs-on: ubuntu-latest
360+
needs: build
361+
if: github.ref == 'refs/heads/main'
362+
363+
steps:
364+
- name: Checkout code
365+
uses: actions/checkout@v4
366+
367+
- name: Run performance tests
368+
run: |
369+
echo "Running performance tests..."
370+
# Add performance test commands here
371+
# Example: locust -f tests/performance/locustfile.py
372+
373+
- name: Generate performance report
374+
run: |
375+
echo "Generating performance report..."
376+
# Add performance report generation here
377+
378+
notify:
379+
name: Notify Results
380+
runs-on: ubuntu-latest
381+
needs: [test, lint, build, integration-test]
382+
if: always()
383+
384+
steps:
385+
- name: Notify success
386+
if: needs.test.result == 'success' && needs.lint.result == 'success' && needs.build.result == 'success'
387+
run: |
388+
echo "All CI/CD jobs completed successfully!"
389+
390+
- name: Notify failure
391+
if: needs.test.result == 'failure' || needs.lint.result == 'failure' || needs.build.result == 'failure'
392+
run: |
393+
echo "Some CI/CD jobs failed. Please check the logs."
394+
exit 1

0 commit comments

Comments
 (0)