Skip to content

Commit 0691fd4

Browse files
YostraYostra
authored
Add hourly production e2e workflow with Sigma reconciliation (#311)
* e2e ci test * run in ci * add warning on non zero exit
1 parent 2d2bac6 commit 0691fd4

2 files changed

Lines changed: 810 additions & 0 deletions

File tree

‎.github/workflows/prod-e2e-test.yml‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
name: Stripe Database Monitor
2+
3+
on:
4+
schedule:
5+
- cron: '0 * * * *' # Every hour
6+
workflow_dispatch:
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
monitor:
13+
name: Create DB & monitor sync progress
14+
runs-on: ubuntu-24.04-arm
15+
timeout-minutes: 75
16+
17+
steps:
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
21+
- name: Setup Node.js
22+
uses: actions/setup-node@v4
23+
with:
24+
node-version: '24'
25+
26+
- name: Install Stripe CLI
27+
run: |
28+
curl -s https://packages.stripe.dev/api/security/keypair/stripe-cli-gpg/public | gpg --dearmor -o /usr/share/keyrings/stripe.gpg
29+
echo "deb [signed-by=/usr/share/keyrings/stripe.gpg] https://packages.stripe.dev/stripe-cli-debian-local stable main" \
30+
| sudo tee /etc/apt/sources.list.d/stripe.list
31+
sudo apt-get update -qq && sudo apt-get install -y -qq stripe
32+
33+
- name: Write Stripe CLI config
34+
env:
35+
STRIPE_CONFIG_TOML: ${{ secrets.STRIPE_CONFIG_TOML }}
36+
run: |
37+
mkdir -p ~/.config/stripe
38+
echo "$STRIPE_CONFIG_TOML" > ~/.config/stripe/config.toml
39+
chmod 600 ~/.config/stripe/config.toml
40+
echo "Stripe CLI config written to ~/.config/stripe/config.toml"
41+
42+
- name: Stripe CLI login
43+
env:
44+
STRIPE_API_KEY: ${{ secrets.GOLDI_LIVE_KEY }}
45+
run: echo "$STRIPE_API_KEY" | stripe login --interactive
46+
47+
- name: Create Stripe database
48+
id: create-db
49+
run: |
50+
echo "Creating Stripe database..."
51+
RESULT=$(stripe databases create)
52+
53+
DB_ID=$(echo "$RESULT" | grep -oE 'db_test_[A-Za-z0-9]+' | head -1)
54+
DB_STRING=$(echo "$RESULT" | grep -oE 'postgresql://[^[:space:]]+')
55+
56+
if [ -z "$DB_ID" ]; then
57+
echo "::error::Could not extract database ID from output (connection string redacted)"
58+
exit 1
59+
fi
60+
if [ -z "$DB_STRING" ]; then
61+
echo "::error::Could not extract connection string from output"
62+
exit 1
63+
fi
64+
65+
# Mask the full connection string and its password component before any further output.
66+
echo "::add-mask::$DB_STRING"
67+
DB_PASSWORD=$(printf '%s' "$DB_STRING" | sed -nE 's#^postgresql://[^:]+:([^@]+)@.*$#\1#p')
68+
if [ -n "$DB_PASSWORD" ]; then
69+
echo "::add-mask::$DB_PASSWORD"
70+
fi
71+
72+
echo "db_id=$DB_ID" >> "$GITHUB_OUTPUT"
73+
echo "db_string=$DB_STRING" >> "$GITHUB_OUTPUT"
74+
echo "Database $DB_ID created successfully"
75+
76+
- name: Install psql
77+
run: sudo apt-get install -y -qq postgresql-client
78+
79+
- name: Monitor sync progress
80+
env:
81+
DB_STRING: ${{ steps.create-db.outputs.db_string }}
82+
DB_ID: ${{ steps.create-db.outputs.db_id }}
83+
run: |
84+
POLL_INTERVAL=15
85+
MAX_POLLS=240
86+
PREV_TOTAL=0
87+
PREV_TIME=$(date +%s)
88+
START_TIME=$PREV_TIME
89+
90+
for i in $(seq 1 $MAX_POLLS); do
91+
STATUS=$(stripe databases retrieve "$DB_ID" 2>&1 | grep -oE 'backfilling|ready|error|failed' | head -1)
92+
93+
SUM_EXPR=$(psql "$DB_STRING" -t -A -c "
94+
SELECT COALESCE(
95+
string_agg('(SELECT count(*) FROM public.' || quote_ident(table_name) || ')', ' + '),
96+
'0'
97+
) FROM information_schema.tables
98+
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
99+
" 2>/dev/null || echo "0")
100+
TOTAL=$(psql "$DB_STRING" -t -A -c "SELECT $SUM_EXPR" 2>/dev/null || echo "0")
101+
102+
NOW=$(date +%s)
103+
ELAPSED=$((NOW - PREV_TIME))
104+
if [ "$ELAPSED" -gt 0 ] && [ "$PREV_TOTAL" -gt 0 ]; then
105+
DELTA=$((TOTAL - PREV_TOTAL))
106+
RPS=$((DELTA / ELAPSED))
107+
echo "[poll $i] status=$STATUS total_rows=$TOTAL delta=$DELTA rows_per_sec=$RPS"
108+
else
109+
echo "[poll $i] status=$STATUS total_rows=$TOTAL (baseline)"
110+
fi
111+
112+
if [ "$STATUS" = "ready" ]; then
113+
TOTAL_ELAPSED=$(( NOW - START_TIME ))
114+
echo ""
115+
echo "Sync complete in ${TOTAL_ELAPSED}s"
116+
break
117+
fi
118+
if [ "$STATUS" = "error" ] || [ "$STATUS" = "failed" ]; then
119+
echo "::error::Database entered $STATUS state"
120+
exit 1
121+
fi
122+
123+
PREV_TOTAL=$TOTAL
124+
PREV_TIME=$NOW
125+
sleep $POLL_INTERVAL
126+
done
127+
128+
if [ "$STATUS" != "ready" ]; then
129+
echo "::error::Timed out waiting for sync to complete"
130+
exit 1
131+
fi
132+
133+
echo ""
134+
echo "=== Final table breakdown ==="
135+
psql "$DB_STRING" -c "
136+
SELECT relname AS table_name, n_live_tup AS row_count
137+
FROM pg_stat_user_tables
138+
WHERE schemaname = 'public'
139+
ORDER BY n_live_tup DESC;
140+
"
141+
142+
- name: Sigma validation
143+
env:
144+
STRIPE_API_KEY: ${{ secrets.GOLDI_LIVE_KEY }}
145+
DATABASE_URL: ${{ steps.create-db.outputs.db_string }}
146+
run: |
147+
set +e
148+
node scripts/reconcile-sigma-vs-postgres.js
149+
STATUS=$?
150+
if [ "$STATUS" -ne 0 ]; then
151+
echo "::warning title=Sigma reconciliation::Postgres is missing rows that exist in Sigma. See the job log for the per-table diff and the list of missing IDs with their created timestamps."
152+
exit "$STATUS"
153+
fi
154+
155+
- name: Delete Stripe database
156+
if: always() && steps.create-db.outputs.db_id
157+
run: |
158+
echo "Deleting database ${{ steps.create-db.outputs.db_id }}..."
159+
echo "remove StripeDB" | stripe databases delete ${{ steps.create-db.outputs.db_id }}

0 commit comments

Comments
 (0)