@@ -37,46 +37,28 @@ every request to BigQuery:
3737#! /bin/bash
3838# log-to-bigquery.sh
3939
40- # Configure BigQuery
41- PROJECT=" my-project"
42- DATASET=" httpjail_logs"
43- TABLE=" requests"
44- BATCH_FILE=" /tmp/requests-$$ .ndjson"
45-
46- # Process requests in batches
47- batch_count=0
48- max_batch=100
40+ LOG_FILE=" /tmp/requests-$$ .ndjson"
4941
42+ # Process each request
5043while read -r line; do
51- # Parse and enrich the request
44+ # Append to newline-delimited JSON file
5245 echo " $line " | jq -c ' {
5346 timestamp: now | todate,
5447 url: .url,
5548 method: .method,
5649 host: .host,
57- path: .path,
58- requester_ip: .requester_ip
59- }' >> " $BATCH_FILE "
60-
61- batch_count=$(( batch_count + 1 ))
62-
63- # Load batch when threshold reached
64- if [ $batch_count -ge $max_batch ]; then
65- bq load --source_format=NEWLINE_DELIMITED_JSON \
66- --autodetect \
67- " $PROJECT :$DATASET .$TABLE " \
68- " $BATCH_FILE "
69-
70- > " $BATCH_FILE " # Clear batch file
71- batch_count=0
72- fi
50+ path: .path
51+ }' >> " $LOG_FILE "
7352
7453 # Allow all requests
7554 echo " true"
7655done
7756
78- # Load any remaining records on exit
79- trap ' bq load --source_format=NEWLINE_DELIMITED_JSON --autodetect "$PROJECT:$DATASET.$TABLE" "$BATCH_FILE"' EXIT
57+ # On exit, load all data to BigQuery
58+ trap ' bq load --source_format=NEWLINE_DELIMITED_JSON \
59+ --autodetect \
60+ my-project:httpjail_logs.requests \
61+ "$LOG_FILE"' EXIT
8062```
8163
8264Usage:
@@ -85,10 +67,8 @@ Usage:
8567httpjail --proc ./log-to-bigquery.sh --request-log local-backup.log -- your-app
8668```
8769
88- This approach :
70+ This example shows how to :
8971
90- - Batches requests for efficient BigQuery loading
91- - Maintains a local backup in ` local-backup.log `
92- - Uses newline-delimited JSON format (required by BigQuery)
93- - Handles graceful shutdown with trap to load remaining data
94- - Avoids per-request overhead of streaming inserts
72+ - Collect requests in newline-delimited JSON format
73+ - Load data to BigQuery on process exit
74+ - Combine local logging with cloud analytics
0 commit comments