Fluent-bit config updates (#341)

* Fluent-bit config updates

Specify "DB" to store file offsets so fluent-bit doesn't send duplicated
logs to output

* mount fluent-bit db folder with rw permissions

Author: june-hua

startupscript/butane/aws/fluent-bit.conf

@@ -26,8 +26,11 @@
     Parser docker
     Path /var/lib/docker/containers/*/*.log
     Path_key filepath
+    DB /var/lib/fluent-bit/flb_docker.db
+    Read_from_Head true
     Skip_Long_Lines On
     Mem_Buf_Limit 10MB
+    Refresh_Interval 5
 
 [OUTPUT]
     Name cloudwatch_logs

startupscript/butane/gcp/fluent-bit.conf

@@ -26,9 +26,12 @@
     Parser docker
     Path /var/lib/docker/containers/*/*.log
     Path_key filepath
+    DB /var/lib/fluent-bit/flb_docker.db
+    Read_from_Head true
     Skip_Long_Lines On
     Mem_Buf_Limit 10MB
-    
+    Refresh_Interval 5
+
 [OUTPUT]
     name stackdriver
     match vm-*

startupscript/butane/run-fluent-bit.sh

@@ -43,6 +43,9 @@ readonly CLOUD="$1"
 # Configuration with defaults
 readonly FLUENT_BIT_IMAGE="${FLUENT_BIT_IMAGE:-cr.fluentbit.io/fluent/fluent-bit:2.0-debug}"
 
+# Db directory to store file offsets
+mkdir -p /var/lib/fluent-bit
+
 # Build Docker run command arguments
 DOCKER_ARGS=(
     --rm
@@ -51,6 +54,7 @@ DOCKER_ARGS=(
     -v /etc/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro
     -v /var/log/journal:/var/log/journal:ro
     -v /var/lib/docker/containers:/var/lib/docker/containers:ro
+    -v /var/lib/fluent-bit:/var/lib/fluent-bit
 )
 
 # Add cloud-specific configuration