Skip to content

Commit 8f60dc5

Browse files
jmene442jmene442
authored
[LOGSS-9925] Updating GitLab pipelines to reflect timestamp changes (DataDog#23292)
* update gitlab pipeline configuration Signed-off-by: Justin Menegoni <justin.menegoni@datadoghq.com> * Update gitlab_tests.yaml to add sample files Signed-off-by: Justin Menegoni <justin.menegoni@datadoghq.com> * Update gitlab.yaml Signed-off-by: Justin Menegoni <justin.menegoni@datadoghq.com> * refactor pipeline change, reset tests file for new tests based on change Signed-off-by: Justin Menegoni <justin.menegoni@datadoghq.com> * adding samples to rerun Signed-off-by: Justin Menegoni <justin.menegoni@datadoghq.com> * refactor tests to correct output Signed-off-by: Justin Menegoni <justin.menegoni@datadoghq.com> --------- Signed-off-by: Justin Menegoni <justin.menegoni@datadoghq.com>
1 parent 411c31d commit 8f60dc5

2 files changed

Lines changed: 109 additions & 1 deletion

File tree

‎gitlab/assets/logs/gitlab.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,32 @@ pipeline:
288288
matchRules: |
289289
gitlab.production Started %{_method} "%{_url}" for %{_client_ip} at %{_date_access}(\n|\t|\s)+Processing by (%{_class}::)?%{_controller}\#%{_action} as %{_format}(\t|\s|\n)+(%{_details})?Completed %{_status_code} %{word} in %{_duration}ms \(Views: %{_view}ms \| ActiveRecord: %{_activerecord}ms\)
290290
291+
- type: grok-parser
292+
name: GitLab Runner Job Logs
293+
enabled: true
294+
source: message
295+
samples:
296+
- '2026-03-08T15:08:43.415459Z 01E {"event": "pikepdf C++ to Python
297+
logger bridge initialized", "level": "info", "timestamp":
298+
"2026-03-08T15:08:43.407711Z", "logger": "pikepdf._core", "file":
299+
"<frozen importlib._bootstrap>:488", "TraceId": "0", "SpanId": "0",
300+
"service_environment": "test-docker"}'
301+
- '2026-03-08T15:08:37.699056Z 01E {"event": "Creating database. Use
302+
`-v/--verbose` to see all logs", "logger": "app.cli", "level": "info",
303+
"timestamp": "2026-03-08T15:08:37.698374Z", "TraceId": "0", "SpanId":
304+
"0", "service_environment": "test-docker"}'
305+
- '2026-03-08T15:08:47.505749Z 01E {"customer_fk_count": 73,
306+
"customer_col_count": 49, "customer_endpoint_count": 251, "event":
307+
"shared money stats", "pg_backend_pid": 71, "logger": "__main__",
308+
"level": "info", "timestamp": "2026-03-08T15:08:47.505258Z",
309+
"TraceId": "0", "SpanId": "0", "service_environment": "test-docker"}'
310+
grok:
311+
supportRules: |-
312+
_date %{date("yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ"):date}
313+
_stream_number %{regex("[0-9a-fA-F]{2}"):stream_number}
314+
_stream_type %{notSpace:stream_type}
315+
matchRules: gitlab_runner_json %{_date}\s+%{_stream_number}%{_stream_type}\s+%{data::json}
316+
291317
# date
292318
- type: date-remapper
293319
name: Define `time`, `date`, `ts`, `date_access` as the official date of the log
@@ -386,6 +412,7 @@ pipeline:
386412
enabled: true
387413
sources:
388414
- severity
415+
- level
389416
390417
# ruby/rails
391418
- type: attribute-remapper

‎gitlab/assets/logs/gitlab_tests.yaml‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ tests:
1010
gitlab_date: 1560846559197
1111
level: "info"
1212
message: "gossip settled; proceeding"
13+
status: "info"
1314
tags:
1415
- "source:LOGS_SOURCE"
1516
timestamp: 1
@@ -39,6 +40,7 @@ tests:
3940
system: "grpc"
4041
time: "2019-06-17T17:20:03Z"
4142
message: "finished unary call with code OK"
43+
status: "info"
4244
tags:
4345
- "source:LOGS_SOURCE"
4446
timestamp: 1560792003000
@@ -579,6 +581,7 @@ tests:
579581
jobid: "ca8a29fc8d6a1d933812e510"
580582
timestamp: 1560767410793
581583
message: "2019-06-17_10:30:10.79384 2019-06-17T10:30:10.793Z 1332 TID-oujf7ktz4 PagesDomainVerificationCronWorker JID-ca8a29fc8d6a1d933812e510 INFO: start"
584+
status: "info"
582585
tags:
583586
- "source:LOGS_SOURCE"
584587
timestamp: 1
@@ -597,6 +600,7 @@ tests:
597600
jobid: "ca8a29fc8d6a1d933812e510"
598601
timestamp: 1560767410802
599602
message: "2019-06-17_10:30:10.80225 2019-06-17T10:30:10.802Z 1332 TID-oujf7ktz4 PagesDomainVerificationCronWorker JID-ca8a29fc8d6a1d933812e510 INFO: done: 0.008 sec"
603+
status: "info"
600604
tags:
601605
- "source:LOGS_SOURCE"
602606
timestamp: 1
@@ -611,6 +615,7 @@ tests:
611615
pid: 1332
612616
timestamp: 1560767626568
613617
message: "2019-06-17_10:33:46.56889 2019-06-17T10:33:46.568Z 1332 TID-oujf7ktz4 INFO: Cleaning working queues"
618+
status: "info"
614619
tags:
615620
- "source:LOGS_SOURCE"
616621
timestamp: 1
@@ -786,6 +791,7 @@ tests:
786791
time: "2019-06-18T00:20:10+00:00"
787792
url: "http://127.0.0.1:8080/api/v4/internal/pre_receive"
788793
message: "finished HTTP request"
794+
status: "info"
789795
tags:
790796
- "source:LOGS_SOURCE"
791797
timestamp: 1560817210000
@@ -798,6 +804,7 @@ tests:
798804
time: "2019-06-18T08:28:31Z"
799805
version: "gitlab-workhorse v8.7.0-20190610.161452"
800806
message: "Starting"
807+
status: "info"
801808
tags:
802809
- "source:LOGS_SOURCE"
803810
timestamp: 1560846511000
@@ -810,6 +817,7 @@ tests:
810817
logger:
811818
thread_id: "9047"
812819
message: "I, [2015-02-13T06:14:46.680381 #9047] INFO -- : Refreshing Gem list"
820+
status: "info"
813821
tags:
814822
- "source:LOGS_SOURCE"
815823
timestamp: 1423808086680
@@ -840,6 +848,7 @@ tests:
840848
ip: "123.123.123.123"
841849
views: 0.2
842850
message: "I, [1999-03-03T02:34:24.895701 #19074] INFO -- : Started GET \"/api/v1/example/query\" for 123.123.123.123 at 1999-03-03 02:34:24 +0000 Processing by MyCheck::CheckController#index as JSON Completed 200 OK in 41ms (Views: 0.2ms | ActiveRecord: 6.2ms)"
851+
status: "info"
843852
tags:
844853
- "source:LOGS_SOURCE"
845854
timestamp: 920428464895
@@ -853,6 +862,7 @@ tests:
853862
name: "Main"
854863
thread_id: "19074"
855864
message: "I, [1999-03-03T02:34:24.895701 #19074] INFO -- Main: Some error."
865+
status: "info"
856866
tags:
857867
- "source:LOGS_SOURCE"
858868
timestamp: 920428464895
@@ -893,6 +903,7 @@ tests:
893903
thread_id: "19074"
894904
views: 0.2
895905
message: "I, [1999-03-03T02:34:24.895701 #19074] INFO -- : Completed 200 OK in 41ms (Views: 0.2ms | ActiveRecord: 6.2ms)"
906+
status: "info"
896907
tags:
897908
- "source:LOGS_SOURCE"
898909
timestamp: 920428464895
@@ -907,6 +918,7 @@ tests:
907918
logger:
908919
thread_id: "648"
909920
message: "I, [2019-06-18T15:18:18.173873 #648] INFO -- : listening on addr=127.0.0.1:8080 fd=21"
921+
status: "info"
910922
tags:
911923
- "source:LOGS_SOURCE"
912924
timestamp: 1560871098173
@@ -928,6 +940,7 @@ tests:
928940
client:
929941
ip: "123.123.123.123"
930942
message: "I, [1999-03-03T02:34:24.895701 #19074] INFO -- : Started GET \"/api/v1/example/query\" for 123.123.123.123 at 1999-03-03 02:34:24+0000"
943+
status: "info"
931944
tags:
932945
- "source:LOGS_SOURCE"
933946
timestamp: 920428464895
@@ -943,6 +956,7 @@ tests:
943956
thread_id: "19074"
944957
processor: "MyCheck::CheckController#index"
945958
message: "I, [1999-03-03T02:34:24.895701 #19074] INFO -- : Processing by MyCheck::CheckController#index as JSON"
959+
status: "info"
946960
tags:
947961
- "source:LOGS_SOURCE"
948962
timestamp: 920428464895
@@ -959,6 +973,7 @@ tests:
959973
id: 1234567
960974
timestamp: 153012312
961975
message: "I, [2018-06-27T08:01:52.991728 #1] INFO -- : Received: {\"action\":\"cancel\",\"order\":{\"id\":1234567,\"timestamp\":153012312}}"
976+
status: "info"
962977
tags:
963978
- "source:LOGS_SOURCE"
964979
timestamp: 1530086512991
@@ -974,6 +989,7 @@ tests:
974989
thread_id: "50"
975990
worker: 2
976991
message: "E, [2015-10-21T22:12:52.402356 #50] ERROR -- : reaped #<Process::Status: pid 305 SIGKILL (signal 9)> worker=2"
992+
status: "error"
977993
tags:
978994
- "source:LOGS_SOURCE"
979995
timestamp: 1445465572402
@@ -986,6 +1002,7 @@ tests:
9861002
logger:
9871003
thread_id: "1310"
9881004
message: "D, [2019-06-18T00:13:22.160544 #1310] DEBUG -- : waiting 31.0s after suspend/hibernation"
1005+
status: "debug"
9891006
tags:
9901007
- "source:LOGS_SOURCE"
9911008
timestamp: 1560816802160
@@ -999,6 +1016,7 @@ tests:
9991016
thread_id: "819"
10001017
worker: 0
10011018
message: "I, [2019-06-18T15:18:18.345105 #819] INFO -- : worker=0 ready"
1019+
status: "info"
10021020
tags:
10031021
- "source:LOGS_SOURCE"
10041022
timestamp: 1560871098345
@@ -1016,6 +1034,7 @@ tests:
10161034
jobid: "39a3193a8e0f66844f802029"
10171035
timestamp: 1664956770904
10181036
message: "2022-10-05T07:59:30.904Z pid=64 tid=a0tc class=Ci::RefDeleteUnlockArtifactsWorker jid=39a3193a8e0f66844f802029 elapsed=0.012 INFO: done"
1037+
status: "info"
10191038
tags:
10201039
- "source:LOGS_SOURCE"
10211040
timestamp: 1
@@ -1032,7 +1051,69 @@ tests:
10321051
jobid: "21b448f228c0a5cc1152d144"
10331052
timestamp: 1664886475010
10341053
message: "2022-10-04T12:27:55.010Z pid=65 tid=s5cg9 class=BuildHooksWorker jid=21b448f228c0a5cc1152d144 INFO: start"
1054+
status: "info"
10351055
tags:
10361056
- "source:LOGS_SOURCE"
10371057
timestamp: 1
1038-
1058+
-
1059+
sample: "2026-03-08T15:08:33.635990Z 01E {\"event\": \"pikepdf C++ to Python logger bridge initialized\", \"level\": \"info\", \"timestamp\": \"2026-03-08T15:08:33.628330Z\", \"logger\": \"pikepdf._core\", \"file\": \"<frozen importlib._bootstrap>:488\", \"TraceId\": \"0\", \"SpanId\": \"0\", \"service_environment\": \"test-docker\"}"
1060+
result:
1061+
custom:
1062+
SpanId: "0"
1063+
TraceId: "0"
1064+
date: 1772982513635
1065+
event: "pikepdf C++ to Python logger bridge initialized"
1066+
file: "<frozen importlib._bootstrap>:488"
1067+
level: "info"
1068+
logger: "pikepdf._core"
1069+
service_environment: "test-docker"
1070+
stream_number: "01"
1071+
stream_type: "E"
1072+
timestamp: "2026-03-08T15:08:33.628330Z"
1073+
message: "2026-03-08T15:08:33.635990Z 01E {\"event\": \"pikepdf C++ to Python logger bridge initialized\", \"level\": \"info\", \"timestamp\": \"2026-03-08T15:08:33.628330Z\", \"logger\": \"pikepdf._core\", \"file\": \"<frozen importlib._bootstrap>:488\", \"TraceId\": \"0\", \"SpanId\": \"0\", \"service_environment\": \"test-docker\"}"
1074+
status: "info"
1075+
tags:
1076+
- "source:LOGS_SOURCE"
1077+
timestamp: 1772982513635
1078+
-
1079+
sample: "2026-03-08T15:08:37.699056Z 01E {\"event\": \"Creating database. Use `-v/--verbose` to see all logs\", \"logger\": \"app.cli\", \"level\": \"info\", \"timestamp\": \"2026-03-08T15:08:37.698374Z\", \"TraceId\": \"0\", \"SpanId\": \"0\", \"service_environment\": \"test-docker\"}"
1080+
result:
1081+
custom:
1082+
SpanId: "0"
1083+
TraceId: "0"
1084+
date: 1772982517699
1085+
event: "Creating database. Use `-v/--verbose` to see all logs"
1086+
level: "info"
1087+
logger: "app.cli"
1088+
service_environment: "test-docker"
1089+
stream_number: "01"
1090+
stream_type: "E"
1091+
timestamp: "2026-03-08T15:08:37.698374Z"
1092+
message: "2026-03-08T15:08:37.699056Z 01E {\"event\": \"Creating database. Use `-v/--verbose` to see all logs\", \"logger\": \"app.cli\", \"level\": \"info\", \"timestamp\": \"2026-03-08T15:08:37.698374Z\", \"TraceId\": \"0\", \"SpanId\": \"0\", \"service_environment\": \"test-docker\"}"
1093+
status: "info"
1094+
tags:
1095+
- "source:LOGS_SOURCE"
1096+
timestamp: 1772982517699
1097+
-
1098+
sample: "2026-03-08T15:08:47.505749Z 01E {\"customer_fk_count\": 73, \"customer_col_count\": 49, \"customer_endpoint_count\": 251, \"event\": \"shared money stats\", \"pg_backend_pid\": 71, \"logger\": \"__main__\", \"level\": \"info\", \"timestamp\": \"2026-03-08T15:08:47.505258Z\", \"TraceId\": \"0\", \"SpanId\": \"0\", \"service_environment\": \"test-docker\"}"
1099+
result:
1100+
custom:
1101+
SpanId: "0"
1102+
TraceId: "0"
1103+
customer_col_count: 49
1104+
customer_endpoint_count: 251
1105+
customer_fk_count: 73
1106+
date: 1772982527505
1107+
event: "shared money stats"
1108+
level: "info"
1109+
logger: "__main__"
1110+
pg_backend_pid: 71
1111+
service_environment: "test-docker"
1112+
stream_number: "01"
1113+
stream_type: "E"
1114+
timestamp: "2026-03-08T15:08:47.505258Z"
1115+
message: "2026-03-08T15:08:47.505749Z 01E {\"customer_fk_count\": 73, \"customer_col_count\": 49, \"customer_endpoint_count\": 251, \"event\": \"shared money stats\", \"pg_backend_pid\": 71, \"logger\": \"__main__\", \"level\": \"info\", \"timestamp\": \"2026-03-08T15:08:47.505258Z\", \"TraceId\": \"0\", \"SpanId\": \"0\", \"service_environment\": \"test-docker\"}"
1116+
status: "info"
1117+
tags:
1118+
- "source:LOGS_SOURCE"
1119+
timestamp: 1772982527505

0 commit comments

Comments
 (0)