Skip to content

Commit dadbe7a

Browse files
Vasavi-YarramneniVasavi-Yarramneni
committed
added syslog receivers support to AST
1 parent de95eaf commit dadbe7a

File tree

20 files changed

+16148
-2
lines changed

20 files changed

+16148
-2
lines changed

‎docker-compose.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ version: '3'
33
volumes:
44
prometheus:
55
grafana:
6+
clickhouse:
67

78
services:
89
prometheus:
@@ -55,5 +56,24 @@ services:
5556
networks:
5657
- 7lc_network
5758

59+
clickhouse:
60+
image: clickhouse/clickhouse-server:25.1.4
61+
user: "101:101"
62+
hostname: clickhouse
63+
ulimits:
64+
nofile:
65+
soft: 262144
66+
hard: 262144
67+
volumes:
68+
- ./services/clickhouse/config.d:/etc/clickhouse-server/config.d
69+
- ./services/clickhouse/users.d:/etc/clickhouse-server/users.d
70+
- ./services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh:/docker-entrypoint-initdb.d/init-otel-db.sh
71+
- clickhouse:/var/lib/clickhouse
72+
env_file: ".env"
73+
healthcheck:
74+
test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
75+
networks:
76+
- 7lc_network
77+
5878
networks:
5979
7lc_network:

‎services/clickhouse/config.d/config.xml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<clickhouse replace="true">
2+
<logger>
3+
<level>notice</level>
4+
<console>true</console>
5+
</logger>
6+
<display_name>clickhouse</display_name>
7+
<listen_host from_env="HOSTNAME" />
8+
<http_port>8123</http_port>
9+
<tcp_port>9000</tcp_port>
10+
<user_directories>
11+
<users_xml>
12+
<path>users.xml</path>
13+
</users_xml>
14+
<local_directory>
15+
<path>/var/lib/clickhouse/access/</path>
16+
</local_directory>
17+
</user_directories>
18+
<prometheus>
19+
<endpoint>/metrics</endpoint>
20+
<port>9126</port>
21+
<metrics>true</metrics>
22+
<events>true</events>
23+
<asynchronous_metrics>true</asynchronous_metrics>
24+
</prometheus>
25+
</clickhouse>

‎services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "=== OTEL DB Initialization Script Started ==="
5+
6+
clickhouse client <<-EOSQL
7+
CREATE DATABASE IF NOT EXISTS otel;
8+
EOSQL
9+
10+
# Create the null log table for incoming otel data
11+
clickhouse client <<-EOSQL
12+
CREATE TABLE IF NOT EXISTS otel.otel_sslo_logs_null
13+
(
14+
Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
15+
ObservedTimestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
16+
TraceId String CODEC(ZSTD(1)),
17+
SpanId String CODEC(ZSTD(1)),
18+
TraceFlags UInt8,
19+
SeverityText LowCardinality(String) CODEC(ZSTD(1)),
20+
SeverityNumber UInt8,
21+
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
22+
Body String CODEC(ZSTD(1)),
23+
ResourceSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
24+
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
25+
ScopeSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
26+
ScopeName String CODEC(ZSTD(1)),
27+
ScopeVersion LowCardinality(String) CODEC(ZSTD(1)),
28+
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
29+
LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
30+
)
31+
ENGINE = Null;
32+
EOSQL
33+
34+
35+
# Create the actual table which logs will be stored in
36+
clickhouse client <<-EOSQL
37+
38+
CREATE TABLE IF NOT EXISTS otel.sslo_logs
39+
(
40+
Timestamp DateTime,
41+
ObservedTimestamp DateTime,
42+
hostname String CODEC(ZSTD(1)),
43+
flow_id String CODEC(ZSTD(1)),
44+
vip String CODEC(ZSTD(1)),
45+
l4_protocol LowCardinality(String) CODEC(ZSTD(1)),
46+
src_ip String CODEC(ZSTD(1)),
47+
src_port UInt16,
48+
dst_ip String CODEC(ZSTD(1)),
49+
dst_port UInt16,
50+
client_ssl_protocol LowCardinality(String) CODEC(ZSTD(1)),
51+
client_ssl_cipher LowCardinality(String) CODEC(ZSTD(1)),
52+
server_ssl_protocol LowCardinality(String) CODEC(ZSTD(1)),
53+
server_ssl_cipher LowCardinality(String) CODEC(ZSTD(1)),
54+
l7_protocol LowCardinality(String) CODEC(ZSTD(1)),
55+
sslo_host String CODEC(ZSTD(1)),
56+
decryption_status LowCardinality(String) CODEC(ZSTD(1)),
57+
duration UInt64,
58+
service_path String CODEC(ZSTD(1)),
59+
client_bytes_in UInt64,
60+
client_bytes_out UInt64,
61+
server_bytes_in UInt64,
62+
server_bytes_out UInt64,
63+
client_tls_handshake LowCardinality(String) CODEC(ZSTD(1)),
64+
server_tls_handshake LowCardinality(String) CODEC(ZSTD(1)),
65+
reset_cause LowCardinality(String) CODEC(ZSTD(1)),
66+
policy_rule LowCardinality(String) CODEC(ZSTD(1)),
67+
url_category LowCardinality(String) CODEC(ZSTD(1)),
68+
ingress String CODEC(ZSTD(1)),
69+
egress String CODEC(ZSTD(1)),
70+
)
71+
ENGINE = MergeTree
72+
ORDER BY (Timestamp)
73+
TTL Timestamp + INTERVAL 24 HOUR;
74+
EOSQL
75+
76+
# Create the Materialized View that populates the access log table from incoming
77+
# null logs.
78+
clickhouse client <<-EOSQL
79+
CREATE MATERIALIZED VIEW IF NOT EXISTS otel.sslo_logs_mv TO otel.sslo_logs AS
80+
SELECT Timestamp::DateTime AS Timestamp,
81+
ObservedTimestamp::DateTime AS ObservedTimestamp,
82+
LogAttributes['hostname'] AS hostname,
83+
LogAttributes['flow_id'] AS flow_id,
84+
LogAttributes['vip'] AS vip,
85+
LogAttributes['l4_protocol'] AS l4_protocol,
86+
LogAttributes['src_ip'] AS src_ip,
87+
LogAttributes['src_port'] AS src_port,
88+
LogAttributes['dst_ip'] AS dst_ip,
89+
LogAttributes['dst_port'] AS dst_port,
90+
LogAttributes['client_ssl_protocol'] AS client_ssl_protocol,
91+
LogAttributes['client_ssl_cipher'] AS client_ssl_cipher,
92+
LogAttributes['server_ssl_protocol'] AS server_ssl_protocol,
93+
LogAttributes['server_ssl_cipher'] AS server_ssl_cipher,
94+
LogAttributes['l7_protocol'] AS l7_protocol,
95+
LogAttributes['sslo_host'] AS sslo_host,
96+
LogAttributes['decryption_status'] AS decryption_status,
97+
LogAttributes['duration'] AS duration,
98+
LogAttributes['service_path'] AS service_path,
99+
LogAttributes['client_bytes_in'] AS client_bytes_in,
100+
LogAttributes['client_bytes_out'] AS client_bytes_out,
101+
LogAttributes['server_bytes_in'] AS server_bytes_in,
102+
LogAttributes['server_bytes_out'] AS server_bytes_out,
103+
LogAttributes['client_tls_handshake'] AS client_tls_handshake,
104+
LogAttributes['server_tls_handshake'] AS server_tls_handshake,
105+
LogAttributes['reset_cause'] AS reset_cause,
106+
LogAttributes['policy_rule'] AS policy_rule,
107+
LogAttributes['url_category'] AS url_category,
108+
LogAttributes['ingress'] AS ingress,
109+
LogAttributes['egress'] AS egress
110+
FROM otel.otel_sslo_logs_null;
111+
EOSQL

‎services/clickhouse/users.d/user-default.xml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
<?xml version="1.0"?>
2+
<clickhouse>
3+
<profiles>
4+
<default>
5+
<max_memory_usage>10000000000</max_memory_usage>
6+
<use_uncompressed_cache>0</use_uncompressed_cache>
7+
<load_balancing>in_order</load_balancing>
8+
<log_queries>1</log_queries>
9+
</default>
10+
</profiles>
11+
<users>
12+
<default>
13+
<access_management>1</access_management>
14+
<profile>default</profile>
15+
<networks>
16+
<host>localhost</host>
17+
</networks>
18+
<quota>default</quota>
19+
<access_management>1</access_management>
20+
<named_collection_control>1</named_collection_control>
21+
<show_named_collections>1</show_named_collections>
22+
<show_named_collections_secrets>1</show_named_collections_secrets>
23+
</default>
24+
</users>
25+
<quotas>
26+
<default>
27+
<interval>
28+
<duration>3600</duration>
29+
<queries>0</queries>
30+
<errors>0</errors>
31+
<result_rows>0</result_rows>
32+
<read_rows>0</read_rows>
33+
<execution_time>0</execution_time>
34+
</interval>
35+
</default>
36+
</quotas>
37+
</clickhouse>

‎services/clickhouse/users.d/user-grafana.xml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0"?>
2+
<clickhouse>
3+
<users>
4+
<grafana>
5+
<profile>default</profile>
6+
<networks>
7+
<host>grafana</host>
8+
</networks>
9+
<quota>default</quota>
10+
<password from_env="GRAFANA_CLICKHOUSE_PASSWORD" />
11+
<grants>
12+
<query>GRANT SELECT ON otel.*</query>
13+
</grants>
14+
</grafana>
15+
</users>
16+
</clickhouse>

‎services/clickhouse/users.d/user-otel-collector.xml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<?xml version="1.0"?>
2+
<clickhouse>
3+
<users>
4+
<otel>
5+
<profile>default</profile>
6+
<networks>
7+
<host>otel-collector</host>
8+
</networks>
9+
<grants>
10+
<query>GRANT INSERT ON otel.*</query>
11+
<query>GRANT SELECT ON otel.*</query>
12+
<query>GRANT CREATE DATABASE ON otel.*</query>
13+
<query>GRANT CREATE TABLE ON otel.*</query>
14+
</grants>
15+
<quota>default</quota>
16+
<password from_env="OTEL_COLLECTOR_CLICKHOUSE_PASSWORD" />
17+
</otel>
18+
</users>
19+
</clickhouse>

0 commit comments

Comments
 (0)