Skip to content

Commit 874083d

Browse files
gsoundargsoundar
committed
feat: add log4j-iceberg appender module for writing logs to Apache Iceberg tables
Adds a new log4j-iceberg module that provides an IcebergAppender plugin for writing log events as Parquet-backed rows in an Iceberg table. The appender buffers events and periodically flushes them as Parquet data files committed to the table. Commit failures are retried with exponential backoff and table metadata refresh; persistent failures crash rather than silently dropping data. Uses Apache Iceberg 1.10.1 and Parquet 1.16.0. Supports any Iceberg catalog implementation (Hadoop, REST, etc.) via configuration.
1 parent c6d8dcc commit 874083d

6 files changed

Lines changed: 764 additions & 0 deletions

File tree

‎log4j-iceberg/pom.xml‎

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to you under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
20+
<modelVersion>4.0.0</modelVersion>
21+
22+
<parent>
23+
<groupId>org.apache.logging.log4j</groupId>
24+
<artifactId>log4j</artifactId>
25+
<version>${revision}</version>
26+
<relativePath>../log4j-parent</relativePath>
27+
</parent>
28+
29+
<artifactId>log4j-iceberg</artifactId>
30+
31+
<name>Apache Log4j Iceberg</name>
32+
<description>Apache Iceberg appender for Log4j. Writes log events as rows in an Iceberg table using Parquet files.</description>
33+
34+
<properties>
35+
<Fragment-Host>org.apache.logging.log4j.core</Fragment-Host>
36+
<iceberg.version>1.10.1</iceberg.version>
37+
<parquet.version>1.16.0</parquet.version>
38+
<hadoop.version>3.4.1</hadoop.version>
39+
</properties>
40+
41+
<dependencyManagement>
42+
<dependencies>
43+
<dependency>
44+
<groupId>org.xerial.snappy</groupId>
45+
<artifactId>snappy-java</artifactId>
46+
<version>1.1.10.7</version>
47+
</dependency>
48+
<dependency>
49+
<groupId>io.airlift</groupId>
50+
<artifactId>aircompressor</artifactId>
51+
<version>2.0.2</version>
52+
</dependency>
53+
<dependency>
54+
<groupId>org.apache.commons</groupId>
55+
<artifactId>commons-text</artifactId>
56+
<version>1.11.0</version>
57+
</dependency>
58+
</dependencies>
59+
</dependencyManagement>
60+
61+
<dependencies>
62+
63+
<dependency>
64+
<groupId>org.apache.logging.log4j</groupId>
65+
<artifactId>log4j-core</artifactId>
66+
</dependency>
67+
68+
<!-- Iceberg core + Parquet support -->
69+
<dependency>
70+
<groupId>org.apache.iceberg</groupId>
71+
<artifactId>iceberg-core</artifactId>
72+
<version>${iceberg.version}</version>
73+
</dependency>
74+
<dependency>
75+
<groupId>org.apache.iceberg</groupId>
76+
<artifactId>iceberg-parquet</artifactId>
77+
<version>${iceberg.version}</version>
78+
</dependency>
79+
<dependency>
80+
<groupId>org.apache.iceberg</groupId>
81+
<artifactId>iceberg-data</artifactId>
82+
<version>${iceberg.version}</version>
83+
</dependency>
84+
<dependency>
85+
<groupId>org.apache.iceberg</groupId>
86+
<artifactId>iceberg-api</artifactId>
87+
<version>${iceberg.version}</version>
88+
</dependency>
89+
90+
<!-- Parquet -->
91+
<dependency>
92+
<groupId>org.apache.parquet</groupId>
93+
<artifactId>parquet-avro</artifactId>
94+
<version>${parquet.version}</version>
95+
</dependency>
96+
97+
<!-- Hadoop (needed by Iceberg for local/S3 file IO and Parquet) -->
98+
<dependency>
99+
<groupId>org.apache.hadoop</groupId>
100+
<artifactId>hadoop-common</artifactId>
101+
<version>${hadoop.version}</version>
102+
<exclusions>
103+
<exclusion>
104+
<groupId>org.slf4j</groupId>
105+
<artifactId>slf4j-reload4j</artifactId>
106+
</exclusion>
107+
<exclusion>
108+
<groupId>ch.qos.reload4j</groupId>
109+
<artifactId>reload4j</artifactId>
110+
</exclusion>
111+
</exclusions>
112+
</dependency>
113+
<dependency>
114+
<groupId>org.apache.hadoop</groupId>
115+
<artifactId>hadoop-mapreduce-client-core</artifactId>
116+
<version>${hadoop.version}</version>
117+
<exclusions>
118+
<exclusion>
119+
<groupId>org.slf4j</groupId>
120+
<artifactId>slf4j-reload4j</artifactId>
121+
</exclusion>
122+
<exclusion>
123+
<groupId>ch.qos.reload4j</groupId>
124+
<artifactId>reload4j</artifactId>
125+
</exclusion>
126+
<exclusion>
127+
<groupId>org.apache.hadoop</groupId>
128+
<artifactId>hadoop-yarn-common</artifactId>
129+
</exclusion>
130+
<exclusion>
131+
<groupId>org.apache.hadoop</groupId>
132+
<artifactId>hadoop-yarn-client</artifactId>
133+
</exclusion>
134+
<exclusion>
135+
<groupId>org.apache.hadoop</groupId>
136+
<artifactId>hadoop-hdfs-client</artifactId>
137+
</exclusion>
138+
</exclusions>
139+
</dependency>
140+
141+
<!-- Test dependencies -->
142+
<dependency>
143+
<groupId>org.junit.jupiter</groupId>
144+
<artifactId>junit-jupiter-engine</artifactId>
145+
<scope>test</scope>
146+
</dependency>
147+
<dependency>
148+
<groupId>org.apache.logging.log4j</groupId>
149+
<artifactId>log4j-core-test</artifactId>
150+
<scope>test</scope>
151+
</dependency>
152+
<dependency>
153+
<groupId>org.mockito</groupId>
154+
<artifactId>mockito-core</artifactId>
155+
<scope>test</scope>
156+
</dependency>
157+
<dependency>
158+
<groupId>org.assertj</groupId>
159+
<artifactId>assertj-core</artifactId>
160+
<version>3.26.3</version>
161+
<scope>test</scope>
162+
</dependency>
163+
164+
</dependencies>
165+
166+
<build>
167+
<plugins>
168+
<!-- No previous release exists for this new module -->
169+
<plugin>
170+
<groupId>biz.aQute.bnd</groupId>
171+
<artifactId>bnd-baseline-maven-plugin</artifactId>
172+
<executions>
173+
<execution>
174+
<id>check-api-compat</id>
175+
<goals>
176+
<goal>baseline</goal>
177+
</goals>
178+
<configuration>
179+
<skip>true</skip>
180+
</configuration>
181+
</execution>
182+
</executions>
183+
</plugin>
184+
<plugin>
185+
<groupId>org.jacoco</groupId>
186+
<artifactId>jacoco-maven-plugin</artifactId>
187+
<version>0.8.12</version>
188+
<executions>
189+
<execution>
190+
<goals>
191+
<goal>prepare-agent</goal>
192+
</goals>
193+
</execution>
194+
<execution>
195+
<id>report</id>
196+
<goals>
197+
<goal>report</goal>
198+
</goals>
199+
<phase>test</phase>
200+
</execution>
201+
<execution>
202+
<id>check</id>
203+
<goals>
204+
<goal>check</goal>
205+
</goals>
206+
<configuration>
207+
<rules>
208+
<rule>
209+
<element>BUNDLE</element>
210+
<limits>
211+
<limit>
212+
<counter>LINE</counter>
213+
<value>COVEREDRATIO</value>
214+
<minimum>1.00</minimum>
215+
</limit>
216+
</limits>
217+
</rule>
218+
</rules>
219+
</configuration>
220+
</execution>
221+
</executions>
222+
</plugin>
223+
</plugins>
224+
</build>
225+
226+
</project>

0 commit comments

Comments
 (0)