feat(kafka): Build kafka-opa-plugin from source (#1177)

* build kafka-opa-plugin from source

* add changelog entry

* Apply suggestions from code review

clean up Dockerfile

Co-authored-by: Lukas Krug <lukas.voetmand@stackable.tech>

---------

Co-authored-by: Lukas Krug <lukas.voetmand@stackable.tech>

Author: labrenbe

CHANGELOG.md

@@ -22,6 +22,7 @@ All notable changes to this project will be documented in this file.
 - spark-connect-client: A new image for Spark connect tests and demos ([#1034])
 - kafka: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1041]).
+- kafka: build kafka-opa-plugin from source ([#1177]).
 - nifi: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
 - nifi: Add [nifi-iceberg-bundle] for NiFi `2.4.0` ([#1060], [#1106]).
@@ -202,6 +203,7 @@ All notable changes to this project will be documented in this file.
 [#1171]: https://github.com/stackabletech/docker-images/pull/1171
 [#1173]: https://github.com/stackabletech/docker-images/pull/1173
 [#1174]: https://github.com/stackabletech/docker-images/pull/1174
+[#1177]: https://github.com/stackabletech/docker-images/pull/1177
 [#1179]: https://github.com/stackabletech/docker-images/pull/1179
 [#1180]: https://github.com/stackabletech/docker-images/pull/1180
 [#1184]: https://github.com/stackabletech/docker-images/pull/1184

conf.py

@@ -38,6 +38,7 @@
 trino_storage_connector = importlib.import_module("trino.storage-connector.versions")
 kafka_testing_tools = importlib.import_module("kafka-testing-tools.versions")
 kcat = importlib.import_module("kafka.kcat.versions")
+kafka_opa_plugin = importlib.import_module("kafka.kafka-opa-plugin.versions")
 testing_tools = importlib.import_module("testing-tools.versions")
 zookeeper = importlib.import_module("zookeeper.versions")
 tools = importlib.import_module("tools.versions")
@@ -73,6 +74,7 @@
     {"name": "trino/storage-connector", "versions": trino_storage_connector.versions},
     {"name": "kafka-testing-tools", "versions": kafka_testing_tools.versions},
     {"name": "kafka/kcat", "versions": kcat.versions},
+    {"name": "kafka/kafka-opa-plugin", "versions": kafka_opa_plugin.versions},
     {"name": "testing-tools", "versions": testing_tools.versions},
     {"name": "zookeeper", "versions": zookeeper.versions},
     {"name": "tools", "versions": tools.versions},

kafka/Dockerfile

@@ -2,13 +2,13 @@
 # check=error=true
 
 FROM stackable/image/kafka/kcat AS kcat
+FROM stackable/image/kafka/kafka-opa-plugin AS kafka-opa-plugin
 
 FROM stackable/image/java-devel AS kafka-builder
 
 ARG PRODUCT
 ARG RELEASE
 ARG SCALA
-ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
 ARG STACKABLE_USER_UID
 
@@ -40,10 +40,6 @@ cp build/reports/bom.json /stackable/kafka_${SCALA}-${NEW_VERSION}.cdx.json
 rm -rf /stackable/kafka_${SCALA}-${NEW_VERSION}/site-docs/
 (cd .. && rm -rf ${PRODUCT})
 
-# TODO (@NickLarsenNZ): Compile from source: https://github.com/StyraInc/opa-kafka-plugin
-curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
-  -o /stackable/kafka_${SCALA}-${NEW_VERSION}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
-
 # JMX exporter
 curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
   -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar
@@ -60,6 +56,7 @@ ARG RELEASE
 ARG PRODUCT
 ARG SCALA
 ARG KAFKA_KCAT
+ARG KAFKA_KAFKA_OPA_PLUGIN
 ARG STACKABLE_USER_UID
 
 LABEL \
@@ -78,6 +75,8 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stack
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/bin/kcat-${KAFKA_KCAT}
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-opa-plugin /stackable/src/kafka/kafka-opa-plugin/patchable-work/worktree/${KAFKA_KAFKA_OPA_PLUGIN}/build/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-opa-plugin /stackable/kafka-opa-plugin-${KAFKA_KAFKA_OPA_PLUGIN}-src.tar.gz /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses
 
@@ -107,6 +106,7 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/kafka
 chmod g=u /stackable/bin
 chmod g=u /stackable/jmx
 chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}
+chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar
 chmod g=u /stackable/*-src.tar.gz
 EOF
 

kafka/kafka-opa-plugin/Dockerfile

@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
+# check=error=true
+
+FROM stackable/image/java-devel
+
+ARG PRODUCT
+ARG STACKABLE_USER_UID
+
+USER ${STACKABLE_USER_UID}
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/patchable.toml /stackable/src/kafka/kafka-opa-plugin/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/${PRODUCT} /stackable/src/kafka/kafka-opa-plugin/stackable/patches/${PRODUCT}
+
+RUN <<EOF
+cd "$(/stackable/patchable --images-repo-root=src checkout kafka/kafka-opa-plugin ${PRODUCT})"
+
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/kafka-opa-plugin-${PRODUCT}-src.tar.gz .
+
+./gradlew clean shadowJar
+EOF

kafka/kafka-opa-plugin/stackable/patches/1.5.1/patchable.toml

@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/opa-kafka-plugin"
+base = "d2c7851cb66dde7903eb4f0d5fab40f1a3d434a4"

kafka/kafka-opa-plugin/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/StyraInc/opa-kafka-plugin"
+default-mirror = "https://github.com/stackabletech/opa-kafka-plugin"

kafka/kafka-opa-plugin/versions.py

@@ -0,0 +1,6 @@
+versions = [
+    {
+        "product": "1.5.1",
+        "java-devel": "11",
+    },
+]

kafka/versions.py

@@ -5,7 +5,7 @@
         "java-devel": "21",
         "scala": "2.13",
         "kafka/kcat": "1.7.0",
-        "opa_authorizer": "1.5.1",
+        "kafka/kafka-opa-plugin": "1.5.1",
         "jmx_exporter": "1.3.0",
     },
     {
@@ -14,7 +14,7 @@
         "java-devel": "21",
         "scala": "2.13",
         "kafka/kcat": "1.7.0",
-        "opa_authorizer": "1.5.1",
+        "kafka/kafka-opa-plugin": "1.5.1",
         "jmx_exporter": "1.3.0",
     },
     {
@@ -23,7 +23,7 @@
         "java-devel": "21",
         "scala": "2.13",
         "kafka/kcat": "1.7.0",
-        "opa_authorizer": "1.5.1",
+        "kafka/kafka-opa-plugin": "1.5.1",
         "jmx_exporter": "1.3.0",
     },
     {
@@ -32,7 +32,7 @@
         "java-devel": "23",
         "scala": "2.13",
         "kafka/kcat": "1.7.0",
-        "opa_authorizer": "1.5.1",
+        "kafka/kafka-opa-plugin": "1.5.1",
         "jmx_exporter": "1.3.0",
     },
 ]