Add launcher

infra/airflow/Dockerfile

@@ -8,8 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 ARG SPARK_VERSION=3.5.6
 RUN set -eux; \
-    curl -fSL --connect-timeout 20 --max-time 900 --retry 5 --retry-connrefused \
-    -o /tmp/spark.tgz "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \
+    curl -fSL -o /tmp/spark.tgz "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \
     tar -xzf /tmp/spark.tgz -C /opt; \
     mv "/opt/spark-${SPARK_VERSION}-bin-hadoop3" /opt/spark; \
     rm /tmp/spark.tgz

infra/debezium/Dockerfile

@@ -2,10 +2,12 @@ FROM debezium/connect:3.0.0.Final
 
 USER root
 
+# Устанавливаем unzip (больше не нужен для avro-конвертера, оставлен на случай других нужд)
 RUN microdnf install -y unzip \
- && curl -fsSL https://hub-downloads.confluent.io/api/plugins/confluentinc/kafka-connect-avro-converter/versions/8.0.0/confluentinc-kafka-connect-avro-converter-8.0.0.zip -o /tmp/avro-converter.zip \
- && unzip -q /tmp/avro-converter.zip -d /kafka/connect \
- && rm -f /tmp/avro-converter.zip \
+ && mkdir -p /kafka/connect/apicurio-avro-converter \
+ && curl -fsSL \
+    https://repo1.maven.org/maven2/io/apicurio/apicurio-registry-distro-connect-converter/2.5.11.Final/apicurio-registry-distro-connect-converter-2.5.11.Final.jar \
+    -o /kafka/connect/apicurio-avro-converter/apicurio-registry-distro-connect-converter-2.5.11.Final.jar \
  && microdnf clean all
 
 COPY config /kafka/connectors

infra/debezium/config/demo-postgres.json

@@ -20,9 +20,9 @@
   "binary.handling.mode": "base64",
   "schema.history.internal.kafka.bootstrap.servers": "kafka:9092",
   "schema.history.internal.kafka.topic": "schema-changes.demo",
-  "key.converter": "io.confluent.connect.avro.AvroConverter",
+  "key.converter": "io.apicurio.registry.serde.avro.AvroConverter",
   "key.converter.schema.registry.url": "http://schema-registry:8081",
-  "value.converter": "io.confluent.connect.avro.AvroConverter",
+  "value.converter": "io.apicurio.registry.serde.avro.AvroConverter",
   "value.converter.schema.registry.url": "http://schema-registry:8081",
   "key.converter.schemas.enable": "true",
   "value.converter.schemas.enable": "true"

infra/jupyterlab/Dockerfile

@@ -1,23 +1,44 @@
-FROM jupyter/pyspark-notebook:spark-3.5.0
+# Python 3.12 base (DockerHub images are not updated; use quay.io)
+FROM quay.io/jupyter/base-notebook:python-3.12.7
 
 USER root
 RUN apt-get update && apt-get install -y \
     curl \
     netcat-openbsd \
     postgresql-client \
+    openjdk-17-jdk-headless \
     && rm -rf /var/lib/apt/lists/*
 
+# Create stable JAVA_HOME symlink: /usr/lib/jvm/default-java
+RUN set -eux; \
+    JAVA_BIN="$(readlink -f "$(command -v java)")"; \
+    JAVA_HOME_DIR="$(dirname "$(dirname "$JAVA_BIN")")"; \
+    echo "Detected JAVA_HOME=$JAVA_HOME_DIR"; \
+    ln -sfn "$JAVA_HOME_DIR" /usr/lib/jvm/default-java
+
+ENV JAVA_HOME=/usr/lib/jvm/default-java
+ENV PATH="$JAVA_HOME/bin:$PATH"
+
 ARG SPARK_VERSION=3.5.6
+
+# Install Spark 3.5.6
 RUN set -eux; \
-    curl -fSL --connect-timeout 20 --max-time 900 --retry 5 --retry-connrefused \
-    -o /tmp/spark.tgz "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \
+    curl -fSL -o /tmp/spark.tgz "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \
     tar -xzf /tmp/spark.tgz -C /opt; \
     mv "/opt/spark-${SPARK_VERSION}-bin-hadoop3" /opt/spark; \
     rm /tmp/spark.tgz
+
 ENV SPARK_HOME=/opt/spark
 ENV PATH="$SPARK_HOME/bin:$PATH"
 
+# Ensure both driver and executors use "python" (Python 3.12 inside this image),
+# and ensure PySpark is imported from /opt/spark.
+ENV PYSPARK_PYTHON=python
+ENV PYSPARK_DRIVER_PYTHON=python
+ENV PYTHONPATH="$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip"
+
 USER $NB_UID
+
 RUN pip install --no-cache-dir \
     trino[sqlalchemy] \
     sqlalchemy-trino \
@@ -71,4 +92,4 @@ USER $NB_UID
 
 WORKDIR /home/jovyan/work
 
-CMD ["start-jupyter.sh"]
+CMD ["start-jupyter.sh"]

infra/schema-registry/Dockerfile

@@ -1,15 +1,19 @@
-FROM confluentinc/cp-schema-registry:latest
+# Этап 1: скачиваем curl и jq
+FROM alpine:latest AS downloader
+RUN apk add --no-cache curl
+RUN curl -L -o /usr/local/bin/jq https://github.com/jqlang/jq/releases/latest/download/jq-linux64 && chmod +x /usr/local/bin/jq
 
+# Этап 2: финальный образ
+FROM confluentinc/cp-schema-registry:latest
 USER root
-RUN yum install -y --allowerasing curl jq && yum clean all
+COPY --from=downloader /usr/bin/curl /usr/local/bin/curl
+COPY --from=downloader /usr/local/bin/jq /usr/local/bin/jq
 
 COPY schemas/ /opt/schemas/
 COPY init-schemas.sh /opt/init-schemas.sh
-
 COPY entrypoint.sh /opt/entrypoint.sh
 
 RUN chmod +x /opt/init-schemas.sh /opt/entrypoint.sh
 
 USER appuser
-
 ENTRYPOINT ["/opt/entrypoint.sh"]

launcher.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR" || exit
+
+docker compose --profile core up -d
+docker compose --profile airflow up -d
+docker compose --profile explore up -d

notebooks/.gitignore

@@ -0,0 +1 @@
+work