diff --git a/infra/airflow/Dockerfile b/infra/airflow/Dockerfile index 1529cd9..6a28284 100644 --- a/infra/airflow/Dockerfile +++ b/infra/airflow/Dockerfile @@ -8,8 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ARG SPARK_VERSION=3.5.6 RUN set -eux; \ - curl -fSL --connect-timeout 20 --max-time 900 --retry 5 --retry-connrefused \ - -o /tmp/spark.tgz "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \ + curl -fSL -o /tmp/spark.tgz "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \ tar -xzf /tmp/spark.tgz -C /opt; \ mv "/opt/spark-${SPARK_VERSION}-bin-hadoop3" /opt/spark; \ rm /tmp/spark.tgz diff --git a/infra/debezium/Dockerfile b/infra/debezium/Dockerfile index 786f6aa..f422b23 100644 --- a/infra/debezium/Dockerfile +++ b/infra/debezium/Dockerfile @@ -2,10 +2,12 @@ FROM debezium/connect:3.0.0.Final USER root +# Устанавливаем unzip (больше не нужен для avro-конвертера, оставлен на случай других нужд) RUN microdnf install -y unzip \ - && curl -fsSL https://hub-downloads.confluent.io/api/plugins/confluentinc/kafka-connect-avro-converter/versions/8.0.0/confluentinc-kafka-connect-avro-converter-8.0.0.zip -o /tmp/avro-converter.zip \ - && unzip -q /tmp/avro-converter.zip -d /kafka/connect \ - && rm -f /tmp/avro-converter.zip \ + && mkdir -p /kafka/connect/apicurio-avro-converter \ + && curl -fsSL \ + https://repo1.maven.org/maven2/io/apicurio/apicurio-registry-distro-connect-converter/2.5.11.Final/apicurio-registry-distro-connect-converter-2.5.11.Final.jar \ + -o /kafka/connect/apicurio-avro-converter/apicurio-registry-distro-connect-converter-2.5.11.Final.jar \ && microdnf clean all COPY config /kafka/connectors diff --git a/infra/debezium/config/demo-postgres.json b/infra/debezium/config/demo-postgres.json index 3d21e16..1c59d86 100644 --- a/infra/debezium/config/demo-postgres.json +++ b/infra/debezium/config/demo-postgres.json @@ -20,9 +20,9 @@ "binary.handling.mode": "base64", "schema.history.internal.kafka.bootstrap.servers": "kafka:9092", "schema.history.internal.kafka.topic": "schema-changes.demo", - "key.converter": "io.confluent.connect.avro.AvroConverter", + "key.converter": "io.apicurio.registry.serde.avro.AvroConverter", "key.converter.schema.registry.url": "http://schema-registry:8081", - "value.converter": "io.confluent.connect.avro.AvroConverter", + "value.converter": "io.apicurio.registry.serde.avro.AvroConverter", "value.converter.schema.registry.url": "http://schema-registry:8081", "key.converter.schemas.enable": "true", "value.converter.schemas.enable": "true" diff --git a/infra/jupyterlab/Dockerfile b/infra/jupyterlab/Dockerfile index 3d8bb57..c84a8c6 100644 --- a/infra/jupyterlab/Dockerfile +++ b/infra/jupyterlab/Dockerfile @@ -1,23 +1,44 @@ -FROM jupyter/pyspark-notebook:spark-3.5.0 +# Python 3.12 base (DockerHub images are not updated; use quay.io) +FROM quay.io/jupyter/base-notebook:python-3.12.7 USER root RUN apt-get update && apt-get install -y \ curl \ netcat-openbsd \ postgresql-client \ + openjdk-17-jdk-headless \ && rm -rf /var/lib/apt/lists/* +# Create stable JAVA_HOME symlink: /usr/lib/jvm/default-java +RUN set -eux; \ + JAVA_BIN="$(readlink -f "$(command -v java)")"; \ + JAVA_HOME_DIR="$(dirname "$(dirname "$JAVA_BIN")")"; \ + echo "Detected JAVA_HOME=$JAVA_HOME_DIR"; \ + ln -sfn "$JAVA_HOME_DIR" /usr/lib/jvm/default-java + +ENV JAVA_HOME=/usr/lib/jvm/default-java +ENV PATH="$JAVA_HOME/bin:$PATH" + ARG SPARK_VERSION=3.5.6 + +# Install Spark 3.5.6 RUN set -eux; \ - curl -fSL --connect-timeout 20 --max-time 900 --retry 5 --retry-connrefused \ - -o /tmp/spark.tgz "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \ + curl -fSL -o /tmp/spark.tgz "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz"; \ tar -xzf /tmp/spark.tgz -C /opt; \ mv "/opt/spark-${SPARK_VERSION}-bin-hadoop3" /opt/spark; \ rm /tmp/spark.tgz + ENV SPARK_HOME=/opt/spark ENV PATH="$SPARK_HOME/bin:$PATH" +# Ensure both driver and executors use "python" (Python 3.12 inside this image), +# and ensure PySpark is imported from /opt/spark. +ENV PYSPARK_PYTHON=python +ENV PYSPARK_DRIVER_PYTHON=python +ENV PYTHONPATH="$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip" + USER $NB_UID + RUN pip install --no-cache-dir \ trino[sqlalchemy] \ sqlalchemy-trino \ @@ -71,4 +92,4 @@ USER $NB_UID WORKDIR /home/jovyan/work -CMD ["start-jupyter.sh"] +CMD ["start-jupyter.sh"] \ No newline at end of file diff --git a/infra/schema-registry/Dockerfile b/infra/schema-registry/Dockerfile index 9c54068..72cb99c 100644 --- a/infra/schema-registry/Dockerfile +++ b/infra/schema-registry/Dockerfile @@ -1,15 +1,19 @@ -FROM confluentinc/cp-schema-registry:latest +# Этап 1: скачиваем curl и jq +FROM alpine:latest AS downloader +RUN apk add --no-cache curl +RUN curl -L -o /usr/local/bin/jq https://github.com/jqlang/jq/releases/latest/download/jq-linux64 && chmod +x /usr/local/bin/jq +# Этап 2: финальный образ +FROM confluentinc/cp-schema-registry:latest USER root -RUN yum install -y --allowerasing curl jq && yum clean all +COPY --from=downloader /usr/bin/curl /usr/local/bin/curl +COPY --from=downloader /usr/local/bin/jq /usr/local/bin/jq COPY schemas/ /opt/schemas/ COPY init-schemas.sh /opt/init-schemas.sh - COPY entrypoint.sh /opt/entrypoint.sh RUN chmod +x /opt/init-schemas.sh /opt/entrypoint.sh USER appuser - ENTRYPOINT ["/opt/entrypoint.sh"] diff --git a/launcher.sh b/launcher.sh new file mode 100755 index 0000000..0ce22f6 --- /dev/null +++ b/launcher.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" || exit + +docker compose --profile core up -d +docker compose --profile airflow up -d +docker compose --profile explore up -d \ No newline at end of file diff --git a/notebooks/.gitignore b/notebooks/.gitignore new file mode 100644 index 0000000..a340c10 --- /dev/null +++ b/notebooks/.gitignore @@ -0,0 +1 @@ +work \ No newline at end of file diff --git a/notebooks/examples/quick-connections.ipynb b/notebooks/examples/quick-connections.ipynb index cb4c547..7a7487a 100644 --- a/notebooks/examples/quick-connections.ipynb +++ b/notebooks/examples/quick-connections.ipynb @@ -27,11 +27,11 @@ "execution_count": 1, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:33:42.604546Z", - "iopub.status.busy": "2025-09-05T09:33:42.604212Z", - "iopub.status.idle": "2025-09-05T09:33:42.609231Z", - "shell.execute_reply": "2025-09-05T09:33:42.608777Z", - "shell.execute_reply.started": "2025-09-05T09:33:42.604532Z" + "iopub.execute_input": "2026-06-16T16:36:34.790421Z", + "iopub.status.busy": "2026-06-16T16:36:34.789883Z", + "iopub.status.idle": "2026-06-16T16:36:34.795366Z", + "shell.execute_reply": "2026-06-16T16:36:34.794597Z", + "shell.execute_reply.started": "2026-06-16T16:36:34.790378Z" }, "tags": [] }, @@ -81,11 +81,11 @@ "execution_count": 2, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:33:44.780870Z", - "iopub.status.busy": "2025-09-05T09:33:44.780626Z", - "iopub.status.idle": "2025-09-05T09:33:53.112408Z", - "shell.execute_reply": "2025-09-05T09:33:53.111912Z", - "shell.execute_reply.started": "2025-09-05T09:33:44.780858Z" + "iopub.execute_input": "2026-06-16T16:36:36.313137Z", + "iopub.status.busy": "2026-06-16T16:36:36.312933Z", + "iopub.status.idle": "2026-06-16T16:36:48.586654Z", + "shell.execute_reply": "2026-06-16T16:36:48.585940Z", + "shell.execute_reply.started": "2026-06-16T16:36:36.313116Z" }, "tags": [] }, @@ -94,8 +94,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "Connecting to Spark: spark://spark-master:7077\n", - "Spark ready: 3.5.0\n", + "Connecting to Spark: spark://spark-master:7077\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "26/06/16 16:36:38 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Spark ready: 3.5.6\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "+-------+---+\n", "| name|age|\n", "+-------+---+\n", @@ -106,6 +134,35 @@ "\n", "Spark session active\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "----------------------------------------\n", + "Exception occurred during processing of request from ('127.0.0.1', 37606)\n", + "Traceback (most recent call last):\n", + " File \"/opt/conda/lib/python3.12/socketserver.py\", line 318, in _handle_request_noblock\n", + " self.process_request(request, client_address)\n", + " File \"/opt/conda/lib/python3.12/socketserver.py\", line 349, in process_request\n", + " self.finish_request(request, client_address)\n", + " File \"/opt/conda/lib/python3.12/socketserver.py\", line 362, in finish_request\n", + " self.RequestHandlerClass(request, client_address, self)\n", + " File \"/opt/conda/lib/python3.12/socketserver.py\", line 761, in __init__\n", + " self.handle()\n", + " File \"/opt/spark/python/pyspark/accumulators.py\", line 295, in handle\n", + " poll(accum_updates)\n", + " File \"/opt/spark/python/pyspark/accumulators.py\", line 267, in poll\n", + " if self.rfile in r and func():\n", + " ^^^^^^\n", + " File \"/opt/spark/python/pyspark/accumulators.py\", line 271, in accum_updates\n", + " num_updates = read_int(self.rfile)\n", + " ^^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/spark/python/pyspark/serializers.py\", line 596, in read_int\n", + " raise EOFError\n", + "EOFError\n", + "----------------------------------------\n" + ] } ], "source": [ @@ -113,8 +170,9 @@ "\n", "print(f\"Connecting to Spark: {SPARK_MASTER}\")\n", "\n", + "\n", "spark = SparkSession.builder \\\n", - " .appName(\"DataForge\") \\\n", + " .appName(\"test\") \\\n", " .master(SPARK_MASTER) \\\n", " .config(\"spark.executor.memory\", \"512m\") \\\n", " .config(\"spark.driver.memory\", \"512m\") \\\n", @@ -122,6 +180,8 @@ " .config(\"spark.cores.max\", \"2\") \\\n", " .getOrCreate()\n", "\n", + "\n", + "\n", "print(f\"Spark ready: {spark.version}\")\n", "\n", "# Test query\n", @@ -143,15 +203,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 9, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:34:01.889988Z", - "iopub.status.busy": "2025-09-05T09:34:01.889211Z", - "iopub.status.idle": "2025-09-05T09:34:02.100123Z", - "shell.execute_reply": "2025-09-05T09:34:02.099701Z", - "shell.execute_reply.started": "2025-09-05T09:34:01.889970Z" - } + "iopub.execute_input": "2026-06-16T13:18:00.220456Z", + "iopub.status.busy": "2026-06-16T13:18:00.220013Z", + "iopub.status.idle": "2026-06-16T13:18:00.470336Z", + "shell.execute_reply": "2026-06-16T13:18:00.469955Z", + "shell.execute_reply.started": "2026-06-16T13:18:00.220433Z" + }, + "tags": [] }, "outputs": [ { @@ -160,7 +221,7 @@ "text": [ "PostgreSQL result:\n", " now\n", - "0 2025-09-05 09:34:02.091861+00:00\n", + "0 2026-06-16 13:18:00.463540+00:00\n", "PostgreSQL ready\n" ] } @@ -188,15 +249,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 10, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:34:04.620517Z", - "iopub.status.busy": "2025-09-05T09:34:04.620186Z", - "iopub.status.idle": "2025-09-05T09:34:04.712040Z", - "shell.execute_reply": "2025-09-05T09:34:04.711587Z", - "shell.execute_reply.started": "2025-09-05T09:34:04.620502Z" - } + "iopub.execute_input": "2026-06-16T13:18:05.324531Z", + "iopub.status.busy": "2026-06-16T13:18:05.324008Z", + "iopub.status.idle": "2026-06-16T13:18:05.411075Z", + "shell.execute_reply": "2026-06-16T13:18:05.410641Z", + "shell.execute_reply.started": "2026-06-16T13:18:05.324505Z" + }, + "tags": [] }, "outputs": [ { @@ -205,7 +267,7 @@ "text": [ "ClickHouse result:\n", " message timestamp\n", - "0 Hello ClickHouse 2025-09-05 09:34:04\n", + "0 Hello ClickHouse 2026-06-16 13:18:05\n", "ClickHouse ready\n" ] } @@ -240,22 +302,23 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:34:07.492606Z", - "iopub.status.busy": "2025-09-05T09:34:07.492186Z", - "iopub.status.idle": "2025-09-05T09:34:07.759377Z", - "shell.execute_reply": "2025-09-05T09:34:07.758932Z", - "shell.execute_reply.started": "2025-09-05T09:34:07.492588Z" - } + "iopub.execute_input": "2026-06-16T15:15:55.296442Z", + "iopub.status.busy": "2026-06-16T15:15:55.295898Z", + "iopub.status.idle": "2026-06-16T15:15:55.509534Z", + "shell.execute_reply": "2026-06-16T15:15:55.508947Z", + "shell.execute_reply.started": "2026-06-16T15:15:55.296417Z" + }, + "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Available buckets: ['test-bucket']\n", + "Available buckets: ['checkpoints', 'iceberg', 'sandbox']\n", "MinIO ready\n" ] } @@ -287,15 +350,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 12, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:34:11.284123Z", - "iopub.status.busy": "2025-09-05T09:34:11.283901Z", - "iopub.status.idle": "2025-09-05T09:34:12.124321Z", - "shell.execute_reply": "2025-09-05T09:34:12.123839Z", - "shell.execute_reply.started": "2025-09-05T09:34:11.284111Z" - } + "iopub.execute_input": "2026-06-16T13:18:30.884909Z", + "iopub.status.busy": "2026-06-16T13:18:30.884378Z", + "iopub.status.idle": "2026-06-16T13:18:31.924011Z", + "shell.execute_reply": "2026-06-16T13:18:31.923496Z", + "shell.execute_reply.started": "2026-06-16T13:18:30.884881Z" + }, + "tags": [] }, "outputs": [ { @@ -303,13 +367,14 @@ "output_type": "stream", "text": [ "Available catalogs:\n", - " Catalog\n", - "0 clickhouse\n", - "1 iceberg\n", - "2 kafka\n", - "3 postgres\n", - "4 redis\n", - "5 system\n", + " Catalog\n", + "0 clickhouse\n", + "1 iceberg\n", + "2 kafka\n", + "3 metastore\n", + "4 pg_demo_retail\n", + "5 redis\n", + "6 system\n", "Trino ready\n" ] } @@ -350,15 +415,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 13, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:34:18.049423Z", - "iopub.status.busy": "2025-09-05T09:34:18.049141Z", - "iopub.status.idle": "2025-09-05T09:34:18.143971Z", - "shell.execute_reply": "2025-09-05T09:34:18.143603Z", - "shell.execute_reply.started": "2025-09-05T09:34:18.049405Z" - } + "iopub.execute_input": "2026-06-16T13:18:47.628713Z", + "iopub.status.busy": "2026-06-16T13:18:47.628469Z", + "iopub.status.idle": "2026-06-16T13:18:47.708255Z", + "shell.execute_reply": "2026-06-16T13:18:47.707755Z", + "shell.execute_reply.started": "2026-06-16T13:18:47.628697Z" + }, + "tags": [] }, "outputs": [ { @@ -405,15 +471,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 14, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:34:21.444045Z", - "iopub.status.busy": "2025-09-05T09:34:21.443828Z", - "iopub.status.idle": "2025-09-05T09:34:21.496303Z", - "shell.execute_reply": "2025-09-05T09:34:21.495751Z", - "shell.execute_reply.started": "2025-09-05T09:34:21.444032Z" - } + "iopub.execute_input": "2026-06-16T13:18:53.525097Z", + "iopub.status.busy": "2026-06-16T13:18:53.524866Z", + "iopub.status.idle": "2026-06-16T13:18:53.671645Z", + "shell.execute_reply": "2026-06-16T13:18:53.669781Z", + "shell.execute_reply.started": "2026-06-16T13:18:53.525083Z" + }, + "tags": [] }, "outputs": [ { @@ -467,15 +534,16 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 15, "metadata": { "execution": { - "iopub.execute_input": "2025-09-05T09:34:24.118164Z", - "iopub.status.busy": "2025-09-05T09:34:24.117843Z", - "iopub.status.idle": "2025-09-05T09:34:24.289574Z", - "shell.execute_reply": "2025-09-05T09:34:24.289053Z", - "shell.execute_reply.started": "2025-09-05T09:34:24.118146Z" - } + "iopub.execute_input": "2026-06-16T13:19:04.257584Z", + "iopub.status.busy": "2026-06-16T13:19:04.257255Z", + "iopub.status.idle": "2026-06-16T13:19:04.506712Z", + "shell.execute_reply": "2026-06-16T13:19:04.505927Z", + "shell.execute_reply.started": "2026-06-16T13:19:04.257570Z" + }, + "tags": [] }, "outputs": [ { @@ -506,7 +574,7 @@ " 'Spark': '✅'}" ] }, - "execution_count": 9, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -650,7 +718,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/notebooks/lessons/iceberg/spark-iceberg-dml.ipynb b/notebooks/lessons/iceberg/spark-iceberg-dml.ipynb index d7fbc67..6657937 100644 --- a/notebooks/lessons/iceberg/spark-iceberg-dml.ipynb +++ b/notebooks/lessons/iceberg/spark-iceberg-dml.ipynb @@ -38,10 +38,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "c1afbdba", - "metadata": {}, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-16T16:40:52.333668Z", + "iopub.status.busy": "2026-06-16T16:40:52.333221Z", + "iopub.status.idle": "2026-06-16T16:40:52.337810Z", + "shell.execute_reply": "2026-06-16T16:40:52.337300Z", + "shell.execute_reply.started": "2026-06-16T16:40:52.333638Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🔧 Environment configured\n", + "MinIO: http://minio:9000\n", + "Hive Metastore: thrift://hive-metastore:9083\n", + "Trino: http://trino:8080\n", + "Spark Master: spark://spark-master:7077\n" + ] + } + ], "source": [ "import os\n", "\n", @@ -84,10 +105,90 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "b8f41189", - "metadata": {}, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-16T16:40:57.695593Z", + "iopub.status.busy": "2026-06-16T16:40:57.695125Z", + "iopub.status.idle": "2026-06-16T16:42:14.472971Z", + "shell.execute_reply": "2026-06-16T16:42:14.472241Z", + "shell.execute_reply.started": "2026-06-16T16:40:57.695562Z" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🚀 Creating Spark session with Iceberg REST Catalog support...\n", + ":: loading settings :: url = jar:file:/opt/spark/jars/ivy-2.5.1.jar!/org/apache/ivy/core/settings/ivysettings.xml\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Ivy Default Cache set to: /home/jovyan/.ivy2/cache\n", + "The jars for the packages stored in: /home/jovyan/.ivy2/jars\n", + "org.apache.iceberg#iceberg-spark-runtime-3.5_2.12 added as a dependency\n", + "org.apache.hadoop#hadoop-aws added as a dependency\n", + "software.amazon.awssdk#bundle added as a dependency\n", + ":: resolving dependencies :: org.apache.spark#spark-submit-parent-ece2fe2f-0608-4df4-929f-65e2c087501f;1.0\n", + "\tconfs: [default]\n", + "\tfound org.apache.iceberg#iceberg-spark-runtime-3.5_2.12;1.9.2 in central\n", + "\tfound org.apache.hadoop#hadoop-aws;3.3.4 in central\n", + "\tfound com.amazonaws#aws-java-sdk-bundle;1.12.262 in central\n", + "\tfound org.wildfly.openssl#wildfly-openssl;1.0.7.Final in central\n", + "\tfound software.amazon.awssdk#bundle;2.20.158 in central\n", + "\tfound software.amazon.eventstream#eventstream;1.0.1 in central\n", + "downloading https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.2/iceberg-spark-runtime-3.5_2.12-1.9.2.jar ...\n", + "\t[SUCCESSFUL ] org.apache.iceberg#iceberg-spark-runtime-3.5_2.12;1.9.2!iceberg-spark-runtime-3.5_2.12.jar (2499ms)\n", + "downloading https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar ...\n", + "\t[SUCCESSFUL ] org.apache.hadoop#hadoop-aws;3.3.4!hadoop-aws.jar (224ms)\n", + "downloading https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.20.158/bundle-2.20.158.jar ...\n", + "\t[SUCCESSFUL ] software.amazon.awssdk#bundle;2.20.158!bundle.jar (40513ms)\n", + "downloading https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar ...\n", + "\t[SUCCESSFUL ] com.amazonaws#aws-java-sdk-bundle;1.12.262!aws-java-sdk-bundle.jar (22675ms)\n", + "downloading https://repo1.maven.org/maven2/org/wildfly/openssl/wildfly-openssl/1.0.7.Final/wildfly-openssl-1.0.7.Final.jar ...\n", + "\t[SUCCESSFUL ] org.wildfly.openssl#wildfly-openssl;1.0.7.Final!wildfly-openssl.jar (173ms)\n", + "downloading https://repo1.maven.org/maven2/software/amazon/eventstream/eventstream/1.0.1/eventstream-1.0.1.jar ...\n", + "\t[SUCCESSFUL ] software.amazon.eventstream#eventstream;1.0.1!eventstream.jar (145ms)\n", + ":: resolution report :: resolve 5181ms :: artifacts dl 66236ms\n", + "\t:: modules in use:\n", + "\tcom.amazonaws#aws-java-sdk-bundle;1.12.262 from central in [default]\n", + "\torg.apache.hadoop#hadoop-aws;3.3.4 from central in [default]\n", + "\torg.apache.iceberg#iceberg-spark-runtime-3.5_2.12;1.9.2 from central in [default]\n", + "\torg.wildfly.openssl#wildfly-openssl;1.0.7.Final from central in [default]\n", + "\tsoftware.amazon.awssdk#bundle;2.20.158 from central in [default]\n", + "\tsoftware.amazon.eventstream#eventstream;1.0.1 from central in [default]\n", + "\t---------------------------------------------------------------------\n", + "\t| | modules || artifacts |\n", + "\t| conf | number| search|dwnlded|evicted|| number|dwnlded|\n", + "\t---------------------------------------------------------------------\n", + "\t| default | 6 | 6 | 6 | 0 || 6 | 6 |\n", + "\t---------------------------------------------------------------------\n", + ":: retrieving :: org.apache.spark#spark-submit-parent-ece2fe2f-0608-4df4-929f-65e2c087501f\n", + "\tconfs: [default]\n", + "\t6 artifacts copied, 0 already retrieved (809160kB/827ms)\n", + "26/06/16 16:42:11 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "26/06/16 16:42:12 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Spark session created successfully with Iceberg REST Catalog\n", + "📊 Spark UI available at: http://localhost:8088\n", + "🔧 Spark version: 3.5.6\n", + "🌐 Iceberg REST Catalog: http://hive-metastore:9001/iceberg\n" + ] + } + ], "source": [ "from pyspark.sql import SparkSession\n", "print(\"🚀 Creating Spark session with Iceberg REST Catalog support...\")\n", @@ -155,10 +256,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "678e7f4c", - "metadata": {}, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-16T16:42:14.474389Z", + "iopub.status.busy": "2026-06-16T16:42:14.474083Z", + "iopub.status.idle": "2026-06-16T16:42:14.821732Z", + "shell.execute_reply": "2026-06-16T16:42:14.821070Z", + "shell.execute_reply.started": "2026-06-16T16:42:14.474363Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Bucket 'iceberg' already exists\n", + "\n", + "📂 Available buckets: ['checkpoints', 'iceberg', 'sandbox']\n" + ] + } + ], "source": [ "import boto3\n", "from botocore.exceptions import ClientError\n", @@ -195,6 +314,45 @@ "print(f\"\\n📂 Available buckets: {[b['Name'] for b in buckets['Buckets']]}\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "30997c5c-d9e4-487c-9e0c-a86782e39151", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-16T16:42:32.210952Z", + "iopub.status.busy": "2026-06-16T16:42:32.210694Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "26/06/16 16:42:48 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n", + "26/06/16 16:43:03 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n", + "26/06/16 16:43:18 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n", + "26/06/16 16:43:33 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n", + "26/06/16 16:43:48 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n" + ] + } + ], + "source": [ + "test_data = [(\"Alice\", 25), (\"Bob\", 30), (\"Charlie\", 35)]\n", + "df = spark.createDataFrame(test_data, [\"name\", \"age\"])\n", + "df.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a249621-83ca-4efa-86c4-cd2fa7558a84", + "metadata": {}, + "outputs": [], + "source": [ + "df.write.parquet" + ] + }, { "cell_type": "markdown", "id": "ecb9fda5", @@ -207,10 +365,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "86ade88e", - "metadata": {}, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-16T16:42:14.822655Z", + "iopub.status.busy": "2026-06-16T16:42:14.822422Z", + "iopub.status.idle": "2026-06-16T16:42:18.470452Z", + "shell.execute_reply": "2026-06-16T16:42:18.469924Z", + "shell.execute_reply.started": "2026-06-16T16:42:14.822629Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🔍 Testing Iceberg catalog integration...\n", + "📋 Available Catalogs:\n", + "+-------------+\n", + "| catalog|\n", + "+-------------+\n", + "| iceberg|\n", + "|spark_catalog|\n", + "+-------------+\n", + "\n", + "✅ Testing iceberg catalog access...\n", + "\n", + "📋 Databases in iceberg catalog:\n", + "+---------+\n", + "|namespace|\n", + "+---------+\n", + "| default|\n", + "+---------+\n", + "\n", + "✅ Iceberg catalog integration verified\n" + ] + } + ], "source": [ "print(\"🔍 Testing Iceberg catalog integration...\")\n", "\n", @@ -858,6 +1050,18 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" } }, "nbformat": 4,