docs: add notebook integration_duckdb_example and refactor names of prev notebooks

Author: federicsp

notebooks/pyiceberg_example.ipynb notebooks/basic_pyiceberg_example.ipynbnotebooks/pyiceberg_example.ipynb renamed to notebooks/basic_pyiceberg_example.ipynb

@@ -0,0 +1,194 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Libraries\n",
+    "import os\n",
+    "import tempfile\n",
+    "\n",
+    "import duckdb\n",
+    "import pyarrow as pa\n",
+    "import pyarrow.compute as pc\n",
+    "\n",
+    "from pyiceberg.catalog.sql import SqlCatalog\n",
+    "\n",
+    "# Create temporary folders for the warehouse and catalog\n",
+    "warehouse_path = tempfile.mkdtemp(prefix=\"iceberg_warehouse_\")\n",
+    "catalog_path = os.path.join(warehouse_path, \"catalog.db\")\n",
+    "print(\"Temporary warehouse:\", warehouse_path)\n",
+    "print(\"Temporary catalog:\", catalog_path)\n",
+    "\n",
+    "# Create a temporary SQL catalog using SQLite\n",
+    "catalog = SqlCatalog(name=\"tmp_sql_catalog\", uri=f\"sqlite:///{catalog_path}\", warehouse=f\"file://{warehouse_path}\", properties={})\n",
+    "# Create the default namespace\n",
+    "catalog.create_namespace(\"default\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2",
+   "metadata": {},
+   "source": [
+    "## First snapshot\n",
+    "We create the initial dataset and save it to an Iceberg table to create the first snapshot."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initial dataset\n",
+    "data1 = {\n",
+    "    \"vendor_id\": [1, 2, 1, 2, 1],\n",
+    "    \"trip_distance\": [1.5, 2.3, 0.8, 5.2, 3.1],\n",
+    "    \"fare_amount\": [10.0, 15.5, 6.0, 22.0, 18.0],\n",
+    "    \"tip_amount\": [2.0, 3.0, 1.0, 4.5, 3.5],\n",
+    "    \"passenger_count\": [1, 2, 1, 3, 2],\n",
+    "}\n",
+    "df1 = pa.table(data1)\n",
+    "\n",
+    "# Create the Iceberg table and append initial data (first snapshot)\n",
+    "table = catalog.create_table(\"default.sample_trips\", schema=df1.schema)\n",
+    "table.append(df1)\n",
+    "print(\"First snapshot rows:\", len(table.scan().to_arrow()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3",
+   "metadata": {},
+   "source": [
+    "## Second snapshot\n",
+    "We add new data to the same table, creating a second snapshot."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# New dataset for the second snapshot\n",
+    "data2 = {\n",
+    "    \"vendor_id\": [3, 1],\n",
+    "    \"trip_distance\": [2.0, 1.0],\n",
+    "    \"fare_amount\": [12.0, 8.0],\n",
+    "    \"tip_amount\": [1.5, 2.0],\n",
+    "    \"passenger_count\": [1, 1],\n",
+    "}\n",
+    "df2 = pa.table(data2)\n",
+    "\n",
+    "# Append new data to the table (second snapshot)\n",
+    "table.append(df2)\n",
+    "print(\"Second snapshot total rows:\", len(table.scan().to_arrow()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4",
+   "metadata": {},
+   "source": [
+    "## Compare snapshots using DuckDB\n",
+    "We load both snapshots into DuckDB as temporary tables to find added and removed rows."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get snapshot IDs\n",
+    "snapshots = table.snapshots()\n",
+    "first_id = snapshots[0].snapshot_id\n",
+    "second_id = snapshots[-1].snapshot_id\n",
+    "print(\"Snapshot IDs:\", first_id, second_id)\n",
+    "\n",
+    "# Load snapshots into PyArrow tables\n",
+    "arrow_first = table.scan(snapshot_id=first_id).to_arrow()\n",
+    "arrow_second = table.scan(snapshot_id=second_id).to_arrow()\n",
+    "\n",
+    "# Connect to DuckDB and register tables\n",
+    "con = duckdb.connect()\n",
+    "con.register(\"first_snap\", arrow_first)\n",
+    "con.register(\"second_snap\", arrow_second)\n",
+    "\n",
+    "# Find added rows in the second snapshot\n",
+    "added_rows = con.execute(\"\"\"\n",
+    "SELECT * FROM second_snap\n",
+    "EXCEPT\n",
+    "SELECT * FROM first_snap\n",
+    "\"\"\").fetchall()\n",
+    "\n",
+    "# Find removed rows compared to the first snapshot\n",
+    "removed_rows = con.execute(\"\"\"\n",
+    "SELECT * FROM first_snap\n",
+    "EXCEPT\n",
+    "SELECT * FROM second_snap\n",
+    "\"\"\").fetchall()\n",
+    "\n",
+    "print(\"=== ADDED ROWS ===\")\n",
+    "for r in added_rows:\n",
+    "    print(r)\n",
+    "\n",
+    "print(\"\\n=== REMOVED ROWS ===\")\n",
+    "for r in removed_rows:\n",
+    "    print(r)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5",
+   "metadata": {},
+   "source": [
+    "## Filters and aggregations on the second snapshot\n",
+    "We add a computed column and perform filtering and aggregation using DuckDB."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Add computed column 'tip_per_mile'\n",
+    "arrow_second = arrow_second.append_column(\"tip_per_mile\", pc.divide(arrow_second[\"tip_amount\"], arrow_second[\"trip_distance\"]))\n",
+    "con.register(\"second_snap\", arrow_second)\n",
+    "\n",
+    "# Filter rows with tip_per_mile > 1.0\n",
+    "filtered_df = con.execute(\"SELECT * FROM second_snap WHERE tip_per_mile > 1.0\").fetchdf()\n",
+    "print(\"Filtered rows (tip_per_mile > 1.0):\")\n",
+    "print(filtered_df)\n",
+    "\n",
+    "# Aggregate total fare by vendor\n",
+    "agg_df = con.execute(\"SELECT vendor_id, SUM(fare_amount) AS total_fare FROM second_snap GROUP BY vendor_id\").fetchdf()\n",
+    "print(\"Total fare per vendor:\")\n",
+    "print(agg_df)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}