Merge branch 'main' into feat/context-memory-entity

Author: pmbrull

CLAUDE.md

@@ -491,6 +491,14 @@ These checks run automatically in CI. Code that violates them **will not merge**
 - This keeps the codebase modular and prevents generic utilities from becoming cluttered with connector-specific edge cases
 - **Use `model_str()` for Pydantic RootModel to string conversion** — OpenMetadata schema types like `ColumnName`, `EntityName`, `FullyQualifiedEntityName`, and `UUID` are Pydantic `RootModel[str]` subclasses where `str()` returns `"root='value'"` instead of the raw value. Always use `model_str()` from `metadata.ingestion.ometa.utils` instead of manual `hasattr(x, "root")` / `str(x.root)` checks.
 
+### Caching
+- **All caches MUST be bounded.** Never use a bare `dict` / `HashMap` / `Map` as a cache without an explicit size cap — they grow with the input and cause OOMs on large catalogs/ingestions. The only exception is when the user explicitly asks for an unbounded cache for a specific case.
+- Pick a sane default (typically 100–1000 entries depending on entity size); if you're unsure, ask the user.
+- **Python**: use `collections.OrderedDict` with `popitem(last=False)` eviction after insert, `@functools.lru_cache(maxsize=N)`, or `cachetools.LRUCache`. Cache both hits and misses (negative caching) — repeated unresolvable lookups are a common hot path.
+- **Java**: use Caffeine (`Caffeine.newBuilder().maximumSize(N).build()`) or Guava `CacheBuilder.newBuilder().maximumSize(N).build()`. Never a bare `HashMap`.
+- **TypeScript**: use `lru-cache` — never a bare `Map` or plain object.
+- **Before adding a cache, check whether the underlying call is already cached at a lower layer.** Example: `OpenMetadata._search_es_entity` is `@lru_cache(maxsize=512)`, so wrapping `get_entity_from_es` / `es_search_container_by_path` calls in a local dict cache is redundant — drop the local cache and rely on the existing LRU.
+
 ### Testing Philosophy
 - **Test real behavior, not mock wiring** - if a test requires mocking 3+ classes just to verify a method call, it's testing the wrong thing
 - **Prefer integration tests** over heavily-mocked unit tests. This project has full integration test infrastructure (OpenMetadataApplicationTest, Docker containers, real OpenSearch). Use it.

docker/development/docker-compose-fuseki.yml

@@ -34,19 +34,41 @@ services:
         condition: service_healthy
 
   fuseki:
-    image: stain/jena-fuseki:5.0.0
+    # Build from the in-repo Dockerfile (Fuseki 5.6.0) instead of the
+    # unmaintained `stain/jena-fuseki` Docker Hub image, which capped at 5.1.0
+    # and never picked up the 2025 admin-side Fuseki CVE fixes (CVE-2025-49656,
+    # CVE-2025-50151 — both fixed in Jena 5.5.0). The `image:` tag below names
+    # the locally-built image so subsequent `docker compose up` runs reuse the
+    # cached build instead of rebuilding from scratch each time.
+    build:
+      context: ../rdf-store
+      dockerfile: Dockerfile
+    image: openmetadata-fuseki:5.6.0
     container_name: openmetadata-fuseki
     hostname: fuseki
     ports:
       - "3030:3030"
     networks:
       - local_app_net
     environment:
-      - ADMIN_PASSWORD=admin
+      # Default for local dev only — production deployments MUST override
+      # via the FUSEKI_ADMIN_PASSWORD env var (and FUSEKI_OPENMETADATA_PASSWORD)
+      # before bringing this stack up. The entrypoint envsubsts these into
+      # shiro.ini at container start so the override actually takes effect.
+      - FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
+      - FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
       - JVM_ARGS=${FUSEKI_JVM_ARGS:--Xmx1500m -Xms256m}
-      - FUSEKI_BASE=/fuseki
     volumes:
-      - fuseki-data:/fuseki
+      # New volume name (was `fuseki-data` mounted at `/fuseki`). The in-repo
+      # Dockerfile stores TDB2 at `/fuseki-data` and the data layout differs
+      # from the old stain/jena-fuseki image — re-using the previous volume
+      # name would mount stale state at a path Fuseki no longer reads from,
+      # silently looking like an empty database. Using a fresh volume name
+      # forces operators to consciously migrate (or accept a re-index). The
+      # orphaned `fuseki-data` volume can be removed manually with
+      # `docker volume rm fuseki-data` after confirming the new stack is
+      # healthy.
+      - fuseki-tdb2-data:/fuseki-data
     deploy:
       resources:
         limits:
@@ -60,8 +82,6 @@ services:
       timeout: 10s
       retries: 20
       start_period: 60s
-    # Create the database directory before starting Fuseki
-    entrypoint: /bin/sh -c "mkdir -p /fuseki/databases/openmetadata && exec /docker-entrypoint.sh /jena-fuseki/fuseki-server --update --loc=/fuseki/databases/openmetadata /openmetadata"
 
 networks:
   local_app_net:
@@ -72,5 +92,5 @@ networks:
         - subnet: "172.16.239.0/24"
 
 volumes:
-  fuseki-data:
+  fuseki-tdb2-data:
     driver: local

docker/development/docker-compose-postgres-fuseki.yml

@@ -15,7 +15,9 @@ volumes:
   ingestion-volume-dags:
   ingestion-volume-tmp:
   es-data:
-  fuseki-data:
+  # See docker-compose-fuseki.yml — renamed from `fuseki-data` to avoid
+  # silently mounting stale state under the new Fuseki layout.
+  fuseki-tdb2-data:
 services:
   postgresql:
     build:
@@ -565,17 +567,29 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:z  # Need 600 permissions to run DockerOperator
 
   fuseki:
-    image: stain/jena-fuseki:5.0.0
+    # See docker-compose-fuseki.yml for the rationale behind building from the
+    # in-repo Dockerfile instead of using `stain/jena-fuseki:*` (unmaintained,
+    # capped at 5.1.0, missing 2025 admin-side CVE fixes).
+    build:
+      context: ../rdf-store
+      dockerfile: Dockerfile
+    image: openmetadata-fuseki:5.6.0
     container_name: openmetadata-fuseki
     hostname: fuseki
     ports:
       - "3030:3030"
     environment:
-      - ADMIN_PASSWORD=admin
+      # Local-dev default — production deployments MUST override via
+      # FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars.
+      - FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
+      - FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
       - JVM_ARGS=-Xmx4g -Xms2g
-      - FUSEKI_BASE=/fuseki
     volumes:
-      - fuseki-data:/fuseki
+      # See docker-compose-fuseki.yml for why the volume was renamed from
+      # `fuseki-data` to `fuseki-tdb2-data` (the data layout differs from the
+      # previous stain/jena-fuseki image and reusing the old name silently
+      # mounts stale state at a path Fuseki no longer reads).
+      - fuseki-tdb2-data:/fuseki-data
     deploy:
       resources:
         limits:
@@ -584,8 +598,6 @@ services:
           memory: 2G
     networks:
       - local_app_net
-    # Create the database directory before starting Fuseki
-    entrypoint: /bin/sh -c "mkdir -p /fuseki/databases/openmetadata && exec /docker-entrypoint.sh /jena-fuseki/fuseki-server --update --loc=/fuseki/databases/openmetadata /openmetadata"
 
 
 

docker/docker-compose-quickstart/Dockerfile.fuseki-alpine

@@ -8,7 +8,7 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 # Set Fuseki version and paths
-ENV FUSEKI_VERSION=4.10.0
+ENV FUSEKI_VERSION=5.6.0
 ENV FUSEKI_HOME=/fuseki
 ENV FUSEKI_BASE=/fuseki
 

docker/docker-compose-quickstart/Dockerfile.fuseki-arm64

@@ -14,7 +14,7 @@ RUN apt-get update || true && \
     rm -rf /var/lib/apt/lists/*
 
 # Set Fuseki version
-ENV FUSEKI_VERSION=5.0.0
+ENV FUSEKI_VERSION=5.6.0
 ENV FUSEKI_HOME=/fuseki
 
 # Download and install Fuseki

docker/docker-compose-quickstart/Dockerfile.fuseki-multiarch

@@ -1,5 +1,6 @@
 # Multi-architecture Fuseki build
-FROM --platform=$TARGETPLATFORM openjdk:17-jdk-slim
+# eclipse-temurin replaces the deprecated openjdk Docker Hub images.
+FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre-jammy
 
 # Install required packages
 RUN apt-get update && \
@@ -9,7 +10,7 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 # Set Fuseki version and paths
-ENV FUSEKI_VERSION=4.10.0
+ENV FUSEKI_VERSION=5.6.0
 ENV FUSEKI_HOME=/fuseki
 ENV FUSEKI_BASE=/fuseki
 

docker/docker-compose-quickstart/Dockerfile.fuseki-simple

@@ -3,15 +3,15 @@ FROM --platform=$BUILDPLATFORM alpine:latest AS downloader
 
 RUN apk add --no-cache wget tar
 
-ENV FUSEKI_VERSION=5.0.0
+ENV FUSEKI_VERSION=5.6.0
 WORKDIR /tmp
 
 RUN wget -q https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
     tar -xzf apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
     mv apache-jena-fuseki-${FUSEKI_VERSION} /fuseki-dist
 
-# Use OpenJDK base image for runtime
-FROM openjdk:17-slim
+# Runtime: eclipse-temurin replaces the deprecated openjdk Docker Hub images.
+FROM eclipse-temurin:17-jre-jammy
 
 ENV FUSEKI_HOME=/fuseki
 ENV FUSEKI_BASE=/fuseki

docker/docker-compose-quickstart/Dockerfile.fuseki-working

@@ -1,8 +1,9 @@
 # Simple Fuseki build that works on ARM64
-FROM openjdk:17-slim
+# eclipse-temurin replaces the deprecated openjdk Docker Hub images.
+FROM eclipse-temurin:17-jre-jammy
 
 # Set Fuseki version and paths
-ENV FUSEKI_VERSION=4.10.0
+ENV FUSEKI_VERSION=5.6.0
 ENV FUSEKI_HOME=/fuseki
 ENV FUSEKI_BASE=/fuseki
 

docker/docker-compose-quickstart/docker-compose-fuseki-rosetta.yml

@@ -3,24 +3,32 @@ services:
   fuseki:
     # Force AMD64 platform with Rosetta 2 emulation
     platform: linux/amd64
-    image: stain/jena-fuseki:5.0.0
+    # Build from the in-repo Dockerfile (Fuseki 5.6.0). See
+    # docker-compose-fuseki.yml for the full rationale.
+    build:
+      context: ../rdf-store
+      dockerfile: Dockerfile
+    image: openmetadata-fuseki:5.6.0
     container_name: fuseki-standalone
     hostname: fuseki
     ports:
       - "3030:3030"
     environment:
-      # Admin credentials
-      - ADMIN_PASSWORD=admin
-      # JVM memory settings
+      # Local-dev default — production deployments MUST override via
+      # FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars.
+      - FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
+      - FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
       - JVM_ARGS=-Xmx8g -Xms4g
-      - FUSEKI_BASE=/fuseki
     volumes:
-      # Mount directory for persistent storage
-      - ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki:/fuseki
+      # Host bind path renamed from `.../fuseki` (used by the old stain image
+      # layout) to `.../fuseki-tdb2-data` so an existing host directory with
+      # the previous layout isn't silently mounted at /fuseki-data — Fuseki
+      # would see an empty TDB2 store and the old data would appear lost.
+      # Operators upgrading can either delete the new dir to start fresh or
+      # migrate old data manually.
+      - ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki-tdb2-data:/fuseki-data
     networks:
       - fuseki-net
-    # Create openmetadata dataset on startup
-    entrypoint: /bin/sh -c "mkdir -p /fuseki/databases/openmetadata && exec /docker-entrypoint.sh /jena-fuseki/fuseki-server --update --loc=/fuseki/databases/openmetadata /openmetadata"
     healthcheck:
       test: ["CMD", "wget", "-q", "--spider", "http://localhost:3030/$/ping"]
       interval: 15s

docker/docker-compose-quickstart/docker-compose-fuseki-standalone.yml

@@ -1,25 +1,29 @@
 # Standalone Apache Jena Fuseki for RDF/Knowledge Graph storage
 services:
   fuseki:
-    image: stain/jena-fuseki:5.0.0
+    # Build from the in-repo Dockerfile (Fuseki 5.6.0). See
+    # ../development/docker-compose-fuseki.yml for the full rationale.
+    build:
+      context: ../rdf-store
+      dockerfile: Dockerfile
+    image: openmetadata-fuseki:5.6.0
     container_name: fuseki-standalone
     hostname: fuseki
     ports:
       - "3030:3030"
     environment:
-      # Admin credentials
-      - ADMIN_PASSWORD=admin
-      # JVM memory settings - adjust based on your system
+      # Local-dev default — production deployments MUST override via
+      # FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars.
+      - FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
+      - FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
       - JVM_ARGS=-Xmx4g -Xms2g
-      # Fuseki configuration
-      - FUSEKI_BASE=/fuseki
     volumes:
-      # Mount directory for persistent storage (configurable via .env)
-      - ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki:/fuseki
+      # See docker-compose-fuseki-rosetta.yml — host bind path renamed so
+      # existing directories with the old stain layout aren't silently
+      # mounted at the new /fuseki-data path.
+      - ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki-tdb2-data:/fuseki-data
     networks:
       - fuseki-net
-    # Create openmetadata dataset on startup
-    entrypoint: /bin/sh -c "mkdir -p /fuseki/databases/openmetadata && exec /docker-entrypoint.sh /jena-fuseki/fuseki-server --update --loc=/fuseki/databases/openmetadata /openmetadata"
     healthcheck:
       test: ["CMD", "wget", "-q", "--spider", "http://localhost:3030/$/ping"]
       interval: 15s