Perf/redis cache metrics and indexes

conf/openmetadata.yaml

@@ -765,6 +765,8 @@ cache:
     # Connection pool settings
     poolSize: ${CACHE_REDIS_POOL_SIZE:-64}
     connectTimeoutMs: ${CACHE_REDIS_CONNECT_TIMEOUT:-2000}
+    # Per-command timeout. Bounds request-thread blocking when Redis is slow.
+    commandTimeoutMs: ${CACHE_REDIS_COMMAND_TIMEOUT:-300}
 
     # AWS ElastiCache IAM Authentication (only if using ElastiCache)
     aws:

docker/development/docker-compose.multiserver.yml

@@ -0,0 +1,86 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+
+# Adds a second OM instance that shares MySQL/Elasticsearch/Redis with the
+# primary one in docker-compose.yml. Used to validate that pub/sub
+# invalidation keeps per-instance Guava caches coherent.
+#
+# Usage:
+#   docker compose -f docker-compose.yml -f docker-compose.redis.yml \
+#                  -f docker-compose.multiserver.yml up -d
+services:
+  openmetadata-server-2:
+    image: development-openmetadata-server
+    build:
+      context: ../../.
+      dockerfile: docker/development/Dockerfile
+    container_name: openmetadata_server_2
+    restart: always
+    networks:
+      - local_app_net
+    depends_on:
+      mysql:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    ports:
+      - "8587:8585"
+      - "8588:8586"
+    environment:
+      OPENMETADATA_CLUSTER_NAME: openmetadata
+      SERVER_PORT: 8585
+      SERVER_ADMIN_PORT: 8586
+      LOG_LEVEL: INFO
+      FERNET_KEY: jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=
+      DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
+      DB_SCHEME: mysql
+      DB_USE_SSL: "false"
+      DB_USER: openmetadata_user
+      DB_USER_PASSWORD: openmetadata_password
+      DB_HOST: mysql
+      DB_PORT: 3306
+      DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
+      OM_DATABASE: openmetadata_db
+      ELASTICSEARCH_HOST: elasticsearch
+      ELASTICSEARCH_PORT: 9200
+      ELASTICSEARCH_SCHEME: http
+      SEARCH_TYPE: elasticsearch
+      ELASTICSEARCH_CLUSTER_ALIAS: openmetadata
+      AUTHENTICATION_PROVIDER: basic
+      AUTHENTICATION_ENABLE_SELF_SIGNUP: "true"
+      AUTHORIZER_CLASS_NAME: org.openmetadata.service.security.DefaultAuthorizer
+      AUTHORIZER_REQUEST_FILTER: org.openmetadata.service.security.JwtFilter
+      AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
+      AUTHORIZER_PRINCIPAL_DOMAIN: open-metadata.org
+      AUTHORIZER_ALLOWED_DOMAINS: "[]"
+      AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: '["all"]'
+      AUTHORIZER_INGESTION_PRINCIPALS: "[ingestion-bot]"
+      AUTHENTICATION_RESPONSE_TYPE: id_token
+      AUTHENTICATION_CLIENT_TYPE: public
+      AUTHENTICATION_PUBLIC_KEYS: "[http://openmetadata-server-2:8585/api/v1/system/config/jwks]"
+      AUTHENTICATION_AUTHORITY: https://accounts.google.com
+      AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: "[email,preferred_username,sub]"
+      RSA_PUBLIC_KEY_FILE_PATH: ./conf/public_key.der
+      RSA_PRIVATE_KEY_FILE_PATH: ./conf/private_key.der
+      JWT_ISSUER: open-metadata.org
+      JWT_KEY_ID: Gb389a-9f76-gdjs-a92j-0242bk94356
+      PIPELINE_SERVICE_CLIENT_ENDPOINT: http://ingestion:8080
+      PIPELINE_SERVICE_CLIENT_CLASS_NAME: org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient
+      AIRFLOW_USERNAME: admin
+      AIRFLOW_PASSWORD: admin
+      AIRFLOW_TIMEOUT: 10
+      SECRET_MANAGER: db
+      SERVER_HOST_API_URL: http://openmetadata-server-2:8585/api
+      EVENT_MONITOR: prometheus
+      OPENMETADATA_HEAP_OPTS: "-Xmx1G -Xms1G"
+      CACHE_PROVIDER: redis
+      CACHE_REDIS_URL: redis://redis:6379
+      CACHE_REDIS_AUTH_TYPE: NONE
+      CACHE_REDIS_KEYSPACE: om:dev
+      CACHE_ENTITY_TTL: 3600
+      CACHE_RELATIONSHIP_TTL: 3600
+      CACHE_TAG_TTL: 3600
+      CACHE_REDIS_COMMAND_TIMEOUT: 300

docker/development/docker-compose.redis.yml

@@ -0,0 +1,36 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+
+# Override that adds a Redis cache to the development stack.
+# Usage:
+#   docker compose -f docker-compose.yml -f docker-compose.redis.yml up -d
+services:
+  redis:
+    image: redis:7-alpine
+    container_name: openmetadata_redis
+    restart: always
+    command: ["redis-server", "--appendonly", "no", "--save", "", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"]
+    networks:
+      - local_app_net
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 5
+
+  openmetadata-server:
+    depends_on:
+      redis:
+        condition: service_healthy
+    environment:
+      CACHE_PROVIDER: redis
+      CACHE_REDIS_URL: redis://redis:6379
+      CACHE_REDIS_AUTH_TYPE: NONE
+      CACHE_REDIS_KEYSPACE: om:dev
+      CACHE_ENTITY_TTL: 3600
+      CACHE_RELATIONSHIP_TTL: 3600
+      CACHE_TAG_TTL: 3600
+      CACHE_REDIS_COMMAND_TIMEOUT: 300

openmetadata-integration-tests/pom.xml

@@ -551,6 +551,95 @@
         </plugins>
       </build>
     </profile>
+    <!-- MySQL + Elasticsearch + Redis (Redis cache enabled) -->
+    <profile>
+      <id>mysql-elasticsearch-redis</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <version>${maven.failsafe.version}</version>
+            <executions>
+              <execution>
+                <id>sequential-tests</id>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+                <configuration>
+                  <forkCount>1</forkCount>
+                  <reuseForks>true</reuseForks>
+                  <argLine>-Xmx4096m -XX:+UseG1GC</argLine>
+                  <includes>
+                    <include>**/TagRecognizerFeedbackIT.java</include>
+                    <include>**/WorkflowDefinitionResourceIT.java</include>
+                    <include>**/AppsResourceIT.java</include>
+                    <include>**/SystemResourceIT.java</include>
+                    <include>**/VectorEmbeddingIntegrationIT.java</include>
+                  </includes>
+                  <systemPropertyVariables>
+                    <databaseType>mysql</databaseType>
+                    <databaseImage>mysql:8.3.0</databaseImage>
+                    <searchType>elasticsearch</searchType>
+                    <searchImage>docker.elastic.co/elasticsearch/elasticsearch:9.3.0</searchImage>
+                    <cacheProvider>redis</cacheProvider>
+                    <redisImage>redis:7-alpine</redisImage>
+
+                    <junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
+                    <junit.jupiter.execution.parallel.enabled>false</junit.jupiter.execution.parallel.enabled>
+                  </systemPropertyVariables>
+                  <trimStackTrace>false</trimStackTrace>
+                  <reportFormat>plain</reportFormat>
+                  <useFile>true</useFile>
+                </configuration>
+              </execution>
+              <execution>
+                <id>parallel-tests</id>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+                <configuration>
+                  <forkCount>1</forkCount>
+                  <reuseForks>true</reuseForks>
+                  <argLine>-Xmx4096m -XX:+UseG1GC</argLine>
+                  <includes>
+                    <include>**/*IT.java</include>
+                    <include>**/*Test.java</include>
+                  </includes>
+                  <excludes>
+                    <exclude>**/TagRecognizerFeedbackIT.java</exclude>
+                    <exclude>**/WorkflowDefinitionResourceIT.java</exclude>
+                    <exclude>**/AppsResourceIT.java</exclude>
+                    <exclude>**/SystemResourceIT.java</exclude>
+                    <exclude>**/VectorEmbeddingIntegrationIT.java</exclude>
+                  </excludes>
+                  <systemPropertyVariables>
+                    <databaseType>mysql</databaseType>
+                    <databaseImage>mysql:8.3.0</databaseImage>
+                    <searchType>elasticsearch</searchType>
+                    <searchImage>docker.elastic.co/elasticsearch/elasticsearch:9.3.0</searchImage>
+                    <cacheProvider>redis</cacheProvider>
+                    <redisImage>redis:7-alpine</redisImage>
+
+                    <junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
+                    <junit.jupiter.execution.parallel.enabled>true</junit.jupiter.execution.parallel.enabled>
+                  </systemPropertyVariables>
+                  <trimStackTrace>false</trimStackTrace>
+                  <reportFormat>plain</reportFormat>
+                  <useFile>true</useFile>
+                </configuration>
+              </execution>
+              <execution>
+                <id>verify</id>
+                <goals>
+                  <goal>verify</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <!-- RDF integration tests profile -->
     <profile>
       <id>postgres-rdf-tests</id>

openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java

@@ -129,10 +129,12 @@ public class TestSuiteBootstrap implements LauncherSessionListener {
   private static String databaseType;
   private static String searchType;
   private static boolean rdfEnabled;
+  private static String cacheProvider;
 
   private static JdbcDatabaseContainer<?> DATABASE_CONTAINER;
   private static GenericContainer<?> SEARCH_CONTAINER;
   private static GenericContainer<?> FUSEKI_CONTAINER;
+  private static GenericContainer<?> REDIS_CONTAINER;
   private static K3sContainer K3S_CONTAINER;
   private static DropwizardAppExtension<OpenMetadataApplicationConfig> APP;
   private static Jdbi jdbi;
@@ -141,6 +143,10 @@ public class TestSuiteBootstrap implements LauncherSessionListener {
   private static int searchPort;
   private static String fusekiEndpoint;
   private static String kubeConfigYaml;
+  private static String redisUrl;
+
+  private static final String DEFAULT_REDIS_IMAGE = "redis:7-alpine";
+  private static final int REDIS_PORT = 6379;
 
   @Override
   public void launcherSessionOpened(LauncherSession session) {
@@ -153,11 +159,13 @@ public void launcherSessionOpened(LauncherSession session) {
     databaseType = System.getProperty("databaseType", "postgres");
     searchType = System.getProperty("searchType", "elasticsearch");
     rdfEnabled = Boolean.parseBoolean(System.getProperty("enableRdf", "false"));
+    cacheProvider = System.getProperty("cacheProvider", "none");
 
     LOG.info("=== TestSuiteBootstrap: Starting test infrastructure ===");
     LOG.info("Database type: {}", databaseType);
     LOG.info("Search type: {}", searchType);
     LOG.info("RDF enabled: {}", rdfEnabled);
+    LOG.info("Cache provider: {}", cacheProvider);
     boolean k8sEnabled = isK8sTestsRequested();
     LOG.info("K8s tests enabled: {}", k8sEnabled);
     long startTime = System.currentTimeMillis();
@@ -168,6 +176,9 @@ public void launcherSessionOpened(LauncherSession session) {
       if (rdfEnabled) {
         startFuseki();
       }
+      if (isRedisEnabled()) {
+        startRedis();
+      }
       if (k8sEnabled) {
         startK3s();
       }
@@ -180,6 +191,9 @@ public void launcherSessionOpened(LauncherSession session) {
       if (rdfEnabled) {
         LOG.info("Fuseki SPARQL: {}", fusekiEndpoint);
       }
+      if (isRedisEnabled()) {
+        LOG.info("Redis: {}", redisUrl);
+      }
       if (k8sEnabled) {
         LOG.info("K3s Kubernetes: enabled");
       }
@@ -340,6 +354,58 @@ private void startSearch() {
     }
   }
 
+  private void startRedis() {
+    String image = System.getProperty("redisImage", DEFAULT_REDIS_IMAGE);
+    LOG.info("Starting Redis container with image: {}", image);
+    REDIS_CONTAINER =
+        new GenericContainer<>(DockerImageName.parse(image))
+            .withExposedPorts(REDIS_PORT)
+            .withCommand(
+                "redis-server",
+                "--appendonly",
+                "no",
+                "--save",
+                "",
+                "--maxmemory",
+                "512mb",
+                "--maxmemory-policy",
+                "allkeys-lru")
+            .waitingFor(Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(1)));
+    REDIS_CONTAINER.start();
+    redisUrl =
+        String.format(
+            "redis://%s:%d", REDIS_CONTAINER.getHost(), REDIS_CONTAINER.getMappedPort(REDIS_PORT));
+    LOG.info("Redis started: {}", redisUrl);
+  }
+
+  public static boolean isRedisEnabled() {
+    return "redis".equalsIgnoreCase(cacheProvider);
+  }
+
+  public static String getRedisUrl() {
+    return redisUrl;
+  }
+
+  private void configureCache(OpenMetadataApplicationConfig config) {
+    if (!isRedisEnabled()) {
+      return;
+    }
+    org.openmetadata.service.cache.CacheConfig cacheConfig = config.getCacheConfig();
+    cacheConfig.provider = org.openmetadata.service.cache.CacheConfig.Provider.redis;
+    cacheConfig.redis.url = redisUrl;
+    cacheConfig.redis.authType = org.openmetadata.service.cache.CacheConfig.AuthType.NONE;
+    cacheConfig.redis.keyspace = "om:it:" + System.currentTimeMillis();
+    cacheConfig.redis.commandTimeoutMs = 1000;
+    cacheConfig.entityTtlSeconds = 3600;
+    cacheConfig.relationshipTtlSeconds = 3600;
+    cacheConfig.tagTtlSeconds = 3600;
+    config.setCacheConfig(cacheConfig);
+    LOG.info(
+        "Configured Redis cache: url={} keyspace={}",
+        cacheConfig.redis.url,
+        cacheConfig.redis.keyspace);
+  }
+
   private void startFuseki() {
     String image = System.getProperty("rdfContainerImage", DEFAULT_FUSEKI_IMAGE);
     LOG.info("Starting Fuseki SPARQL container...");
@@ -451,6 +517,7 @@ private void startApplication() throws Exception {
 
     configurePipelineServiceClient(config);
     configureRdf(config);
+    configureCache(config);
 
     IndexMappingLoader.init(getBaseSearchConfig());
 
@@ -718,6 +785,14 @@ private void cleanup() {
       LOG.warn("Error stopping Fuseki container", e);
     }
 
+    try {
+      if (REDIS_CONTAINER != null) {
+        REDIS_CONTAINER.stop();
+      }
+    } catch (Exception e) {
+      LOG.warn("Error stopping Redis container", e);
+    }
+
     try {
       if (K3S_CONTAINER != null) {
         K3S_CONTAINER.stop();