Perceived-latency P2 + P3.5 servlet/HTTP2: hydrate, SWR cache, skeletons, Vary, h2c/h2 opt-in

conf/openmetadata.yaml

@@ -64,7 +64,10 @@ server:
       acceptorThreads: 1  # Admin endpoint needs minimal resources
       selectorThreads: 1
 
-  # Response compression disabled for maximum throughput
+  # Response compression for entity GETs / list endpoints — Dropwizard delegates to Jetty's
+  # GzipHandler. Default min size is ~256 bytes; below that the CPU cost outweighs the wire
+  # savings. Pre-compressed UI assets (.gz / .br produced at vite build time) are served by
+  # OpenMetadataAssetServlet's content negotiation, not by this handler.
   gzip:
     enabled: true
 
@@ -154,6 +157,50 @@ server:
   #    allowRenegotiation: true
   #    endpointIdentificationAlgorithm: (none)
 
+# HTTP/2 support — opt-in. The dropwizard-http2 module is on the classpath, so the
+# `type: h2` (TLS) and `type: h2c` (cleartext) connector types are available out of the
+# box. Both are backward-compatible: HTTP/1.1 clients still work on the same port (h2c
+# negotiates via the HTTP/1.1 Upgrade header; h2 negotiates via TLS ALPN), so flipping
+# from `type: http` to `type: h2c` does not break the SDK or browser.
+#
+# When to enable at Jetty:
+#   * Self-hosted single-node deploys with no LB in front (Jetty IS the edge).
+#   * Production deploys that want end-to-end HTTP/2 — multiplexing all the way through
+#     instead of LB↔pod fanning out to many HTTP/1.1 connections under high parallelism.
+#   * Behind h2c-aware proxies (envoy, nginx with `proxy_http_version 2`).
+#
+# When to leave HTTP/2 to the LB and keep `type: http` here:
+#   * Production deploys behind nginx-ingress / ALB / Traefik that already terminate
+#     HTTP/2 from clients. The LB↔pod hop in HTTP/1.1 is "fine" intra-VPC, and not
+#     worth the cert-management cost for marginal multiplexing gains.
+#
+# HTTP/2 cleartext (h2c) on the existing port — most common opt-in shape:
+#server:
+#  applicationConnectors:
+#    - type: h2c
+#      port: 8585
+#      maxConcurrentStreams: 1024
+#      initialStreamRecvWindow: 65535
+#  adminConnectors:
+#    - type: http      # admin endpoint stays HTTP/1.1; ops scrapers don't need HTTP/2
+#      port: 8586
+#
+# HTTP/2 over TLS (h2) for browser-direct deploys — replaces the https sample above:
+#server:
+#  applicationConnectors:
+#    - type: h2
+#      port: 8585
+#      keyStorePath: ./conf/keystore.jks
+#      keyStorePassword: changeit
+#      keyStoreType: JKS
+#      supportedProtocols: [TLSv1.2, TLSv1.3]
+#      # h2 mandates the cipher list include AEAD suites (RFC 7540 §9.2.2):
+#      supportedCipherSuites:
+#        - TLS_AES_256_GCM_SHA384
+#        - TLS_AES_128_GCM_SHA256
+#        - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#        - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+
 qos:
   # Virtual threads remove Jetty's natural back-pressure. Keep request admission below the DB
   # pool ceiling so excess load queues instead of failing with connection exhaustion.

openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/LineageHydrateIT.java

@@ -0,0 +1,247 @@
+/*
+ *  Copyright 2026 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.openmetadata.it.tests;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.openmetadata.it.factories.DashboardServiceTestFactory;
+import org.openmetadata.it.factories.DatabaseSchemaTestFactory;
+import org.openmetadata.it.factories.DatabaseServiceTestFactory;
+import org.openmetadata.it.util.SdkClients;
+import org.openmetadata.it.util.TestNamespace;
+import org.openmetadata.schema.api.data.CreateDashboard;
+import org.openmetadata.schema.api.data.CreateTable;
+import org.openmetadata.schema.api.lineage.HydrateLineageRequest;
+import org.openmetadata.schema.entity.data.Dashboard;
+import org.openmetadata.schema.entity.data.DatabaseSchema;
+import org.openmetadata.schema.entity.data.Table;
+import org.openmetadata.schema.entity.services.DashboardService;
+import org.openmetadata.schema.entity.services.DatabaseService;
+import org.openmetadata.schema.type.Column;
+import org.openmetadata.schema.type.EntityReference;
+import org.openmetadata.sdk.client.OpenMetadataClient;
+import org.openmetadata.sdk.fluent.builders.ColumnBuilder;
+import org.openmetadata.sdk.network.HttpMethod;
+import org.openmetadata.sdk.network.RequestOptions;
+
+/**
+ * Integration tests for {@code POST /v1/lineage/hydrate} — the batch entity hydration endpoint.
+ *
+ * <p>Replaces N per-node entity GETs with one round-trip. Tests cover the happy paths (single
+ * type, mixed types, fields propagation), request validation, and the request-shape contract
+ * of the silent-drop authorization mode (non-existent ids do not fail the batch — the response
+ * omits them).
+ *
+ * <p>The full "permitted vs denied principal" silent-drop contract is enforced at the
+ * implementation level by {@code LineageResource.filterAuthorizedIds} (which calls
+ * {@code authorizer.getPermission} and keeps only ids whose {@code VIEW_BASIC} access is
+ * {@code ALLOW} or {@code CONDITIONAL_ALLOW}). End-to-end coverage with a restricted-permission
+ * principal is left as a follow-up — it requires bootstrapping a team / domain / policy stack
+ * that's heavier than this IT's scope.
+ */
+@Execution(ExecutionMode.CONCURRENT)
+public class LineageHydrateIT {
+
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+  private static final String HYDRATE_PATH = "/v1/lineage/hydrate";
+
+  @Test
+  void hydrateReturnsTablesGroupedByType() throws Exception {
+    OpenMetadataClient client = SdkClients.adminClient();
+    TestNamespace namespace = new TestNamespace("LineageHydrateIT");
+
+    Table t1 = createTable(client, namespace, "hydrate_one");
+    Table t2 = createTable(client, namespace, "hydrate_two");
+
+    HydrateLineageRequest request =
+        new HydrateLineageRequest()
+            .withEntities(
+                List.of(
+                    new EntityReference().withType("table").withId(t1.getId()),
+                    new EntityReference().withType("table").withId(t2.getId())));
+
+    JsonNode response = postHydrate(client, request);
+
+    assertTrue(response.has("table"), "response must group by entityType");
+    JsonNode tables = response.get("table");
+    assertEquals(2, tables.size(), "both requested tables must be returned");
+    assertNotNull(tables.get(0).get("fullyQualifiedName"));
+    assertNotNull(tables.get(0).get("version"), "hydrated entities should include version");
+  }
+
+  @Test
+  void hydrateMixedTypesReturnsSeparateGroups() throws Exception {
+    OpenMetadataClient client = SdkClients.adminClient();
+    TestNamespace namespace = new TestNamespace("LineageHydrateIT");
+
+    Table table = createTable(client, namespace, "hydrate_mixed_table");
+    Dashboard dashboard = createDashboard(client, namespace, "hydrate_mixed_dash");
+
+    HydrateLineageRequest request =
+        new HydrateLineageRequest()
+            .withEntities(
+                List.of(
+                    new EntityReference().withType("table").withId(table.getId()),
+                    new EntityReference().withType("dashboard").withId(dashboard.getId())));
+
+    JsonNode response = postHydrate(client, request);
+
+    assertEquals(1, response.get("table").size());
+    assertEquals(1, response.get("dashboard").size());
+    assertEquals(table.getId().toString(), response.get("table").get(0).get("id").asText());
+    assertEquals(dashboard.getId().toString(), response.get("dashboard").get(0).get("id").asText());
+  }
+
+  @Test
+  void hydrateAppliesFieldsParameter() throws Exception {
+    OpenMetadataClient client = SdkClients.adminClient();
+    TestNamespace namespace = new TestNamespace("LineageHydrateIT");
+
+    Table table = createTable(client, namespace, "hydrate_with_fields");
+
+    HydrateLineageRequest withoutFields =
+        new HydrateLineageRequest()
+            .withEntities(List.of(new EntityReference().withType("table").withId(table.getId())));
+    HydrateLineageRequest withFields =
+        new HydrateLineageRequest()
+            .withEntities(List.of(new EntityReference().withType("table").withId(table.getId())))
+            .withFields("tags,owners");
+
+    JsonNode bare = postHydrate(client, withoutFields);
+    JsonNode rich = postHydrate(client, withFields);
+
+    JsonNode bareTable = bare.get("table").get(0);
+    JsonNode richTable = rich.get("table").get(0);
+
+    // tags / owners are not populated on a bare GET unless explicitly requested.
+    assertFalse(
+        bareTable.has("tags")
+            && bareTable.get("tags").isArray()
+            && bareTable.get("tags").size() > 0,
+        "bare hydration should not populate tags");
+    // With fields requested, the keys must be present (may be empty arrays).
+    assertTrue(richTable.has("tags"), "fields=tags must include tags key");
+    assertTrue(richTable.has("owners"), "fields=owners must include owners key");
+  }
+
+  @Test
+  void hydrateSilentlyDropsMissingIds() throws Exception {
+    // The endpoint's silent-drop contract: ids the batch cannot resolve (because they're
+    // unauthorized OR non-existent) are omitted from the response rather than failing the
+    // entire batch. This test exercises the shape using a non-existent UUID alongside a
+    // valid table — full per-principal authz coverage requires team/domain bootstrapping and
+    // is tracked as follow-up (see class JavaDoc).
+    OpenMetadataClient client = SdkClients.adminClient();
+    TestNamespace namespace = new TestNamespace("LineageHydrateIT");
+    Table table = createTable(client, namespace, "hydrate_silent_drop");
+
+    HydrateLineageRequest request =
+        new HydrateLineageRequest()
+            .withEntities(
+                List.of(
+                    new EntityReference().withType("table").withId(table.getId()),
+                    new EntityReference()
+                        .withType("table")
+                        .withId(java.util.UUID.randomUUID())));
+
+    JsonNode response = postHydrate(client, request);
+
+    // The batch returns 200 with the resolvable id, omitting the missing one — not 404 or
+    // empty.
+    assertTrue(response.has("table"), "response must include the resolvable table");
+    JsonNode tables = response.get("table");
+    assertEquals(1, tables.size(), "only the existing table should be returned");
+    assertEquals(table.getId().toString(), tables.get(0).get("id").asText());
+  }
+
+  @Test
+  void hydrateRejectsEmptyEntities() {
+    OpenMetadataClient client = SdkClients.adminClient();
+    HydrateLineageRequest empty = new HydrateLineageRequest().withEntities(List.of());
+    Exception thrown = assertThrows(Exception.class, () -> postHydrate(client, empty));
+    // Either 400 from bean validation (@Size min=1) or 400 from our own check.
+    String msg = thrown.getMessage() == null ? "" : thrown.getMessage();
+    assertTrue(
+        msg.contains("400") || msg.toLowerCase().contains("size") || msg.contains("entities"),
+        "empty entities must yield a 4xx, got: " + msg);
+  }
+
+  @Test
+  void hydrateUnknownTypeFailsCleanly() throws Exception {
+    OpenMetadataClient client = SdkClients.adminClient();
+    HydrateLineageRequest request =
+        new HydrateLineageRequest()
+            .withEntities(
+                List.of(
+                    new EntityReference()
+                        .withType("nonexistent_type_xyz")
+                        .withId(java.util.UUID.randomUUID())));
+    Exception thrown = assertThrows(Exception.class, () -> postHydrate(client, request));
+    String msg = thrown.getMessage() == null ? "" : thrown.getMessage();
+    assertTrue(
+        msg.contains("nonexistent_type_xyz")
+            || msg.toLowerCase().contains("entity type")
+            || msg.contains("400")
+            || msg.contains("404"),
+        "unknown entity type must yield a 4xx, got: " + msg);
+  }
+
+  private static JsonNode postHydrate(OpenMetadataClient client, HydrateLineageRequest request)
+      throws Exception {
+    String body =
+        client
+            .getHttpClient()
+            .executeForString(
+                HttpMethod.POST, HYDRATE_PATH, request, RequestOptions.builder().build());
+    return MAPPER.readTree(body);
+  }
+
+  private Table createTable(OpenMetadataClient client, TestNamespace namespace, String tableName)
+      throws Exception {
+    DatabaseService service = DatabaseServiceTestFactory.createPostgres(namespace);
+    DatabaseSchema schema = DatabaseSchemaTestFactory.createSimple(namespace, service);
+
+    CreateTable createTable = new CreateTable();
+    createTable.setName(namespace.prefix(tableName));
+    createTable.setDatabaseSchema(schema.getFullyQualifiedName());
+
+    List<Column> columns =
+        List.of(
+            ColumnBuilder.of("id", "BIGINT").primaryKey().notNull().build(),
+            ColumnBuilder.of("name", "VARCHAR").dataLength(255).build());
+    createTable.setColumns(columns);
+
+    return client.tables().create(createTable);
+  }
+
+  private Dashboard createDashboard(
+      OpenMetadataClient client, TestNamespace namespace, String dashboardName) throws Exception {
+    DashboardService service = DashboardServiceTestFactory.createMetabase(namespace);
+
+    CreateDashboard createDashboard = new CreateDashboard();
+    createDashboard.setName(namespace.prefix(dashboardName));
+    createDashboard.setService(service.getFullyQualifiedName());
+
+    return client.dashboards().create(createDashboard);
+  }
+}

openmetadata-service/pom.xml

@@ -191,6 +191,18 @@
       <groupId>io.dropwizard</groupId>
       <artifactId>dropwizard-jdbi3</artifactId>
     </dependency>
+    <!--
+      dropwizard-http2 registers `type: h2` (TLS) and `type: h2c` (cleartext) connector
+      factories via META-INF/services. Adding the dep is what makes those connector types
+      available in conf/openmetadata.yaml — without it, "type: h2c" fails at startup with
+      "unknown connector type". Both factories are backward-compatible with HTTP/1.1 clients
+      on the same port (cleartext via Upgrade, TLS via ALPN), so adopting them does not
+      break existing clients.
+    -->
+    <dependency>
+      <groupId>io.dropwizard</groupId>
+      <artifactId>dropwizard-http2</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.jdbi</groupId>
       <artifactId>jdbi3-sqlobject</artifactId>