diff --git a/.github/workflows/maven-build-collate.yml b/.github/workflows/maven-build-collate.yml index 964625b795b5..af9152f33e06 100644 --- a/.github/workflows/maven-build-collate.yml +++ b/.github/workflows/maven-build-collate.yml @@ -79,11 +79,12 @@ jobs: uses: the-actions-org/workflow-dispatch@v4 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SHA: ${{ github.event_name == 'push' && github.event.after || github.event.pull_request.head.sha }} + SHA: ${{ github.event_name == 'push' && github.event.after || (github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.head.sha || github.event.pull_request.merge_commit_sha || github.event.pull_request.base.sha) }} + COLLATE_EVENT: ${{ github.event_name == 'push' && 'push' || 'pull_request' }} with: workflow: OpenMetadata Collate Test ref: main repo: open-metadata/openmetadata-collate token: ${{ secrets.COLLATE_PAT }} wait-for-completion: true - inputs: '{ "sha": "${{ env.SHA }}", "event": "${{ github.event_name }}" }' + inputs: '{ "sha": "${{ env.SHA }}", "event": "${{ env.COLLATE_EVENT }}" }' diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index e926b324259d..b8ab5348bcbf 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -60,9 +60,15 @@ ENV LD_LIBRARY_PATH=/instantclient # Install DB2 iAccess Driver RUN if [ $(uname -m) = "x86_64" ]; \ then \ - curl https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list | tee /etc/apt/sources.list.d/ibmi-acs-1.1.0.list \ - && apt update \ - && apt install ibm-iaccess; \ + if curl -f --connect-timeout 10 --max-time 30 \ + https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list \ + -o /etc/apt/sources.list.d/ibmi-acs-1.1.0.list; then \ + apt update \ + && apt install -y ibm-iaccess \ + || echo "Warning: Failed to install ibm-iaccess, continuing without it"; \ + else \ + echo "Warning: Failed to fetch IBM iAccess repo list, continuing without ibm-iaccess"; \ + fi; \ fi # Required for Starting Ingestion Container in Docker Compose diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index a2773d3828a5..ee935402249c 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -60,9 +60,15 @@ ENV LD_LIBRARY_PATH=/instantclient # Install DB2 iAccess Driver RUN if [ $(uname -m) = "x86_64" ]; \ then \ - curl https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list | tee /etc/apt/sources.list.d/ibmi-acs-1.1.0.list \ - && apt update \ - && apt install ibm-iaccess; \ + if curl -f --connect-timeout 10 --max-time 30 \ + https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list \ + -o /etc/apt/sources.list.d/ibmi-acs-1.1.0.list; then \ + apt update \ + && apt install -y ibm-iaccess \ + || echo "Warning: Failed to install ibm-iaccess, continuing without it"; \ + else \ + echo "Warning: Failed to fetch IBM iAccess repo list, continuing without ibm-iaccess"; \ + fi; \ fi # Required for Starting Ingestion Container in Docker Compose diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 20e45e510fc8..baa2d9b16abe 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -63,16 +63,18 @@ RUN if [ $(uname -m) = "arm64" || $(uname -m) = "aarch64" ]; \ ENV LD_LIBRARY_PATH=/instantclient # Install DB2 iAccess driver -# The IBM repository approach is unreliable in CI environments, so we download the package directly -# Use dpkg --force-depends because the package declares old Debian package names (libodbc1, odbcinst1debian2) -# that don't exist in Debian 12, but the actual dependencies (unixodbc, odbcinst) are already installed -RUN if [ $(uname -m) = "x86_64" ]; then \ - wget -q https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/main/binary-amd64/ibm-iaccess-1.1.0.13-1.0.amd64.deb \ - -O /tmp/ibm-iaccess.deb && \ - dpkg -i --force-depends /tmp/ibm-iaccess.deb && \ - apt-get install -f -y --no-install-recommends && \ - rm -f /tmp/ibm-iaccess.deb; \ -fi +RUN if [ $(uname -m) = "x86_64" ]; \ + then \ + if curl -f --connect-timeout 10 --max-time 30 \ + https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list \ + -o /etc/apt/sources.list.d/ibmi-acs-1.1.0.list; then \ + apt update \ + && apt install -y ibm-iaccess \ + || echo "Warning: Failed to install ibm-iaccess, continuing without it"; \ + else \ + echo "Warning: Failed to fetch IBM iAccess repo list, continuing without ibm-iaccess"; \ + fi; \ + fi WORKDIR ingestion/ diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index 5f99d8076cae..8124f9d5f1af 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -63,16 +63,18 @@ RUN if [ $(uname -m) = "arm64" ] | [ $(uname -m) = "aarch64" ]; \ ENV LD_LIBRARY_PATH=/instantclient # Install DB2 iAccess Driver -# The IBM repository approach is unreliable in CI environments, so we download the package directly -# Use dpkg --force-depends because the package declares old Debian package names (libodbc1, odbcinst1debian2) -# that don't exist in Debian 12, but the actual dependencies (unixodbc, odbcinst) are already installed -RUN if [ $(uname -m) = "x86_64" ]; then \ - wget -q https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/main/binary-amd64/ibm-iaccess-1.1.0.13-1.0.amd64.deb \ - -O /tmp/ibm-iaccess.deb && \ - dpkg -i --force-depends /tmp/ibm-iaccess.deb && \ - apt-get install -f -y --no-install-recommends && \ - rm -f /tmp/ibm-iaccess.deb; \ -fi +RUN if [ $(uname -m) = "x86_64" ]; \ + then \ + if curl -f --connect-timeout 10 --max-time 30 \ + https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list \ + -o /etc/apt/sources.list.d/ibmi-acs-1.1.0.list; then \ + apt update \ + && apt install -y ibm-iaccess \ + || echo "Warning: Failed to install ibm-iaccess, continuing without it"; \ + else \ + echo "Warning: Failed to fetch IBM iAccess repo list, continuing without ibm-iaccess"; \ + fi; \ + fi WORKDIR /ingestion diff --git a/ingestion/tests/integration/trino/conftest.py b/ingestion/tests/integration/trino/conftest.py index 0d72df1776ca..128e5f83cc70 100644 --- a/ingestion/tests/integration/trino/conftest.py +++ b/ingestion/tests/integration/trino/conftest.py @@ -206,7 +206,7 @@ def _execute_with_connect(sql): ).fetchall() _execute_with_connect( - "create schema minio.my_schema WITH (location = 's3a://hive-warehouse/')" + "create schema minio.my_schema WITH (location = 's3a://hive-warehouse/my_schema/')" ) data_dir = os.path.dirname(__file__) + "/data" for file in os.listdir(data_dir): @@ -218,7 +218,9 @@ def _execute_with_connect(sql): create_test_data_from_parquet(engine, file_path) sleep(1) - _execute_with_connect("ANALYZE " + f'minio."my_schema"."{file_path.stem}"') + retry(wait=wait_fixed(2), stop=stop_after_delay(120))(_execute_with_connect)( + "ANALYZE " + f'minio."my_schema"."{file_path.stem}"' + ) _execute_with_connect( "CALL system.drop_stats(schema_name => 'my_schema', table_name => 'empty')" ) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryOntologyExportIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryOntologyExportIT.java index 8a2f50f062af..da9a9a88e7ae 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryOntologyExportIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryOntologyExportIT.java @@ -8,6 +8,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; import java.time.Duration; import java.util.List; import java.util.UUID; @@ -52,6 +53,8 @@ public class GlossaryOntologyExportIT { private static final Logger LOG = LoggerFactory.getLogger(GlossaryOntologyExportIT.class); private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + private static final Duration DEFAULT_EXPORT_TIMEOUT = Duration.ofMinutes(3); + private static final Duration RDF_XML_EXPORT_TIMEOUT = Duration.ofMinutes(6); private static final String TURTLE_CONTENT_TYPE = "text/turtle"; private static final String RDF_XML_CONTENT_TYPE = "application/rdf+xml"; @@ -428,11 +431,25 @@ private HttpResponse exportGlossaryRaw( .uri(URI.create(url)) .header("Authorization", "Bearer " + token) .header("Accept", acceptHeader) - .timeout(Duration.ofSeconds(60)) + .timeout(getExportTimeout(format)) .GET() .build(); - return HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString()); + return executeExportRequest(request); + } + + private Duration getExportTimeout(String format) { + return "rdfxml".equalsIgnoreCase(format) || "xml".equalsIgnoreCase(format) + ? RDF_XML_EXPORT_TIMEOUT + : DEFAULT_EXPORT_TIMEOUT; + } + + private HttpResponse executeExportRequest(HttpRequest request) throws Exception { + try { + return HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (HttpTimeoutException timeoutException) { + return HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString()); + } } private int countOccurrences(String text, String pattern) { diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java index 38510781c3e3..1869be9a26ac 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java @@ -36,6 +36,7 @@ import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.schema.utils.JsonUtils; @@ -45,6 +46,7 @@ import org.openmetadata.sdk.models.ListResponse; import org.openmetadata.sdk.network.HttpMethod; import org.openmetadata.sdk.network.RequestOptions; +import org.openmetadata.service.Entity; import org.openmetadata.service.resources.dqtests.TestCaseResource; /** @@ -184,6 +186,24 @@ private Table createTable(TestNamespace ns, List { + TestCase fetched = client.testCases().get(created.getId().toString(), "testSuites"); + assertNotNull(fetched.getTestSuites()); + assertTrue( + fetched.getTestSuites().stream() + .anyMatch(ts -> ts.getId().equals(logicalSuite.getId()))); + }); + + TestSuite updatedSuite = client.testSuites().get(logicalSuite.getId().toString(), "tests"); + assertNotNull(updatedSuite.getTests()); + assertTrue( + updatedSuite.getTests().stream().anyMatch(ref -> ref.getId().equals(created.getId()))); + } + + @Test + void test_upsertTestCaseWithLogicalTestSuites(TestNamespace ns) throws Exception { + OpenMetadataClient client = SdkClients.adminClient(); + Table table = createTable(ns); + + CreateTestSuite logicalSuiteReq = new CreateTestSuite(); + logicalSuiteReq.setName(ns.prefix("logical_suite_upsert_request")); + TestSuite logicalSuite = client.testSuites().create(logicalSuiteReq); + + EntityReference logicalSuiteRef = + new EntityReference() + .withId(logicalSuite.getId()) + .withType(Entity.TEST_SUITE) + .withName(logicalSuite.getName()); + + CreateTestCase createRequest = + TestCaseBuilder.create(client) + .name(ns.prefix("upsert_suite_request_tc")) + .forTable(table) + .testDefinition("tableRowCountToEqual") + .parameter("value", "100") + .description("Initial") + .testSuites(List.of(logicalSuiteRef)) + .build(); + + TestCase created = client.testCases().upsert(createRequest); + assertNotNull(created); + + CreateTestCase updateRequest = + TestCaseBuilder.create(client) + .name(ns.prefix("upsert_suite_request_tc")) + .forTable(table) + .testDefinition("tableRowCountToEqual") + .parameter("value", "200") + .description("Updated") + .testSuites(List.of(logicalSuiteRef)) + .build(); + + TestCase updated = client.testCases().upsert(updateRequest); + assertEquals(created.getId(), updated.getId()); + assertEquals("Updated", updated.getDescription()); + + Awaitility.await("test case upsert should preserve logical test suite membership") + .atMost(Duration.ofSeconds(30)) + .pollInterval(Duration.ofSeconds(2)) + .untilAsserted( + () -> { + TestCase fetched = client.testCases().get(created.getId().toString(), "testSuites"); + assertNotNull(fetched.getTestSuites()); + assertTrue( + fetched.getTestSuites().stream() + .anyMatch(ts -> ts.getId().equals(logicalSuite.getId()))); + }); + } + + @Test + void test_createTestCaseWithLogicalTestSuites_basicSuiteRejected(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + Table table = createTable(ns); + + CreateTestSuite basicSuiteReq = new CreateTestSuite(); + basicSuiteReq.setName(table.getFullyQualifiedName()); + basicSuiteReq.setBasicEntityReference(table.getFullyQualifiedName()); + TestSuite basicSuite = createBasicTestSuite(basicSuiteReq); + + EntityReference basicSuiteRef = + new EntityReference().withId(basicSuite.getId()).withType(Entity.TEST_SUITE); + + CreateTestCase request = + TestCaseBuilder.create(client) + .name(ns.prefix("create_suite_request_basic_rejected_tc")) + .forTable(table) + .testDefinition("tableRowCountToEqual") + .parameter("value", "100") + .testSuites(List.of(basicSuiteRef)) + .build(); + + assertThrows(Exception.class, () -> client.testCases().create(request)); + } + @Test void test_bulkAddTestCasesToLogicalTestSuiteByIds(TestNamespace ns) throws Exception { OpenMetadataClient client = SdkClients.adminClient(); @@ -1058,7 +1202,7 @@ void test_bulkAddTestCasesToLogicalTestSuiteByIds_basicSuiteRejected(TestNamespa CreateTestSuite basicSuiteReq = new CreateTestSuite(); basicSuiteReq.setName(table.getFullyQualifiedName()); basicSuiteReq.setBasicEntityReference(table.getFullyQualifiedName()); - TestSuite basicSuite = client.testSuites().create(basicSuiteReq); + TestSuite basicSuite = createBasicTestSuite(basicSuiteReq); Map request = new HashMap<>(); request.put("testSuiteId", basicSuite.getId().toString()); diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/fluent/builders/TestCaseBuilder.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/fluent/builders/TestCaseBuilder.java index 1abcca3290ce..5e6397ed4381 100644 --- a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/fluent/builders/TestCaseBuilder.java +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/fluent/builders/TestCaseBuilder.java @@ -165,6 +165,14 @@ public TestCaseBuilder useDynamicAssertion(boolean value) { return this; } + /** + * Set test suite references (logical suites) for this test case. + */ + public TestCaseBuilder testSuites(List testSuites) { + request.setTestSuites(testSuites); + return this; + } + /** * Build the CreateTestCase request without executing it. */ diff --git a/openmetadata-sdk/src/test/java/org/openmetadata/sdk/entities/TestCaseMockTest.java b/openmetadata-sdk/src/test/java/org/openmetadata/sdk/entities/TestCaseMockTest.java index 3e14fe418ce1..d8f3ee7f0cfa 100644 --- a/openmetadata-sdk/src/test/java/org/openmetadata/sdk/entities/TestCaseMockTest.java +++ b/openmetadata-sdk/src/test/java/org/openmetadata/sdk/entities/TestCaseMockTest.java @@ -48,7 +48,7 @@ void testCreateTestCase() { createRequest.setName("null-check-test"); createRequest.setDisplayName("Null Check Test"); createRequest.setDescription("Test to check for null values in column"); - // createRequest.setTestSuite("quality-test-suite"); // Not available in CreateTestCase + // createRequest.setTestSuites(...); // Optional on CreateTestCase TestCase expectedTestCase = new TestCase(); expectedTestCase.setId(UUID.randomUUID()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 0f3057a29475..9916e78f49cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -145,7 +145,7 @@ public void setFields(TestCase test, Fields fields, RelationIncludes relationInc fields.contains(Entity.FIELD_TEST_SUITES) ? getTestSuites(test) : test.getTestSuites()); test.setTestSuite( fields.contains(TEST_SUITE_FIELD) - ? getTestSuite(test.getId(), entityType, TEST_SUITE, Direction.FROM) + ? getTestSuite(test.getId(), entityType, TEST_SUITE, Direction.FROM, true) : test.getTestSuite()); test.setTestDefinition( fields.contains(TEST_DEFINITION) ? getTestDefinition(test) : test.getTestDefinition()); @@ -632,6 +632,8 @@ public void setFullyQualifiedName(TestCase test) { @Override public void prepare(TestCase test, boolean update) { + validateLogicalTestSuites(test, update); + EntityLink entityLink = EntityLink.parse(test.getEntityLink()); EntityUtil.validateEntityLink(entityLink); @@ -667,6 +669,50 @@ public void prepare(TestCase test, boolean update) { test.setTestSuite(testSuite); } + private void validateLogicalTestSuites(TestCase test, boolean update) { + List testSuites = test.getTestSuites(); + if (testSuites == null || testSuites.isEmpty()) return; + + final Set existingSuiteIds = new HashSet<>(); + if (update && test.getId() != null) { + existingSuiteIds.addAll( + findFromRecords(test.getId(), TEST_CASE, Relationship.CONTAINS, TEST_SUITE).stream() + .map(CollectionDAO.EntityRelationshipRecord::getId) + .collect(Collectors.toSet())); + } + + List newSuiteIds = + testSuites.stream() + .filter(ts -> ts != null && ts.getId() != null) + .map(TestSuite::getId) + .filter(id -> !existingSuiteIds.contains(id)) + .distinct() + .toList(); + + if (newSuiteIds.isEmpty()) { + return; + } + + List suiteReferences = + newSuiteIds.stream() + .map(id -> new EntityReference().withId(id).withType(TEST_SUITE)) + .toList(); + List fetchedSuites = Entity.getEntities(suiteReferences, "basic", NON_DELETED); + Map testSuiteById = + fetchedSuites.stream().collect(Collectors.toMap(TestSuite::getId, Function.identity())); + + for (UUID suiteId : newSuiteIds) { + TestSuite testSuite = testSuiteById.get(suiteId); + if (testSuite == null) { + throw new EntityNotFoundException(entityNotFound(TEST_SUITE, suiteId)); + } + if (Boolean.TRUE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException( + "You are trying to add test cases to a basic test suite."); + } + } + } + /* * Get the test suite for a test case. We'll use the entity linked to the test case * to find the basic test suite. If it doesn't exist, create a new one. @@ -679,7 +725,7 @@ EntityReference getOrCreateTestSuite(TestCase test) { private EntityReference getOrCreateTestSuite(TestCase test, EntityInterface tableEntity) { try { - return getTestSuite(tableEntity.getId(), TEST_SUITE, TABLE, Direction.TO); + return getTestSuite(tableEntity.getId(), TEST_SUITE, TABLE, Direction.TO, false); } catch (EntityNotFoundException e) { var entityLink = EntityLink.parse(test.getEntityLink()); var testSuiteRepository = (TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE); @@ -695,7 +741,8 @@ private EntityReference getOrCreateTestSuite(TestCase test, EntityInterface tabl } } - private EntityReference getTestSuite(UUID id, String to, String from, Direction direction) + private EntityReference getTestSuite( + UUID id, String to, String from, Direction direction, boolean allowMissingExecutable) throws EntityNotFoundException { // `testSuite` field returns the executable `testSuite` linked to that testCase List records = new ArrayList<>(); @@ -703,12 +750,19 @@ private EntityReference getTestSuite(UUID id, String to, String from, Direction case FROM -> records = findFromRecords(id, to, Relationship.CONTAINS, from); case TO -> records = findToRecords(id, from, Relationship.CONTAINS, to); } + EntityReference firstSuite = null; for (CollectionDAO.EntityRelationshipRecord testSuiteId : records) { TestSuite testSuite = Entity.getEntity(TEST_SUITE, testSuiteId.getId(), "", Include.ALL); + if (firstSuite == null) { + firstSuite = testSuite.getEntityReference(); + } if (Boolean.TRUE.equals(testSuite.getBasic())) { return testSuite.getEntityReference(); } } + if (allowMissingExecutable) { + return firstSuite; + } throw new EntityNotFoundException( String.format( "Error occurred when retrieving executable test suite for testCase %s. ", @@ -857,6 +911,7 @@ public void storeRelationships(TestCase test) { TEST_DEFINITION, TEST_CASE, Relationship.CONTAINS); + addLogicalTestSuiteRelationships(test); } @Override @@ -878,6 +933,11 @@ public void clearParentCache() { } private void updateTestSuite(TestCase testCase) { + if (testCase == null + || testCase.getTestSuite() == null + || testCase.getTestSuite().getId() == null) { + return; + } var testSuiteRepository = (TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE); TestSuite testSuite = Entity.getEntity(testCase.getTestSuite(), "*", ALL); var original = TestSuiteRepository.copyTestSuite(testSuite); @@ -891,6 +951,20 @@ private void updateLogicalTestSuite(UUID testSuiteId) { testSuiteRepository.postUpdate(original, testSuite); } + private void addLogicalTestSuiteRelationships(TestCase testCase) { + if (testCase == null || nullOrEmpty(testCase.getTestSuites())) return; + + Set suiteIds = + testCase.getTestSuites().stream() + .filter(ts -> ts != null && ts.getId() != null) + .map(TestSuite::getId) + .collect(Collectors.toSet()); + + for (UUID suiteId : suiteIds) { + addRelationship(suiteId, testCase.getId(), TEST_SUITE, TEST_CASE, Relationship.CONTAINS); + } + } + @Transaction @Override protected void deleteChildren( @@ -1371,6 +1445,17 @@ public void entitySpecificUpdate(boolean consolidatingChanges) { Relationship.CONTAINS, TEST_CASE, updated.getId())); + compareAndUpdate( + "testSuites", + () -> + updateFromRelationships( + "testSuites", + TEST_SUITE, + getTestSuiteReferences(original.getTestSuites()), + getTestSuiteReferences(updated.getTestSuites()), + Relationship.CONTAINS, + TEST_CASE, + updated.getId())); compareAndUpdate( "parameterValues", () -> @@ -2069,4 +2154,13 @@ private List parseParameterValues(String parameterValues .collect(Collectors.toList()); } } + + private List getTestSuiteReferences(List testSuites) { + return nullOrEmpty(testSuites) + ? List.of() + : testSuites.stream() + .filter(testSuite -> testSuite != null && testSuite.getId() != null) + .map(TestSuite::getEntityReference) + .toList(); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java index 9b231cff49fb..d3c28226c340 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java @@ -2,8 +2,11 @@ import static org.openmetadata.service.util.EntityUtil.getEntityReference; +import java.util.List; import org.openmetadata.schema.api.tests.CreateTestCase; import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; import org.openmetadata.service.mapper.EntityMapper; import org.openmetadata.service.resources.feeds.MessageParser; @@ -12,6 +15,12 @@ public class TestCaseMapper implements EntityMapper { @Override public TestCase createToEntity(CreateTestCase create, String user) { MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(create.getEntityLink()); + + List testSuites = + create.getTestSuites() == null + ? null + : create.getTestSuites().stream().map(this::toTestSuiteStub).toList(); + return copy(new TestCase(), create, user) .withDescription(create.getDescription()) .withName(create.getName()) @@ -24,7 +33,15 @@ public TestCase createToEntity(CreateTestCase create, String user) { .withTopDimensions(create.getTopDimensions()) .withEntityFQN(entityLink.getFullyQualifiedFieldValue()) .withTestDefinition(getEntityReference(Entity.TEST_DEFINITION, create.getTestDefinition())) + .withTestSuites(testSuites) .withTags(create.getTags()) .withCreatedBy(user); } + + private TestSuite toTestSuiteStub(EntityReference suiteReference) { + return new TestSuite() + .withId(suiteReference.getId()) + .withFullyQualifiedName(suiteReference.getFullyQualifiedName()) + .withName(suiteReference.getName()); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index f5140b493459..5601b66b9314 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -53,6 +53,7 @@ import org.openmetadata.schema.tests.type.TestCaseResult; import org.openmetadata.schema.tests.type.TestCaseStatus; import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.TableData; @@ -692,6 +693,8 @@ public Response create( new CreateResourceContext<>(entityType, test), new OperationContext(Entity.TEST_CASE, MetadataOperation.CREATE_TESTS)); + authorizeOptionalLogicalTestSuites(create, securityContext); + OperationContext tableOpContext = new OperationContext(Entity.TABLE, MetadataOperation.CREATE_TESTS); ResourceContextInterface tableResourceContext = @@ -761,6 +764,7 @@ public Response createMany( createTestCases.forEach( create -> { + authorizeOptionalLogicalTestSuites(create, securityContext); TestCase test = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); limits.enforceLimits( @@ -860,6 +864,7 @@ public Response createOrUpdate( new AuthRequest(testCaseOpCreate, testCaseRC), new AuthRequest(testCaseOpUpdate, testCaseRC)); authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); + TestCase test = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); repository.prepareInternal(test, true); PutResponse response = @@ -1584,6 +1589,32 @@ private void validateTestSuiteOps(TestSuite testSuite, SecurityContext securityC } } + private void authorizeOptionalLogicalTestSuites( + CreateTestCase create, SecurityContext securityContext) { + if (create.getTestSuites() == null || create.getTestSuites().isEmpty()) return; + + for (EntityReference suiteReference : create.getTestSuites()) { + if (nullOrEmpty(suiteReference) || nullOrEmpty(suiteReference.getId())) { + throw new IllegalArgumentException("Test suite reference must include a valid id."); + } + if (!nullOrEmpty(suiteReference.getType()) + && !Entity.TEST_SUITE.equals(suiteReference.getType())) { + throw new IllegalArgumentException( + String.format( + "Invalid test suite reference type '%s'. Expected '%s'.", + suiteReference.getType(), Entity.TEST_SUITE)); + } + TestSuite testSuite = + Entity.getEntity( + Entity.TEST_SUITE, + suiteReference.getId(), + "domains,owners", + Include.NON_DELETED, + false); + validateTestSuiteOps(testSuite, securityContext); + } + } + private List getExcludedIdsFromSelection(BundleSuiteBulkAddRequestBulkAll bulkAll) { if (bulkAll == null || bulkAll.getFilter() == null) { return List.of(); diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json index e58fc63c471f..b82a4f8e41dc 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json @@ -26,6 +26,10 @@ "entityLink": { "$ref": "../../type/basic.json#/definitions/entityLink" }, + "testSuites": { + "description": "Logical test suites this test case belongs to. Basic suite membership is derived automatically from entityLink/testSuite and should not be provided here.", + "$ref": "../../type/entityReferenceList.json" + }, "parameterValues": { "type": "array", "items": { diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index d2930139d60c..e101eca152d0 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -26,7 +26,7 @@ "license-header-check": "license-check-and-add check -f .licenseheaderrc.json", "license-header-fix": "license-check-and-add add -f .licenseheaderrc.json -r $(date +%Y)", "parse-schema": "node parseSchemas", - "js-antlr": "PWD=$(echo $PWD) antlr4 -Dlanguage=JavaScript -o src/generated/antlr \"$PWD\"/../../../../../openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4", + "js-antlr": "if command -v antlr4 >/dev/null 2>&1; then PWD=$(echo $PWD) antlr4 -Dlanguage=JavaScript -o src/generated/antlr \"$PWD\"/../../../../../openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4; else echo 'antlr4 CLI not found, using checked-in generated parser files'; fi", "i18n": "sync-i18n --files '**/locale/languages/*.json' --primary en-us --space 2 --fn", "check-i18n": "npm run i18n -- --check", "lint:base": "NODE_OPTIONS='--max-old-space-size=8192' eslint --no-error-on-unmatched-pattern", diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/user/UserClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/user/UserClass.ts index 42c5a745bec7..73b57fd847d9 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/user/UserClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/user/UserClass.ts @@ -35,12 +35,14 @@ let dataStewardTeam: TeamClass; export class UserClass { data: UserData; + private readonly hasCustomData: boolean; responseData: UserResponseDataType = {} as UserResponseDataType; isUserDataSteward = false; constructor(data?: UserData) { this.data = data ? data : generateRandomUsername(); + this.hasCustomData = Boolean(data); } async create(apiContext: APIRequestContext, assignRole = true) { @@ -50,10 +52,29 @@ export class UserClass { const dataConsumerRole = await dataConsumerRoleResponse.json(); - const response = await apiContext.post('/api/v1/users/signup', { + let response = await apiContext.post('/api/v1/users/signup', { data: this.data, }); + if (!response.ok()) { + const errorText = await response.text(); + const canRetryWithNewUser = + !this.hasCustomData && + response.status() === 400 && + errorText.includes('User with Email Already Exists'); + + if (canRetryWithNewUser) { + this.data = generateRandomUsername(); + response = await apiContext.post('/api/v1/users/signup', { + data: this.data, + }); + } else { + throw new Error( + `UserClass.create() failed with status ${response.status()}: ${errorText}` + ); + } + } + if (!response.ok()) { throw new Error( `UserClass.create() failed with status ${response.status()}: ${await response.text()}` diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts index ecd114126fa2..9966eef10a4e 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts @@ -107,12 +107,6 @@ export const visitUserProfilePage = async (page: Page, userName: string) => { const userResponse = page.waitForResponse( '/api/v1/search/query?q=*&index=*&from=0&size=*' ); - const loaderPromise = page - .getByTestId('user-list-v1-component') - .getByTestId('loader') - .waitFor({ - state: 'detached', - }); const searchBar = page.getByTestId('searchbar'); await expect @@ -122,12 +116,18 @@ export const visitUserProfilePage = async (page: Page, userName: string) => { await searchBar.fill(''); await searchBar.fill(userName); await searchRequest; - await loaderPromise.catch(() => undefined); + await page + .getByTestId('user-list-v1-component') + .getByTestId('loader') + .waitFor({ + state: 'detached', + }) + .catch(() => undefined); return await page.getByTestId(userName).count(); }, { - timeout: 60000, + timeout: 120000, intervals: [1000, 2000, 5000], message: `Timed out waiting for user ${userName} to become visible in the user list`, } @@ -512,8 +512,10 @@ export const revokeToken = async (page: Page, isBot?: boolean) => { ); await page.click('[data-testid="save-button"]'); - - await expect(page.locator('[data-testid="revoke-button"]')).not.toBeVisible(); + await waitForAllLoadersToDisappear(page); + await expect(page.locator('[data-testid="revoke-button"]')).toBeHidden({ + timeout: 30_000, + }); }; export const updateExpiration = async (page: Page, expiry: number) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts index 92a77fad3a0b..bfd6fb6759b8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts @@ -55,6 +55,11 @@ export interface CreateTestCase { * Fully qualified name of the test definition. */ testDefinition: string; + /** + * Logical test suites this test case belongs to. Basic suite membership is derived + * automatically from entityLink/testSuite and should not be provided here. + */ + testSuites?: EntityReference[]; /** * Number of top dimension values to show before grouping the rest as Others. Controls the * cardinality of dimensional test results. Defaults to 5 when not specified.