diff --git a/.gitignore b/.gitignore index 622bbb564a00..3f46a7e975ad 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,7 @@ openmetadata-ui/src/main/resources/ui/.env openmetadata-ui/src/main/resources/ui/playwright/.auth openmetadata-ui/src/main/resources/ui/blob-report openmetadata-ui/src/main/resources/ui/test-results/ +openmetadata-ui/src/main/resources/ui/debug.json #UI - Dereferenced Schemas openmetadata-ui/src/main/resources/ui/src/jsons/* diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java index 046896796584..f9e3d3549fba 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java @@ -5647,6 +5647,45 @@ void test_importExportRoundTrip(TestNamespace ns) { } } + /** + * Test: CSV import/export round-trip preserves Unicode text. + * Verifies non-ASCII content survives export, import, and re-export. + */ + @Test + void test_importExportRoundTripUnicode(TestNamespace ns) { + Assumptions.assumeTrue(supportsImportExport, "Entity does not support import/export"); + + org.openmetadata.sdk.services.EntityServiceBase service = getEntityService(); + Assumptions.assumeTrue(service != null, "Entity service not provided"); + + String containerName = getImportExportContainerName(ns); + Assumptions.assumeTrue(containerName != null, "Container name not provided"); + + K createRequest = createMinimalRequest(ns); + String unicodeDescription = "中文描述 - CSV import/export round trip"; + setDescription(createRequest, unicodeDescription); + T entity = createEntity(createRequest); + assertNotNull(entity, "Entity should be created"); + + try { + String exportedCsv = service.exportCsv(containerName); + assertNotNull(exportedCsv, "Export should return CSV data"); + assertTrue( + exportedCsv.contains(unicodeDescription), "Exported CSV should contain Unicode text"); + + CsvImportResult importResult = performImportCsv(ns, exportedCsv, false); + assertEquals( + ApiStatus.SUCCESS, importResult.getStatus(), "Unicode round-trip should succeed"); + + String reExportedCsv = service.exportCsv(containerName); + assertNotNull(reExportedCsv, "Re-export should return CSV data"); + assertTrue( + reExportedCsv.contains(unicodeDescription), "Re-exported CSV should retain Unicode text"); + } catch (Exception e) { + fail("Unicode import/export round-trip failed: " + e.getMessage()); + } + } + // =================================================================== // COMPREHENSIVE CSV IMPORT/EXPORT TESTS // Template-based tests that work with any entity CSV structure diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 9c1199a29557..b25762fedbb8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -40,6 +40,7 @@ public final class CsvUtil { public static final String SEPARATOR = ","; public static final String FIELD_SEPARATOR = ";"; + public static final String UTF8_BOM = "\uFEFF"; public static final String ENTITY_TYPE_SEPARATOR = ":"; public static final String LINE_SEPARATOR = "\r\n"; @@ -50,6 +51,13 @@ private CsvUtil() { // Utility class hides the constructor } + public static String stripUtf8Bom(String value) { + if (value == null || value.isEmpty() || !value.startsWith(UTF8_BOM)) { + return value; + } + return value.substring(1); + } + public static String formatCsv(CsvFile csvFile) throws IOException { // CSV file is generated by the backend and the data exported is expected to be correct. Hence, // no validation diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java index bc75c9b27b57..12b6153b27e5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java @@ -37,6 +37,7 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.csv.CsvUtil; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.data.BulkColumnUpdatePreview; import org.openmetadata.schema.api.data.BulkColumnUpdateRequest; @@ -875,7 +876,7 @@ public CsvImportResult importColumnsCSV( result.setNumberOfRowsPassed(0); result.setNumberOfRowsFailed(0); - String[] lines = csv.split("\n"); + String[] lines = CsvUtil.stripUtf8Bom(csv).split("\n"); if (lines.length <= 1) { result.setStatus(ApiStatus.ABORTED); result.setAbortReason("No data to import"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index c8c5a48af470..f566a66b54fd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -53,6 +53,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.csv.CsvExportProgressCallback; import org.openmetadata.csv.CsvImportProgressCallback; +import org.openmetadata.csv.CsvUtil; import org.openmetadata.schema.BulkAssetsRequestInterface; import org.openmetadata.schema.CreateEntity; import org.openmetadata.schema.EntityInterface; @@ -1029,18 +1030,19 @@ protected CsvImportResult importCsvInternal( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.EDIT_ALL); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); + String normalizedCsv = CsvUtil.stripUtf8Bom(csv); CsvImportResult result = nullOrEmpty(versioningEntityType) ? repository.importFromCsv( name, - csv, + normalizedCsv, dryRun, securityContext.getUserPrincipal().getName(), recursive, progressCallback) : repository.importFromCsv( name, - csv, + normalizedCsv, dryRun, securityContext.getUserPrincipal().getName(), recursive, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/columns/ColumnResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/columns/ColumnResource.java index ed29a6d17ebb..5327b889b277 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/columns/ColumnResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/columns/ColumnResource.java @@ -40,6 +40,7 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.csv.CsvUtil; import org.openmetadata.schema.api.data.BulkColumnUpdatePreview; import org.openmetadata.schema.api.data.BulkColumnUpdateRequest; import org.openmetadata.schema.api.data.ColumnGridResponse; @@ -356,7 +357,7 @@ public Response bulkUpdateColumnsPreview( @GET @Path("/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({"text/csv; charset=UTF-8"}) @Operation( operationId = "exportUniqueColumns", summary = "Export unique column names to CSV", @@ -370,7 +371,7 @@ public Response bulkUpdateColumnsPreview( description = "CSV export of unique columns", content = @Content( - mediaType = "text/plain", + mediaType = "text/csv; charset=UTF-8", schema = @Schema(implementation = String.class))), @ApiResponse(responseCode = "400", description = "Bad request") }) @@ -402,7 +403,7 @@ public String exportUniqueColumns( @POST @Path("/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Operation( operationId = "importUniqueColumns", summary = "Import column metadata from CSV (with dry-run)", @@ -442,11 +443,12 @@ public Response importUniqueColumns( @Parameter(description = "Filter by domain ID") @QueryParam("domainId") String domainId, String csv) { + String normalizedCsv = CsvUtil.stripUtf8Bom(csv); CsvImportResult result = repository.importColumnsCSV( uriInfo, securityContext, - csv, + normalizedCsv, dryRun, entityTypes, serviceName, @@ -459,7 +461,7 @@ public Response importUniqueColumns( @POST @Path("/import-async") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Operation( operationId = "importUniqueColumnsAsync", summary = "Import column metadata from CSV asynchronously", @@ -492,6 +494,7 @@ public Response importUniqueColumnsAsync( @Parameter(description = "Filter by domain ID") @QueryParam("domainId") String domainId, String csv) { + String normalizedCsv = CsvUtil.stripUtf8Bom(csv); String jobId = UUID.randomUUID().toString(); CSVImportResponse responseEntity = new CSVImportResponse(jobId, "CSV column import is in progress."); @@ -508,7 +511,7 @@ public Response importUniqueColumnsAsync( repository.importColumnsCSV( uriInfo, securityContext, - csv, + normalizedCsv, false, entityTypes, serviceName, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index fea914d1726f..1fa118d5397d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -582,7 +582,7 @@ public Response patch( @GET @Path("/name/{name}/exportAsync") - @Produces(MediaType.TEXT_PLAIN) + @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( operationId = "exportTable", @@ -606,7 +606,7 @@ public Response exportCsvAsync( @GET @Path("/name/{name}/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({"text/csv; charset=UTF-8"}) @Valid @Operation( operationId = "exportTable", @@ -617,7 +617,7 @@ public Response exportCsvAsync( description = "Exported csv with columns from the table", content = @Content( - mediaType = "application/json", + mediaType = "text/csv; charset=UTF-8", schema = @Schema(implementation = String.class))) }) public String exportCsv( @@ -631,7 +631,7 @@ public String exportCsv( @PUT @Path("/name/{name}/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importTable", @@ -665,7 +665,7 @@ public CsvImportResult importCsv( @PUT @Path("/name/{name}/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importTableAsync", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index f5140b493459..7fa6cc601fda 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -1252,7 +1252,7 @@ public Response addManyTestCasesToBundleTestSuite( @GET @Path("/name/{name}/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({"text/csv; charset=UTF-8"}) @Valid @Operation( operationId = "exportTestCases", @@ -1267,7 +1267,9 @@ public Response addManyTestCasesToBundleTestSuite( responseCode = "200", description = "Exported CSV with test cases", content = - @Content(mediaType = "text/plain", schema = @Schema(implementation = String.class))) + @Content( + mediaType = "text/csv; charset=UTF-8", + schema = @Schema(implementation = String.class))) }) public String exportCsv( @Context SecurityContext securityContext, @@ -1315,7 +1317,7 @@ public Response exportCsvAsync( @PUT @Path("/name/{name}/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( @@ -1360,7 +1362,7 @@ public CsvImportResult importCsv( @PUT @Path("/name/{name}/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java index 57b8c03e391f..92c6d890150f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java @@ -536,7 +536,7 @@ public String getCsvDocumentation(@Context SecurityContext securityContext) { @GET @Path("/name/{name}/exportAsync") - @Produces(MediaType.TEXT_PLAIN) + @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( operationId = "exportGlossary", @@ -560,7 +560,7 @@ public Response exportCsvAsync( @GET @Path("/name/{name}/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "exportGlossary", @@ -571,7 +571,7 @@ public Response exportCsvAsync( description = "Exported csv with glossary terms", content = @Content( - mediaType = "application/json", + mediaType = "text/plain; charset=UTF-8", schema = @Schema(implementation = String.class))) }) public String exportCsv( @@ -585,7 +585,7 @@ public String exportCsv( @PUT @Path("/name/{name}/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importGlossary", @@ -619,7 +619,7 @@ public CsvImportResult importCsv( @PUT @Path("/name/{name}/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index e7e2eef75782..259943eb4470 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -1217,7 +1217,7 @@ public Response getTermRelationGraph( @GET @Path("/name/{fqn}/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({"text/csv; charset=UTF-8"}) @Valid @Operation( operationId = "exportGlossaryTerm", @@ -1227,7 +1227,7 @@ public Response getTermRelationGraph( @ApiResponse( responseCode = "200", description = "Exported csv with glossary terms", - content = @Content(mediaType = "text/plain")) + content = @Content(mediaType = "text/csv; charset=UTF-8")) }) public String exportCsv( @Context SecurityContext securityContext, @@ -1242,7 +1242,7 @@ public String exportCsv( @GET @Path("/name/{fqn}/exportAsync") - @Produces(MediaType.TEXT_PLAIN) + @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( operationId = "exportGlossaryTermAsync", @@ -1272,7 +1272,7 @@ public Response exportCsvAsync( @PUT @Path("/name/{fqn}/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importGlossaryTerm", @@ -1310,7 +1310,7 @@ public CsvImportResult importCsv( @PUT @Path("/name/{fqn}/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index 221ecec01627..a5bfbcf15f34 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -401,7 +401,7 @@ public Response searchDataQualityLineage( @GET @Path("/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({"text/csv; charset=UTF-8"}) @Operation( operationId = "exportLineage", summary = "Export lineage", @@ -411,8 +411,8 @@ public Response searchDataQualityLineage( description = "search response", content = @Content( - mediaType = "application/json", - schema = @Schema(implementation = SearchResponse.class))) + mediaType = "text/csv; charset=UTF-8", + schema = @Schema(implementation = String.class))) }) public String exportLineage( @Context UriInfo uriInfo, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java index c5143f25fa32..68cccd25691d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java @@ -719,7 +719,7 @@ public String getCsvDocumentation(@Context SecurityContext securityContext) { @GET @Path("/name/{name}/exportAsync") - @Produces(MediaType.TEXT_PLAIN) + @Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "exportTeams", @@ -740,7 +740,7 @@ public Response exportCsvAsync( @GET @Path("/name/{name}/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "exportTeams", @@ -751,7 +751,7 @@ public Response exportCsvAsync( description = "Exported csv with teams information", content = @Content( - mediaType = "application/json", + mediaType = "text/plain; charset=UTF-8", schema = @Schema(implementation = String.class))) }) public String exportCsv(@Context SecurityContext securityContext, @PathParam("name") String name) @@ -761,7 +761,7 @@ public String exportCsv(@Context SecurityContext securityContext, @PathParam("na @PUT @Path("/name/{name}/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importTeams", @@ -854,7 +854,7 @@ public Response deleteTeamUser( @PUT @Path("/name/{name}/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java index c1610ad70230..0446410c23b1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java @@ -1685,7 +1685,7 @@ public Response exportUsersCsvAsync( @GET @Path("/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "exportUsers", @@ -1696,7 +1696,7 @@ public Response exportUsersCsvAsync( description = "Exported csv with user information", content = @Content( - mediaType = "application/json", + mediaType = "text/plain; charset=UTF-8", schema = @Schema(implementation = String.class))) }) public String exportUsersCsv( @@ -1713,11 +1713,11 @@ public String exportUsersCsv( @PUT @Path("/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( - operationId = "importTeams", - summary = "Import from CSV to create, and update teams.", + operationId = "importUsers", + summary = "Import from CSV to create, and update users.", responses = { @ApiResponse( responseCode = "200", @@ -1750,11 +1750,11 @@ public CsvImportResult importCsv( @PUT @Path("/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( - operationId = "importTeamsAsync", - summary = "Import from CSV to create, and update teams asynchronously.", + operationId = "importUsersAsync", + summary = "Import from CSV to create, and update users asynchronously.", responses = { @ApiResponse( responseCode = "200", diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java index 74b2b8d8a66e..6e96f66fbad6 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java @@ -31,6 +31,8 @@ import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.csv.CsvFile; +import org.openmetadata.schema.type.csv.CsvHeader; public class CsvUtilTest { @Test @@ -257,4 +259,31 @@ public static void assertCsv(String expectedCsv, String actualCsv) { assertEquals(expectedCsvRecords.get(i), actualCsvRecords.get(i)); } } + + @Test + void testFormatCsvPreservesChineseCharacters() throws Exception { + CsvFile csvFile = + new CsvFile() + .withHeaders( + List.of( + new CsvHeader().withName("name"), + new CsvHeader().withName("description"), + new CsvHeader().withName("owner"))) + .withRecords(List.of(List.of("中文表", "这是中文描述", "数据平台团队"))); + + String csv = CsvUtil.formatCsv(csvFile); + + assertTrue(csv.contains("中文表")); + assertTrue(csv.contains("这是中文描述")); + assertTrue(csv.contains("数据平台团队")); + } + + @Test + void testStripUtf8Bom() { + String csv = "name,description\n中文表,中文描述"; + assertEquals(csv, CsvUtil.stripUtf8Bom(CsvUtil.UTF8_BOM + csv)); + assertEquals(csv, CsvUtil.stripUtf8Bom(csv)); + assertEquals("", CsvUtil.stripUtf8Bom("")); + assertNull(CsvUtil.stripUtf8Bom(null)); + } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts index 92ba4fd5b490..6b82323c67eb 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts @@ -63,6 +63,13 @@ const propertiesList = Object.values(CUSTOM_PROPERTIES_TYPES); const propertyListName: Record = {}; const additionalGlossaryTerm = createGlossaryTermRowDetails(); +const chineseGlossaryTermDetails = { + name: `术语${uuid()}`, + displayName: '中文术语展示名', + description: '这是用于验证导入导出编码的中文描述。', + synonyms: '中文同义词;测试', + references: '参考;https://example.com/%E4%B8%AD%E6%96%87', +}; test.describe('Glossary Bulk Import Export', () => { test.slow(true); @@ -183,6 +190,7 @@ test.describe('Glossary Bulk Import Export', () => { await fillGlossaryRowDetails( { ...additionalGlossaryTerm, + ...chineseGlossaryTermDetails, owners: [user1.responseData?.['displayName']], reviewers: [user2.responseData?.['displayName']], relatedTerm: { @@ -214,6 +222,9 @@ test.describe('Glossary Bulk Import Export', () => { const rowStatus = ['Entity updated', 'Entity created']; await expect(page.locator('.rdg-cell-details')).toHaveText(rowStatus); + await expect( + page.getByText(chineseGlossaryTermDetails.name) + ).toBeVisible(); await page.getByRole('button', { name: 'Update' }).click(); await page diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts index 835d27fb0772..a267d7394e9b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts @@ -21,5 +21,5 @@ export interface UploadFileProps { file: RcFile, FileList: RcFile[] ) => BeforeUploadValueType | Promise; - onCSVUploaded: (event: ProgressEvent) => void; + onCSVUploaded: (event: ProgressEvent) => void | Promise; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx index b9a69089435f..2d61b3759839 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx @@ -45,8 +45,28 @@ const UploadFile: FC = ({ showErrorToast(new Error(t('server.unexpected-error')) as AxiosError); }; try { +<<<<<<< HEAD + const reader = new FileReader(); + reader.onload = async (event) => { + try { + await onCSVUploaded(event); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setUploading(false); + } + }; + reader.onerror = () => { + showErrorToast(reader.error?.message ?? t('server.unexpected-error')); + setUploading(false); + }; + reader.readAsText(options.file as Blob, 'utf-8'); + } catch (error) { + showErrorToast(error as AxiosError); +======= reader.readAsText(options.file as Blob); } catch (error) { +>>>>>>> origin/main setUploading(false); showErrorToast(error as AxiosError); } @@ -66,24 +86,25 @@ const UploadFile: FC = ({ disabled={disabled} multiple={false} showUploadList={false}> - - - - } - values={{ - text: t('label.browse'), - }} - /> - - - - ); + try { + const reader = new FileReader(); + reader.onload = async (event) => { + try { + await onCSVUploaded(event); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setUploading(false); + } + }; + reader.onerror = () => { + showErrorToast(reader.error?.message ?? t('server.unexpected-error')); + setUploading(false); + }; + reader.readAsText(options.file as Blob, 'utf-8'); + } catch (error) { + setUploading(false); + showErrorToast(error as AxiosError); }; export default UploadFile; diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts index 55435953b565..19a87e0f0377 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts @@ -166,7 +166,7 @@ export const importColumnsCSV = async ( `/columns/import?${queryParams.toString()}`, params.csv, { - headers: { 'Content-Type': 'text/plain' }, + headers: { 'Content-Type': 'text/plain; charset=UTF-8' }, } ); @@ -198,7 +198,7 @@ export const importColumnsCSVAsync = async ( string, AxiosResponse >(`/columns/import-async?${queryParams.toString()}`, params.csv, { - headers: { 'Content-Type': 'text/plain' }, + headers: { 'Content-Type': 'text/plain; charset=UTF-8' }, }); return response.data; diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts index 0f0a570e46b2..6320d109e413 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts @@ -307,7 +307,7 @@ export const importDatabaseInCSVFormat = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put( `/databases/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`, @@ -340,7 +340,7 @@ export const importDatabaseSchemaInCSVFormat = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put( `/databaseSchemas/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts index 4dde904bd6ef..b5ce076e950e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts @@ -50,7 +50,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -78,7 +78,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=false&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -105,7 +105,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=true&recursive=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -132,7 +132,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.stringContaining('/dataQuality/testCases/name/'), mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -177,7 +177,7 @@ describe('importExportAPI tests', () => { }); expect(mockPut).toHaveBeenCalledWith(expect.any(String), '', { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }); }); }); @@ -205,7 +205,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/tables/name/database.schema.table/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -232,7 +232,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/databases/name/service.database/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -258,7 +258,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/databaseSchemas/name/service.database.schema/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -286,7 +286,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/tables/name/database.schema.table/importAsync?dryRun=false&recursive=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -312,7 +312,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.any(String), expect.any(String), - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -361,7 +361,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( 'services/databaseServices/name/my-database-service/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -388,7 +388,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( 'services/messagingServices/name/my-messaging-service/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -416,7 +416,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( 'services/databaseServices/name/my-service/importAsync?dryRun=false&recursive=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -465,7 +465,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaries/name/my-glossary/importAsync?dryRun=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -493,7 +493,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaries/name/my-glossary/importAsync?dryRun=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -547,7 +547,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.stringContaining('/glossaries/name/'), mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -596,7 +596,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaryTerms/name/glossary.term/importAsync?dryRun=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -624,7 +624,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaryTerms/name/glossary.term/importAsync?dryRun=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -678,7 +678,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.stringContaining('/glossaryTerms/name/'), mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -724,7 +724,7 @@ describe('importExportAPI tests', () => { }); expect(mockPut).toHaveBeenCalledWith(expect.any(String), largeCsvData, { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts index 10714ae70a6e..0d8e8e8e13d8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts @@ -33,7 +33,7 @@ export const importTestCaseInCSVFormat = async ({ targetEntityType, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; let url = `/dataQuality/testCases/name/${getEncodedFqn( name @@ -58,7 +58,7 @@ export const importEntityInCSVFormat = async ({ recursive = false, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put< string, @@ -82,7 +82,7 @@ export const importServiceInCSVFormat = async ({ recursive = false, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put< string, @@ -104,7 +104,7 @@ export const importGlossaryInCSVFormat = async ({ dryRun = true, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const response = await APIClient.put< string, @@ -124,7 +124,7 @@ export const importGlossaryTermInCSVFormat = async ({ dryRun = true, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const response = await APIClient.put< string, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts index c66fff886616..265450e79e30 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts @@ -264,7 +264,7 @@ export const importTableInCSVFormat = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put( `/tables/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts index a8c9b75c7720..7b85d6910b32 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts @@ -130,7 +130,7 @@ export const importTeam = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, params: { dryRun, }, @@ -149,7 +149,7 @@ export const importUserInTeam = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, params: { team, dryRun, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx index 2a2146f577fc..0919b247112d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx @@ -29,6 +29,10 @@ jest.mock('../ToastUtils', () => ({ })); describe('ExportUtils', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + describe('downloadFile', () => { const mockLink = { href: '', @@ -36,6 +40,9 @@ describe('ExportUtils', () => { style: { visibility: '' }, click: jest.fn(), }; + const originalBlob = global.Blob; + const originalCreateObjectURL = global.URL.createObjectURL; + const originalRevokeObjectURL = global.URL.revokeObjectURL; let mockCreateObjectURL: jest.Mock; let mockRevokeObjectURL: jest.Mock; @@ -57,7 +64,9 @@ describe('ExportUtils', () => { }); afterEach(() => { - jest.restoreAllMocks(); + global.Blob = originalBlob; + global.URL.createObjectURL = originalCreateObjectURL; + global.URL.revokeObjectURL = originalRevokeObjectURL; }); it('creates an anchor element and triggers a click', () => { @@ -92,15 +101,39 @@ describe('ExportUtils', () => { expect(mockRevokeObjectURL).toHaveBeenCalledWith('blob:mock-url'); }); - it('uses the provided mimeType when creating the Blob', () => { + it('normalizes CSV mimeType and prepends BOM when creating the Blob', () => { const mockBlob = {}; const MockBlob = jest.fn().mockReturnValue(mockBlob); global.Blob = MockBlob as unknown as typeof Blob; downloadFile('content', 'file.csv', 'text/csv;charset=utf-8;'); + expect(MockBlob).toHaveBeenCalledWith(['\uFEFFcontent'], { + type: 'text/csv; charset=utf-8', + }); + }); + + it('does not prepend a duplicate BOM when content already has one', () => { + const mockBlob = {}; + const MockBlob = jest.fn().mockReturnValue(mockBlob); + global.Blob = MockBlob as unknown as typeof Blob; + + downloadFile('\uFEFFcontent', 'file.csv', 'text/csv;charset=utf-8;'); + + expect(MockBlob).toHaveBeenCalledWith(['\uFEFFcontent'], { + type: 'text/csv; charset=utf-8', + }); + }); + + it('does not prepend BOM for non-csv files', () => { + const mockBlob = {}; + const MockBlob = jest.fn().mockReturnValue(mockBlob); + global.Blob = MockBlob as unknown as typeof Blob; + + downloadFile('content', 'file.txt'); + expect(MockBlob).toHaveBeenCalledWith(['content'], { - type: 'text/csv;charset=utf-8;', + type: 'text/plain', }); }); }); @@ -121,10 +154,6 @@ describe('ExportUtils', () => { } as unknown as HTMLAnchorElement); }); - afterEach(() => { - mockCreateElement.mockRestore(); - }); - it('should create and trigger download with correct attributes', () => { const dataUrl = 'data:image/png;base64,test'; const fileName = 'test-image'; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts index e7bbe3e57d1a..ad2509336db9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts @@ -23,7 +23,17 @@ export const downloadFile = ( fileName: string, mimeType: string = 'text/plain' ): void => { - const blob = new Blob([content], { type: mimeType }); + const isCsvFile = fileName.toLowerCase().endsWith('.csv'); + const isCsvMime = mimeType.toLowerCase().includes('text/csv'); + const csvMimeType = 'text/csv; charset=utf-8'; + const effectiveMimeType = isCsvFile || isCsvMime ? csvMimeType : mimeType; + const effectiveContent = + isCsvFile || isCsvMime + ? content.startsWith('\uFEFF') + ? content + : `\uFEFF${content}` + : content; + const blob = new Blob([effectiveContent], { type: effectiveMimeType }); const link = document.createElement('a'); link.href = URL.createObjectURL(blob);