Merge pull request #6231 from hjmjohnson/windows-utf8-active-codepage

ENH: Embed UTF-8 active-code-page manifest in Windows test executables (supersedes #4390)

Author: hjmjohnson

CMake/ITKModuleTest.cmake

@@ -90,6 +90,7 @@ EM_ASM(
     PRIVATE
       "$<$<AND:$<C_COMPILER_ID:AppleClang>,$<VERSION_GREATER_EQUAL:$<C_COMPILER_VERSION>,15.0>>:LINKER:-no_warn_duplicate_libraries>"
   )
+  itk_target_attach_windows_utf8_manifest(${KIT}TestDriver)
   itk_module_target_label(${KIT}TestDriver)
 endfunction()
 
@@ -286,6 +287,7 @@ function(CreateGoogleTestDriver KIT KIT_LIBS KitTests)
     PRIVATE
       "$<$<AND:$<C_COMPILER_ID:AppleClang>,$<VERSION_GREATER_EQUAL:$<C_COMPILER_VERSION>,15.0>>:LINKER:-no_warn_duplicate_libraries>"
   )
+  itk_target_attach_windows_utf8_manifest(${exe})
   itk_module_target_label(${exe})
 
   include(GoogleTest)

CMake/ITKWindowsUtf8.cmake

@@ -0,0 +1,16 @@
+# Attach the UTF-8 active-code-page manifest to a Windows executable so its
+# narrow-char Win32 APIs (CreateFileA, fopen, std::ifstream(const char*), ...)
+# treat byte strings as UTF-8 on Windows 10 1903 and later. No-op on
+# non-MSVC toolchains and on platforms where the manifest is irrelevant.
+#
+# Usage: itk_target_attach_windows_utf8_manifest(<target>)
+set(
+  _itk_windows_utf8_manifest
+  "${CMAKE_CURRENT_LIST_DIR}/Windows-utf8-codepage.manifest"
+)
+
+function(itk_target_attach_windows_utf8_manifest _target)
+  if(MSVC)
+    target_sources(${_target} PRIVATE "${_itk_windows_utf8_manifest}")
+  endif()
+endfunction()

CMake/Windows-utf8-codepage.manifest

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
+</assembly>

CMakeLists.txt

@@ -180,6 +180,7 @@ include(PreventInSourceBuilds)
 include(PreventInBuildInstalls)
 include(ITKModuleMacros)
 include(ITKExternalData)
+include(ITKWindowsUtf8)
 include(ITKModuleTest)
 include(itkCheckSourceTree)
 

Modules/Core/TestKernel/include/itkTestDriverBeforeTest.inc

@@ -19,6 +19,12 @@
 // CMake command create_test_sourcelist. This is included directly before
 // the call to the test main function.
 
+#ifdef _WIN32
+  // Render UTF-8 console output correctly. The active code page itself
+  // is set to UTF-8 by the Windows-utf8-codepage.manifest attached at
+  // link time; this call only affects what stdout / stderr render.
+  SetConsoleOutputCP(CP_UTF8);
+#endif
   vnl_sample_reseed(8775070);
   try
     {

Modules/Core/TestKernel/include/itkTestDriverInclude.h

@@ -52,6 +52,10 @@
 #include "itkIntTypes.h"
 #include "itkWin32Header.h"
 
+#ifdef _WIN32
+#  include <windows.h>
+#endif
+
 constexpr int ITK_TEST_DIMENSION_MAX{ 6 };
 
 extern int

Modules/Core/TestKernel/src/CMakeLists.txt

@@ -12,6 +12,7 @@ if(NOT DO_NOT_BUILD_ITK_TEST_DRIVER)
   add_executable(itkTestDriver itkTestDriver.cxx)
   target_link_libraries(itkTestDriver PRIVATE ${ITKTestKernel_LIBRARIES})
 
+  itk_target_attach_windows_utf8_manifest(itkTestDriver)
   itk_module_target_label(itkTestDriver)
   itk_module_target_export(itkTestDriver)
   target_link_options(

Modules/Core/TestKernel/src/itkTestDriver.cxx

@@ -263,6 +263,9 @@ TestDriverInvokeProcess(const ArgumentsList & args)
 int
 main(int argc, char * argv[])
 {
+#ifdef _WIN32
+  SetConsoleOutputCP(CP_UTF8);
+#endif
   try
   {
     RegisterRequiredFactories();

Modules/Core/TestKernel/test/CMakeLists.txt

@@ -49,6 +49,7 @@ set(
   itkGoogleTest.cxx
   itkGoogleTestFixture.cxx
   itkTestingComparisonImageFilterGTest.cxx
+  itkUtf8FilenameGTest.cxx
 )
 
 creategoogletestdriver(ITKTestKernel "${ITKTestKernel-Test_LIBRARIES}" "${ITKTestKernelGTests}")

Modules/Core/TestKernel/test/itkUtf8FilenameGTest.cxx

@@ -0,0 +1,76 @@
+/*=========================================================================
+ *
+ *  Copyright NumFOCUS
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *         https://www.apache.org/licenses/LICENSE-2.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *=========================================================================*/
+
+// Round-trip a small file at a path that contains non-ASCII (UTF-8)
+// characters. On Windows, the Windows-utf8-codepage.manifest attached
+// to every ITK test driver makes the narrow Win32 *A APIs treat byte
+// strings as UTF-8 (Windows 10 1903 and later), so std::ofstream and
+// std::ifstream constructed from a UTF-8 const char * "just work". On
+// POSIX platforms the encoding is byte-transparent.
+
+#include <cstdio>
+#include <fstream>
+#include <string>
+
+#include "gtest/gtest.h"
+
+namespace
+{
+
+// "speciäl-fil👍.txt" encoded as UTF-8 bytes.
+const char *
+SpecialBasename()
+{
+  return "speci\xC3\xA4l-fil\xF0\x9F\x91\x8D.txt";
+}
+
+std::string
+TempDirFromEnv()
+{
+  for (const char * var : { "ITK_TEST_OUTPUT_DIR", "TEMP", "TMP", "TMPDIR" })
+  {
+    if (const char * envDir = std::getenv(var))
+    {
+      return envDir;
+    }
+  }
+  return ".";
+}
+
+} // namespace
+
+TEST(WindowsUtf8Codepage, RoundTripFileAtNonAsciiPath)
+{
+  const std::string path = TempDirFromEnv() + "/" + SpecialBasename();
+  const std::string payload = "round-trip payload\n";
+
+  {
+    std::ofstream out(path.c_str());
+    ASSERT_TRUE(out.is_open()) << "failed to open " << path << " for writing";
+    out << payload;
+  }
+
+  std::ifstream     in(path.c_str());
+  const bool        readOpened = in.is_open();
+  const std::string actual =
+    readOpened ? std::string((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>()) : std::string{};
+  std::remove(path.c_str());
+
+  ASSERT_TRUE(readOpened) << "failed to open " << path << " for reading";
+  EXPECT_EQ(actual, payload);
+}