Merge branch 'master' into fix-vulkan-iGPU-llvmpipe-selection

Author: TheMostDiligent

.github/workflows/build-apple.yml

@@ -61,6 +61,14 @@ jobs:
       with:
         platform: ${{ matrix.platform }}
 
+    - name: Test Build Scripts
+      if: success()
+      shell: bash
+      run: |
+        # Test File2Include utility
+        cd "${{ github.workspace }}/BuildTools/File2Include"
+        python -m unittest tests.py
+
     - name: Configure CMake
       if:   success()
       uses: DiligentGraphics/github-action/configure-cmake@v10

.github/workflows/build-linux.yml

@@ -90,6 +90,14 @@ jobs:
         platform:        Linux
         cmake-generator: ${{ matrix.cmake_generator }}
 
+    - name: Test Build Scripts
+      if: success()
+      shell: bash
+      run: |
+        # Test File2Include utility
+        cd "${{ github.workspace }}/BuildTools/File2Include"
+        python -m unittest tests.py
+
     - name: Configure CMake
       if:   success()
       uses: DiligentGraphics/github-action/configure-cmake@v10

.github/workflows/build-windows.yml

@@ -83,6 +83,14 @@ jobs:
         cmake-generator: ${{ matrix.cmake_generator }}
         ninja-vs-arch:   ${{ matrix.toolset }}
 
+    - name: Test Build Scripts
+      if: success()
+      shell: bash
+      run: |
+        # Test File2Include utility
+        cd "${{ github.workspace }}/BuildTools/File2Include"
+        python -m unittest tests.py
+
     - name: Configure CMake
       if: success()
       uses: DiligentGraphics/github-action/configure-cmake@v10

BuildTools/File2Include/script.py

@@ -41,25 +41,37 @@ def strip_c_comments(text):
     line_has_comment = False
     line_has_noncomment_code = False
 
+    block_comment_buf = []
+    line_comment_buf = []
+
     def append_char(c):
         nonlocal line_has_noncomment_code
         current_line.append(c)
         if not c.isspace():
             line_has_noncomment_code = True
 
+    def append_text(s):
+        for ch in s:
+            append_char(ch)
+
+    def should_preserve_block_comment(comment_text):
+        inner = comment_text[2:-2].strip()
+        return inner.startswith("format")
+
+    def should_preserve_line_comment(comment_text):
+        inner = comment_text[2:].strip()
+        return inner.startswith("format")
+
     def flush_line(add_newline):
         nonlocal current_line, line_has_comment, line_has_noncomment_code
 
-        # Remove lines that contain only a comment (possibly with whitespace around it)
         if not (line_has_comment and not line_has_noncomment_code and "".join(current_line).strip() == ""):
             result.extend(current_line)
             if add_newline:
                 result.append("\n")
 
         current_line = []
         line_has_noncomment_code = False
-
-        # If we are still inside a block comment, the next physical line is also comment-only so far
         line_has_comment = in_block_comment
 
     i = 0
@@ -71,19 +83,36 @@ def flush_line(add_newline):
 
         if in_line_comment:
             if c == "\n":
+                comment_text = "".join(line_comment_buf)
+                if should_preserve_line_comment(comment_text):
+                    append_text(comment_text)
+
+                line_comment_buf = []
                 in_line_comment = False
                 flush_line(True)
+                i += 1
+                continue
+
+            line_comment_buf.append(c)
             i += 1
             continue
 
         if in_block_comment:
+            block_comment_buf.append(c)
+
             if c == "*" and nxt == "/":
+                block_comment_buf.append("/")
+                comment_text = "".join(block_comment_buf)
+
                 in_block_comment = False
+                block_comment_buf = []
+
+                if should_preserve_block_comment(comment_text):
+                    append_text(comment_text)
+
                 i += 2
                 continue
 
-            if c == "\n":
-                flush_line(True)
             i += 1
             continue
 
@@ -112,12 +141,14 @@ def flush_line(add_newline):
         if c == "/" and nxt == "/":
             line_has_comment = True
             in_line_comment = True
+            line_comment_buf = ["//"]
             i += 2
             continue
 
         if c == "/" and nxt == "*":
             line_has_comment = True
             in_block_comment = True
+            block_comment_buf = ["/*"]
             i += 2
             continue
 
@@ -141,13 +172,42 @@ def flush_line(add_newline):
         append_char(c)
         i += 1
 
+    if in_line_comment:
+        comment_text = "".join(line_comment_buf)
+        if should_preserve_line_comment(comment_text):
+            append_text(comment_text)
+
     if current_line:
         if not (line_has_comment and not line_has_noncomment_code and "".join(current_line).strip() == ""):
             result.extend(current_line)
 
     return "".join(result)
 
 
+def prepare_content(text, strip_comments=False):
+    if strip_comments:
+        return strip_c_comments(text)
+    return text
+
+
+def convert_to_string_lines(text):
+    special_chars = "'\"\\"
+    output = []
+
+    for line in text.splitlines():
+        quoted = ['"']
+
+        for c in line.rstrip():
+            if c in special_chars:
+                quoted.append("\\")
+            quoted.append(c)
+
+        quoted.append('\\n"\n')
+        output.append("".join(quoted))
+
+    return "".join(output)
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Convert a file into quoted string lines."
@@ -167,23 +227,9 @@ def main():
             raise ValueError("Source and destination files must be different")
 
         with open(args.src, "r") as src_file, open(args.dst, "w") as dst_file:
-            special_chars = "\'\"\\"
-
-            if args.strip_comments:
-                content = strip_c_comments(src_file.read())
-                src_iter = io.StringIO(content)
-            else:
-                src_iter = src_file
-
-            for line in src_iter:
-                dst_file.write('\"')
-
-                for i, c in enumerate(line.rstrip()):
-                    if special_chars.find(c) != -1:
-                        dst_file.write('\\')
-                    dst_file.write(c)
-
-                dst_file.write('\\n\"\n')
+            content = src_file.read()
+            content = prepare_content(content, args.strip_comments)
+            dst_file.write(convert_to_string_lines(content))
 
         print("File2String: successfully converted {} to {}".format(args.src, args.dst))
 

BuildTools/File2Include/tests.py

@@ -0,0 +1,157 @@
+import unittest
+
+from script import strip_c_comments, convert_to_string_lines
+
+
+class StripCCommentsTests(unittest.TestCase):
+    def test_keeps_plain_code(self):
+        src = "int x = 1;\nfloat y = 2;\n"
+        self.assertEqual(strip_c_comments(src), src)
+
+    def test_removes_line_comment(self):
+        src = "int x = 1; // comment\n"
+        self.assertEqual(strip_c_comments(src), "int x = 1; \n")
+
+    def test_removes_block_comment(self):
+        src = "int /* comment */ x = 1;\n"
+        self.assertEqual(strip_c_comments(src), "int  x = 1;\n")
+
+    def test_removes_multiline_block_comment(self):
+        src = "int x = 1; /* comment\nstill comment */\nfloat y = 2;\n"
+        self.assertEqual(strip_c_comments(src), "int x = 1; \nfloat y = 2;\n")
+
+    def test_removes_comment_only_line(self):
+        src = "int x = 1;\n// comment only\nfloat y = 2;\n"
+        self.assertEqual(strip_c_comments(src), "int x = 1;\nfloat y = 2;\n")
+
+    def test_removes_whitespace_and_comment_only_line(self):
+        src = "int x = 1;\n   // comment only\nfloat y = 2;\n"
+        self.assertEqual(strip_c_comments(src), "int x = 1;\nfloat y = 2;\n")
+
+    def test_preserves_comment_like_text_in_string(self):
+        src = 'const char* s = "not // comment and not /* comment */";\n'
+        self.assertEqual(strip_c_comments(src), src)
+
+    def test_preserves_comment_like_text_in_char_literal(self):
+        src = "char c = '/'; // real comment\n"
+        self.assertEqual(strip_c_comments(src), "char c = '/'; \n")
+
+    def test_preserves_escaped_quotes_in_string(self):
+        src = 'const char* s = "\\"quoted\\" // still string"; // comment\n'
+        expected = 'const char* s = "\\"quoted\\" // still string"; \n'
+        self.assertEqual(strip_c_comments(src), expected)
+
+    def test_preserves_format_block_comment(self):
+        src = "RWTexture2D<float /* format = r32f */ >\n"
+        self.assertEqual(strip_c_comments(src), src)
+
+    def test_preserves_format_line_comment(self):
+        src = "RWTexture2D<float // format = r32f\n>\n"
+        self.assertEqual(strip_c_comments(src), src)
+
+    def test_mixed_preserved_and_normal_comments(self):
+        src = (
+            "RWTexture2D<float /* format = r32f */ > // remove this\n"
+            "int x = 1; /* remove this */\n"
+        )
+        expected = (
+            "RWTexture2D<float /* format = r32f */ > \n"
+            "int x = 1; \n"
+        )
+        self.assertEqual(strip_c_comments(src), expected)
+
+    def test_escaped_backslash_at_end_of_string(self):
+        # A backslash escaping another backslash shouldn't escape the quote
+        src = 'const char* s = "backslash\\\\"; // comment\n'
+        expected = 'const char* s = "backslash\\\\"; \n'
+        self.assertEqual(strip_c_comments(src), expected)
+
+    def test_comment_start_inside_char_literal(self):
+        # Testing if it handles /* inside a character literal correctly
+        src = "char a = '/'; char b = '*';\n"
+        self.assertEqual(strip_c_comments(src), src)
+
+    def test_block_comment_preserved_with_internal_stars(self):
+        # Ensure the 'format' check doesn't fail if there are internal decorators
+        src = "/* format: r32f ******* */\n"
+        self.assertEqual(strip_c_comments(src), src)
+        
+    def test_multiple_block_comments_one_line(self):
+        src = "/* remove */ code(); /* format: preserve */\n"
+        expected = " code(); /* format: preserve */\n"
+        self.assertEqual(strip_c_comments(src), expected)
+
+    def test_multiline_block_comment_inline_does_not_insert_newline(self):
+        src = "x /* comment\nstill comment */ y\n"
+        self.assertEqual(strip_c_comments(src), "x  y\n")
+
+    def test_multiline_preserved_block_comment_stays_inline(self):
+        src = "x /* format = r32f\nstill format */ y\n"
+        self.assertEqual(
+            strip_c_comments(src),
+            "x /* format = r32f\nstill format */ y\n",
+        )
+
+    def test_removes_line_comment_at_eof(self):
+        src = "int x = 1; // comment"
+        self.assertEqual(strip_c_comments(src), "int x = 1; ")
+
+    def test_preserves_format_line_comment_at_eof(self):
+        src = "x // format = r32f"
+        self.assertEqual(strip_c_comments(src), "x // format = r32f")
+
+class ConvertToStringLinesTests(unittest.TestCase):
+    def test_converts_single_line(self):
+        src = "hello\n"
+        self.assertEqual(convert_to_string_lines(src), '"hello\\n"\n')
+
+    def test_converts_multiple_lines(self):
+        src = "hello\nworld\n"
+        self.assertEqual(convert_to_string_lines(src), '"hello\\n"\n"world\\n"\n')
+
+    def test_escapes_double_quotes(self):
+        src = '"abc"\n'
+        self.assertEqual(convert_to_string_lines(src), '"\\"abc\\"\\n"\n')
+
+    def test_escapes_single_quotes(self):
+        src = "'a'\n"
+        self.assertEqual(convert_to_string_lines(src), '"\\\'a\\\'\\n"\n')
+
+    def test_escapes_backslashes(self):
+        src = "C:\\temp\\file\n"
+        self.assertEqual(convert_to_string_lines(src), '"C:\\\\temp\\\\file\\n"\n')
+
+    def test_preserves_empty_line(self):
+        src = "\n"
+        self.assertEqual(convert_to_string_lines(src), '"\\n"\n')
+
+    def test_handles_text_without_trailing_newline(self):
+        src = "hello"
+        self.assertEqual(convert_to_string_lines(src), '"hello\\n"\n')
+
+    def test_handles_text_with_trailing_spaces(self):
+        src = "hello   "
+        self.assertEqual(convert_to_string_lines(src), '"hello\\n"\n')
+
+    def test_strip_then_convert_integration(self):
+        src = 'int x = 1; // comment\nconst char* s = "a";\n'
+        stripped = strip_c_comments(src)
+        expected = (
+            '"int x = 1;\\n"\n'
+            '"const char* s = \\"a\\";\\n"\n'
+        )
+        self.assertEqual(convert_to_string_lines(stripped), expected)
+
+    def test_empty_input(self):
+        self.assertEqual(convert_to_string_lines(""), "")
+
+    def test_preserves_blank_line_in_middle(self):
+        src = "a\n\nb\n"
+        self.assertEqual(convert_to_string_lines(src), '"a\\n"\n"\\n"\n"b\\n"\n')
+
+    def test_trims_trailing_tabs_too(self):
+        src = "hello\t\t\n"
+        self.assertEqual(convert_to_string_lines(src), '"hello\\n"\n')
+
+if __name__ == "__main__":
+    unittest.main()