Snow-2091473: converter doesnt properly convert lists of bytes (#2734)

Co-authored-by: Jeff Sawatzky <jeffsawatzky@users.noreply.github.com>

Author: sfc-gh-fpawlowski

DESCRIPTION.md

@@ -8,6 +8,7 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne
 
 # Release Notes
 - v4.2.1(TBD)
+  - Ensured proper list conversion - the converter runs to_snowflake on all list elements.
   - Made the parameter `server_session_keep_alive` in `SnowflakeConnection` skip checking for pending async queries, providing faster connection close times especially when many async queries are executed.
   - Fix string representation of INTERVAL YEAR and INTERVAL MONTH types.
 

src/snowflake/connector/converter.py

@@ -648,7 +648,7 @@ def _decimal_to_snowflake(self, value: decimal.Decimal) -> str | None:
     def _list_to_snowflake(self, value: list) -> list:
         return [
             SnowflakeConverter.quote(v0)
-            for v0 in [SnowflakeConverter.escape(v) for v in value]
+            for v0 in [SnowflakeConverter.escape(self.to_snowflake(v)) for v in value]
         ]
 
     _tuple_to_snowflake = _list_to_snowflake

test/integ/aio_it/test_converter_async.py

@@ -468,6 +468,72 @@ async def test_franction_followed_by_year_format(conn_cnx):
             assert rec[0] == "05:34:56.123456 Jan 03, 2012"
 
 
+async def test_binary_list_parameter_in_query(conn_cnx):
+    """Test that binary data in list parameters works correctly.
+
+    Regression test for GitHub Issue #2311.
+    When binary data (bytes) is passed in a list parameter for an IN clause,
+    the converter should properly hex-encode each item in the list.
+    Without the fix, this raises UnicodeDecodeError when bytes contain non-ASCII values.
+    """
+    import uuid
+
+    # Use UUIDs that generate non-ASCII bytes (bytes > 127)
+    business_uuid = uuid.UUID("4363f57d-c9ca-4e63-92c7-3e67786c8a30")
+    user_uuid1 = uuid.UUID("4363f57d-c9ca-4e63-92c7-3e67786c8a30")
+    user_uuid2 = uuid.UUID("f5abcdef-1234-5678-9abc-def012345678")
+
+    async with conn_cnx() as conn:
+        cur = conn.cursor()
+        try:
+            # Create a temporary table for testing
+            await cur.execute(
+                "CREATE OR REPLACE TEMPORARY TABLE test_binary_list_param "
+                "(business_uuid BINARY, user_uuid BINARY)"
+            )
+
+            # Insert test data
+            await cur.execute(
+                "INSERT INTO test_binary_list_param VALUES (%(business_uuid)s, %(user_uuid)s)",
+                params={
+                    "business_uuid": business_uuid.bytes,
+                    "user_uuid": user_uuid1.bytes,
+                },
+            )
+
+            # Test single binary parameter (this should work)
+            await cur.execute(
+                "SELECT * FROM test_binary_list_param WHERE business_uuid = %(business_uuid)s",
+                params={"business_uuid": business_uuid.bytes},
+            )
+            results = await cur.fetchall()
+            assert len(results) == 1
+
+            # Test binary in list parameter (this was failing with UnicodeDecodeError)
+            await cur.execute(
+                "SELECT * FROM test_binary_list_param "
+                "WHERE business_uuid = %(business_uuid)s AND user_uuid IN (%(user_uuids)s)",
+                params={
+                    "business_uuid": business_uuid.bytes,
+                    "user_uuids": [user_uuid1.bytes],
+                },
+            )
+            results = await cur.fetchall()
+            assert len(results) == 1
+
+            # Test with multiple binary values in list
+            await cur.execute(
+                "SELECT * FROM test_binary_list_param "
+                "WHERE user_uuid IN (%(user_uuids)s)",
+                params={"user_uuids": [user_uuid1.bytes, user_uuid2.bytes]},
+            )
+            results = await cur.fetchall()
+            assert len(results) == 1  # Only user_uuid1 exists in the table
+
+        finally:
+            await cur.execute("DROP TABLE IF EXISTS test_binary_list_param")
+
+
 async def test_fetch_fraction_timestamp(conn_cnx):
     """Additional fetch timestamp tests. Mainly used for SnowSQL which converts to string representations."""
     PST_TZ = "America/Los_Angeles"

test/integ/test_converter.py

@@ -535,3 +535,66 @@ def test_fetch_fraction_timestamp(conn_cnx):
         assert ret[9] == "2100-01-01 05:00:00.012000000"
         assert ret[10] == "1970-01-01 00:00:00.000000000 +0000"
         assert ret[11] == "1970-01-01 00:00:00.000000000"
+
+
+def test_binary_list_parameter_in_query(conn_cnx):
+    """Test that binary data in list parameters works correctly.
+
+    Regression test for GitHub Issue #2311.
+    When binary data (bytes) is passed in a list parameter for an IN clause,
+    the converter should properly hex-encode each item in the list.
+    Without the fix, this raises UnicodeDecodeError when bytes contain non-ASCII values.
+    """
+    import uuid
+
+    # Use UUIDs that generate non-ASCII bytes (bytes > 127)
+    business_uuid = uuid.UUID("4363f57d-c9ca-4e63-92c7-3e67786c8a30")
+    user_uuid1 = uuid.UUID("4363f57d-c9ca-4e63-92c7-3e67786c8a30")
+    user_uuid2 = uuid.UUID("f5abcdef-1234-5678-9abc-def012345678")
+
+    with conn_cnx() as conn:
+        cur = conn.cursor()
+        try:
+            # Create a temporary table for testing
+            cur.execute(
+                "CREATE OR REPLACE TEMPORARY TABLE test_binary_list_param "
+                "(business_uuid BINARY, user_uuid BINARY)"
+            )
+
+            # Insert test data
+            cur.execute(
+                "INSERT INTO test_binary_list_param VALUES (%(business_uuid)s, %(user_uuid)s)",
+                params={
+                    "business_uuid": business_uuid.bytes,
+                    "user_uuid": user_uuid1.bytes,
+                },
+            )
+
+            # Test single binary parameter (this should work)
+            results = cur.execute(
+                "SELECT * FROM test_binary_list_param WHERE business_uuid = %(business_uuid)s",
+                params={"business_uuid": business_uuid.bytes},
+            ).fetchall()
+            assert len(results) == 1
+
+            # Test binary in list parameter (this was failing with UnicodeDecodeError)
+            results = cur.execute(
+                "SELECT * FROM test_binary_list_param "
+                "WHERE business_uuid = %(business_uuid)s AND user_uuid IN (%(user_uuids)s)",
+                params={
+                    "business_uuid": business_uuid.bytes,
+                    "user_uuids": [user_uuid1.bytes],
+                },
+            ).fetchall()
+            assert len(results) == 1
+
+            # Test with multiple binary values in list
+            results = cur.execute(
+                "SELECT * FROM test_binary_list_param "
+                "WHERE user_uuid IN (%(user_uuids)s)",
+                params={"user_uuids": [user_uuid1.bytes, user_uuid2.bytes]},
+            ).fetchall()
+            assert len(results) == 1  # Only user_uuid1 exists in the table
+
+        finally:
+            cur.execute("DROP TABLE IF EXISTS test_binary_list_param")

test/unit/test_converter.py

@@ -4,6 +4,7 @@
 from datetime import timedelta
 from decimal import Decimal
 from logging import getLogger
+from uuid import UUID
 
 import numpy
 import pytest
@@ -133,3 +134,78 @@ def test_year_month_interval_to_timedelta(months):
     assert converter.INTERVAL_YEAR_MONTH_to_numpy_timedelta(
         months, scale=0
     ) == numpy.timedelta64(months, "M")
+
+
+def test_converter_to_snowflake_bytes():
+    """Test that bytes in a list are properly hex-encoded."""
+    uuid = UUID("12345678-1234-5678-1234-567812345678")
+
+    converter = SnowflakeConverter()
+    # After fix for GH-2311: bytes are now properly hex-encoded via to_snowflake()
+    assert converter.to_snowflake([uuid.bytes]) == [
+        "X'12345678123456781234567812345678'"
+    ]
+
+
+def test_converter_list_with_binary_to_snowflake():
+    """Test that bytes values in lists are properly converted to hex format.
+
+    Regression test for GitHub Issue #2311.
+    When binary data (bytes) is passed in a list parameter, each item in the list
+    should have to_snowflake called on it, converting bytes to hex format.
+    Without this fix, a UnicodeDecodeError occurs when bytes contain non-ASCII values.
+    """
+    # Use a UUID that generates non-ASCII bytes (contains bytes > 127)
+    test_uuid = UUID("4363f57d-c9ca-4e63-92c7-3e67786c8a30")
+
+    converter = SnowflakeConverter()
+
+    # Single bytes should work - this calls _bytes_to_snowflake which hex-encodes
+    single_result = converter.to_snowflake(test_uuid.bytes)
+    # The result should be hex-encoded bytes (ASCII safe)
+    assert isinstance(single_result, (bytes, bytearray))
+    expected_hex = single_result.decode("ascii")
+    assert expected_hex == "4363F57DC9CA4E6392C73E67786C8A30"
+
+    # List of bytes should also work - currently fails with UnicodeDecodeError
+    # because _list_to_snowflake doesn't call to_snowflake on each item.
+    # After fix, each item in the list should be properly quoted with hex format.
+    list_result = converter.to_snowflake([test_uuid.bytes])
+    assert list_result == [f"X'{expected_hex}'"]
+
+
+def test_converter_list_with_non_ascii_bytes_to_snowflake():
+    """Test that non-ASCII bytes in lists don't cause UnicodeDecodeError.
+
+    Regression test for GitHub Issue #2311.
+    This test specifically uses bytes that contain values > 127 which cannot
+    be decoded as ASCII, ensuring the fix properly hex-encodes them first.
+    """
+    # These bytes contain values that cannot be decoded as ASCII
+    non_ascii_bytes = bytes([0xF5, 0xAB, 0xCD, 0xEF])
+
+    converter = SnowflakeConverter()
+
+    # Single bytes should work
+    single_result = converter.to_snowflake(non_ascii_bytes)
+    assert single_result == b"F5ABCDEF"
+
+    # List of non-ASCII bytes should also work without UnicodeDecodeError
+    list_result = converter.to_snowflake([non_ascii_bytes])
+    assert list_result == ["X'F5ABCDEF'"]
+
+
+def test_converter_list_with_mixed_types_to_snowflake():
+    """Test that lists with mixed types including bytes work correctly."""
+    converter = SnowflakeConverter()
+
+    # Mix of strings and bytes
+    mixed_list = [b"\x00\xFF", "hello", 42, None, True]
+    result = converter.to_snowflake(mixed_list)
+
+    # Verify each element is properly converted
+    assert "X'00FF'" in result  # bytes should be hex-encoded
+    assert "'hello'" in result  # string should be quoted
+    assert "42" in result  # number should be stringified
+    assert "NULL" in result  # None should become NULL
+    assert "TRUE" in result  # bool should become TRUE/FALSE