Code review changes

Author: VeaaC

flatdata-py/flatdata/lib/data_access.py

@@ -24,6 +24,8 @@ def read_value(data, offset_bits, num_bits, is_signed):
 
     if num_bits < 64:
         result = result & ((1 << num_bits) - 1)
+    elif offset_extra_bits > 0:
+        result = result & ((1 << num_bits) - 1)
 
     if not is_signed:
         return result
@@ -75,21 +77,38 @@ def read_field_vectorized(raw_bytes_2d, field_offset_bits, field_width_bits, is_
     :param is_signed: whether to sign-extend the result
     :return: numpy array of field values
     """
+    if field_width_bits == 1:
+        byte_idx = field_offset_bits // 8
+        bit_idx = field_offset_bits % 8
+        return ((raw_bytes_2d[:, byte_idx].astype(np.uint64) >> np.uint64(bit_idx)) &
+                np.uint64(1))
+
     byte_start = field_offset_bits // 8
     bit_shift = field_offset_bits % 8
     bytes_needed = (bit_shift + field_width_bits + 7) // 8
 
+    # Use Python int arithmetic for the shift to avoid numpy overflow,
+    # then broadcast back to the array.
     result = np.zeros(raw_bytes_2d.shape[0], dtype=np.uint64)
-    for b in range(bytes_needed):
+    for b in range(min(bytes_needed, 8)):
         result |= raw_bytes_2d[:, byte_start + b].astype(np.uint64) << np.uint64(b * 8)
     result >>= np.uint64(bit_shift)
 
+    # If the field spans more than 8 bytes (unaligned 64-bit field), merge the extra byte.
+    bits_so_far = 8 * min(bytes_needed, 8) - bit_shift
+    if bits_so_far < field_width_bits and bytes_needed > 8:
+        extra = raw_bytes_2d[:, byte_start + 8].astype(np.uint64)
+        result |= extra << np.uint64(bits_so_far)
+
     if field_width_bits < 64:
         result &= np.uint64((1 << field_width_bits) - 1)
 
     if is_signed:
+        if field_width_bits == 64:
+            return result.view(np.int64)
         sign_bit = np.uint64(1 << (field_width_bits - 1))
-        signed = result.astype(np.int64) - np.int64(1 << field_width_bits)
+        offset = -(1 << field_width_bits)
+        signed = result.astype(np.int64) + np.int64(offset)
         result = np.where(result & sign_bit, signed, result.astype(np.int64))
 
     return result

flatdata-py/flatdata/lib/resources.py

@@ -76,7 +76,6 @@ def __init__(self, s, sequence):
 
     def to_numpy(self, limit=None):
         raw_2d = self._sequence._as_numpy_2d()
-        indices = self._slice.indices(len(self._sequence))
         sliced = raw_2d[self._slice]
         if limit is not None:
             sliced = sliced[:limit]
@@ -98,8 +97,11 @@ def __iter__(self):
             yield self._sequence[i]
 
     def __getattr__(self, name):
+        try:
+            field = self._sequence._element_type._FIELDS[name]
+        except KeyError:
+            raise AttributeError("Field %s not found in structure" % name)
         raw_2d = self._sequence._as_numpy_2d()[self._slice]
-        field = self._sequence._element_type._FIELDS[name]
         values = read_field_vectorized(raw_2d, field.offset, field.width, field.is_signed)
         return pd.DataFrame(data=values, columns=[name])
 
@@ -148,8 +150,11 @@ def __iter__(self):
             yield element_type(mem, SIZE_OFFSET_IN_BYTES + size_bytes * i)
 
     def __getattr__(self, name):
+        try:
+            field = self._element_type._FIELDS[name]
+        except KeyError:
+            raise AttributeError("Field %s not found in structure" % name)
         raw_2d = self._as_numpy_2d()
-        field = self._element_type._FIELDS[name]
         values = read_field_vectorized(raw_2d, field.offset, field.width, field.is_signed)
         return pd.DataFrame(data=values, columns=[name])
 

flatdata-py/tests/test_data_access.py

@@ -1124,6 +1124,25 @@ def _test_reader(buffer, offset, num_bits, is_signed, expected):
     _test_reader(b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 3, 2, True, 0)
 
 
+def test_reader_unaligned_64bit():
+    """read_value must return at most 64 bits for a 64-bit field at a non-byte-aligned offset."""
+    data = b"\xff" * 9
+    # 64 bits starting at bit 1, all set → 0xFFFFFFFFFFFFFFFF
+    result = read_value(data, 1, 64, False)
+    assert result == 0xFFFFFFFFFFFFFFFF, \
+        f"Expected 0xFFFFFFFFFFFFFFFF, got {result:#x} ({result.bit_length()} bits)"
+
+    # Signed: all 1s → -1
+    result_signed = read_value(data, 1, 64, True)
+    assert result_signed == -1
+
+    # Other non-byte-aligned offsets
+    for offset in range(1, 8):
+        result = read_value(data, offset, 64, False)
+        assert result == 0xFFFFFFFFFFFFFFFF, \
+            f"offset={offset}: expected 0xFFFFFFFFFFFFFFFF, got {result:#x}"
+
+
 def test_writer():
     """
     Following tests were generated from C++ counterparts. Reasoning: python implementation lacks