Remove duplicated GGUF calls and use endiag for fromByteArray factories

Related-To: #121 #95

Author: michalharakal

skainet-core/skainet-tensors/src/commonMain/kotlin/sk/ainet/core/tensor/backend/CpuBackend.kt

@@ -218,6 +218,40 @@ public class CpuTensorFP32(
         public fun full(shape: Shape, value: Float): CpuTensorFP32 {
             return CpuTensorFP32(shape, FloatArray(shape.volume) { value })
         }
+
+        /**
+         * Creates a 2D tensor from a nested list structure.
+         * Similar to numpy.array([[2.0, 1.5, 4.2, 3.1], [1.0, 2.5, 3.0, 4.5]])
+         * 
+         * @param data The nested list where outer list represents rows and inner lists represent columns
+         * @return A new CpuTensorFP32 instance with shape inferred from the nested structure
+         * @throws IllegalArgumentException if the nested list structure is invalid or inconsistent
+         */
+        public fun fromNestedList(data: List<List<Float>>): CpuTensorFP32 {
+            require(data.isNotEmpty()) { "Data cannot be empty" }
+            
+            val rows = data.size
+            val cols = data[0].size
+            require(cols > 0) { "Each row must have at least one element" }
+            
+            // Validate that all rows have the same number of columns
+            for (i in data.indices) {
+                require(data[i].size == cols) {
+                    "All rows must have the same number of columns. Row 0 has $cols columns, but row $i has ${data[i].size} columns"
+                }
+            }
+            
+            // Create the flat array in row-major order
+            val flatArray = FloatArray(rows * cols)
+            var index = 0
+            for (row in data) {
+                for (value in row) {
+                    flatArray[index++] = value
+                }
+            }
+            
+            return CpuTensorFP32(Shape(rows, cols), flatArray)
+        }
     }
 }
 
@@ -1294,6 +1328,40 @@ public class CpuTensorInt32(
         public fun full(shape: Shape, value: Int): CpuTensorInt32 {
             return CpuTensorInt32(shape, IntArray(shape.volume) { value })
         }
+
+        /**
+         * Creates a 2D tensor from a nested list structure.
+         * Similar to numpy.array([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]])
+         * 
+         * @param data The nested list where outer list represents rows and inner lists represent columns
+         * @return A new CpuTensorInt32 instance with shape inferred from the nested structure
+         * @throws IllegalArgumentException if the nested list structure is invalid or inconsistent
+         */
+        public fun fromNestedList(data: List<List<Int>>): CpuTensorInt32 {
+            require(data.isNotEmpty()) { "Data cannot be empty" }
+            
+            val rows = data.size
+            val cols = data[0].size
+            require(cols > 0) { "Each row must have at least one element" }
+            
+            // Validate that all rows have the same number of columns
+            for (i in data.indices) {
+                require(data[i].size == cols) {
+                    "All rows must have the same number of columns. Row 0 has $cols columns, but row $i has ${data[i].size} columns"
+                }
+            }
+            
+            // Create the flat array in row-major order
+            val flatArray = IntArray(rows * cols)
+            var index = 0
+            for (row in data) {
+                for (value in row) {
+                    flatArray[index++] = value
+                }
+            }
+            
+            return CpuTensorInt32(Shape(rows, cols), flatArray)
+        }
     }
 }
 

skainet-core/skainet-tensors/src/commonMain/kotlin/sk/ainet/core/tensor/factory/FP16TensorFactory.kt

@@ -7,30 +7,30 @@ import sk.ainet.core.tensor.Tensor
 /**
  * Factory for creating FP16 tensors from byte data.
  * This is currently a placeholder implementation for future development.
- * 
+ *
  */
 public object FP16TensorFactory : TensorFactoryRegistry.TensorFromBytesFactory<FP16, Float> {
-    
+
     /**
-     * Creates a tensor from GGUF-style byte data with FP16 values.
-     * 
+     * Creates a tensor from byte data with FP16 values.
+     *
      * 1. Convert bytes to FP16 values with proper endianness handling
      * 2. Validate input data size matches expected FP16 count (shape.volume * 2 bytes)
      * 3. Handle FP16 to FP32 conversion if needed
      * 4. Create and return proper CpuTensorFP16 instance
-     * 
+     *
      * @param shape The desired shape of the tensor
      * @param data The byte array containing FP16 data (2 bytes per value)
      * @return A new FP16 tensor instance
      * @throws NotImplementedError Currently not implemented
      */
-    override fun fromByteArray(shape: Shape, data: ByteArray): Tensor<FP16, Float> {
+    override fun fromByteArray(shape: Shape, data: ByteArray, littleEndian: Boolean): Tensor<FP16, Float> {
         // Validate data size for FP16 (2 bytes per value)
         val expectedBytes = shape.volume * 2
         require(data.size == expectedBytes) {
             "Data size mismatch: expected $expectedBytes bytes for FP16 data, got ${data.size}"
         }
-        
+
         // Create CpuTensorFP16 instance with FP16 byte data
         return sk.ainet.core.tensor.backend.CpuTensorFP16.fromFP16ByteArray(shape, data)
     }

skainet-core/skainet-tensors/src/commonMain/kotlin/sk/ainet/core/tensor/factory/FP32TensorFactory.kt

@@ -12,13 +12,14 @@ import sk.ainet.core.tensor.backend.CpuTensorFP32
 public object FP32TensorFactory : TensorFactoryRegistry.TensorFromBytesFactory<FP32, Float> {
 
     /**
-     * Creates a CpuTensorFP32 from GGUF byte data with the specified shape.
+     * Creates a CpuTensorFP32 from byte data with the specified shape.
      * @param shape The desired shape of the tensor
      * @param data The byte array containing float data in binary format
+     * @param littleEndian true for little-endian, false for big-endian
      * @return A new CpuTensorFP32 instance
      * @throws IllegalArgumentException if data size doesn't match expected float count
      */
-    override fun fromByteArray(shape: Shape, data: ByteArray): Tensor<FP32, Float> {
+    override fun fromByteArray(shape: Shape, data: ByteArray, littleEndian: Boolean): Tensor<FP32, Float> {
         // Validate input data size matches shape requirements
         val expectedFloatCount = shape.volume
         val expectedByteSize = expectedFloatCount * 4 // 4 bytes per float
@@ -29,33 +30,9 @@ public object FP32TensorFactory : TensorFactoryRegistry.TensorFromBytesFactory<F
         }
 
         // Convert byte array to float array using little-endian (GGUF standard)
-        val floatArray = ByteArrayConverter.convertBytesToFloatArray(data, littleEndian = true)
+        val floatArray = ByteArrayConverter.convertBytesToFloatArray(data, littleEndian = littleEndian)
 
         // Create and return the tensor using the existing factory method
         return CpuTensorFP32.fromArray(shape, floatArray)
     }
-    
-    /**
-     * Creates a tensor from byte data with specified endianness.
-     * @param shape The desired shape of the tensor
-     * @param data The byte array containing float data
-     * @param littleEndian true for little-endian, false for big-endian
-     * @return A new CpuTensorFP32 instance
-     */
-    public fun fromGGUFData(shape: Shape, data: ByteArray, littleEndian: Boolean): Tensor<FP32, Float> {
-        // Validate input data size matches shape requirements
-        val expectedFloatCount = shape.volume
-        val expectedByteSize = expectedFloatCount * 4
-        
-        require(data.size == expectedByteSize) {
-            "Input data size (${data.size} bytes) does not match expected size for shape $shape " +
-            "(expected $expectedByteSize bytes for $expectedFloatCount floats)"
-        }
-        
-        // Convert byte array to float array with specified endianness
-        val floatArray = ByteArrayConverter.convertBytesToFloatArray(data, littleEndian)
-        
-        // Create and return the tensor
-        return CpuTensorFP32.fromArray(shape, floatArray)
-    }
 }

skainet-core/skainet-tensors/src/commonMain/kotlin/sk/ainet/core/tensor/factory/Int32TensorFactory.kt

@@ -10,74 +10,29 @@ import sk.ainet.core.tensor.backend.CpuTensorInt32
  * Handles byte-to-int conversion with proper endianness support.
  */
 public object Int32TensorFactory : TensorFactoryRegistry.TensorFromBytesFactory<Int32, Int> {
-    
+
     /**
      * Creates a CpuTensorInt32 from GGUF byte data with the specified shape.
      * @param shape The desired shape of the tensor
      * @param data The byte array containing int data in binary format
+     * @param littleEndian true for little-endian, false for big-endian
      * @return A new CpuTensorInt32 instance
      * @throws IllegalArgumentException if data size doesn't match expected int count
      */
-    override fun fromByteArray(shape: Shape, data: ByteArray): Tensor<Int32, Int> {
+    override fun fromByteArray(shape: Shape, data: ByteArray, littleEndian: Boolean): Tensor<Int32, Int> {
         // Validate input data size matches shape requirements
         val expectedIntCount = shape.volume
         val expectedByteSize = expectedIntCount * 4 // 4 bytes per int
-        
+
         require(data.size == expectedByteSize) {
             "Input data size (${data.size} bytes) does not match expected size for shape $shape " +
-            "(expected $expectedByteSize bytes for $expectedIntCount ints)"
+                    "(expected $expectedByteSize bytes for $expectedIntCount ints)"
         }
-        
+
         // Convert byte array to int array using little-endian (GGUF standard)
         val intArray = ByteArrayConverter.convertBytesToIntArray(data, littleEndian = true)
-        
+
         // Create and return the tensor using the existing factory method
         return CpuTensorInt32.fromArray(shape, intArray)
     }
-    
-    /**
-     * Creates a tensor from byte data with specified endianness.
-     * @param shape The desired shape of the tensor
-     * @param data The byte array containing int data
-     * @param littleEndian true for little-endian, false for big-endian
-     * @return A new CpuTensorInt32 instance
-     */
-    public fun fromGGUFData(shape: Shape, data: ByteArray, littleEndian: Boolean): Tensor<Int32, Int> {
-        // Validate input data size matches shape requirements
-        val expectedIntCount = shape.volume
-        val expectedByteSize = expectedIntCount * 4
-        
-        require(data.size == expectedByteSize) {
-            "Input data size (${data.size} bytes) does not match expected size for shape $shape " +
-            "(expected $expectedByteSize bytes for $expectedIntCount ints)"
-        }
-        
-        // Convert byte array to int array with specified endianness
-        val intArray = ByteArrayConverter.convertBytesToIntArray(data, littleEndian)
-        
-        // Create and return the tensor
-        return CpuTensorInt32.fromArray(shape, intArray)
-    }
-    
-    /**
-     * Validates that the input data is properly aligned for int conversion.
-     * @param shape The tensor shape
-     * @param data The input data
-     * @throws IllegalArgumentException if validation fails
-     */
-    private fun validateInput(shape: Shape, data: ByteArray) {
-        val expectedIntCount = shape.volume
-        val expectedByteSize = expectedIntCount * 4
-        
-        require(data.size == expectedByteSize) {
-            "Input data size (${data.size} bytes) does not match expected size for shape $shape " +
-            "(expected $expectedByteSize bytes for $expectedIntCount ints)"
-        }
-        require(data.isNotEmpty()) {
-            "Input data cannot be empty"
-        }
-        require(data.size % 4 == 0) {
-            "Input data size (${data.size} bytes) must be a multiple of 4 for int conversion"
-        }
-    }
 }

skainet-core/skainet-tensors/src/commonMain/kotlin/sk/ainet/core/tensor/factory/Int4TensorFactory.kt

@@ -7,40 +7,41 @@ import sk.ainet.core.tensor.Tensor
 /**
  * Factory for creating Int4 tensors from byte data.
  * This is currently a placeholder implementation for future development.
- * 
+ *
  * Int4 tensors use packed 4-bit integer values, storing 2 values per byte.
- * 
+ *
  */
 public object Int4TensorFactory : TensorFactoryRegistry.TensorFromBytesFactory<Int4, Byte> {
-    
+
     /**
      * Creates a tensor from GGUF-style byte data with packed Int4 values.
-     * 
+     *
      * When CpuTensorInt4 becomes available, this should:
      * 1. Implement bit unpacking for 4-bit values (2 values per byte)
      * 2. Handle signed 4-bit integers (range -8 to 7)
      * 3. Validate input data size matches expected packed size (ceil(shape.volume / 2) bytes)
      * 4. Implement unpacking strategy for accessing individual 4-bit values
      * 5. Create and return proper CpuTensorInt4 instance
-     * 
+     *
      * Storage strategy considerations:
      * - Each byte contains 2 Int4 values
      * - High nibble (bits 4-7): first value
      * - Low nibble (bits 0-3): second value
      * - Handle odd tensor volumes (last byte may only use high nibble)
-     * 
+     *
      * @param shape The desired shape of the tensor
      * @param data The byte array containing packed Int4 data (2 values per byte)
+     * @param littleEndian true for little-endian, false for big-endian
      * @return A new Int4 tensor instance
      * @throws NotImplementedError Currently not implemented
      */
-    override fun fromByteArray(shape: Shape, data: ByteArray): Tensor<Int4, Byte> {
+    override fun fromByteArray(shape: Shape, data: ByteArray, littleEndian: Boolean): Tensor<Int4, Byte> {
         // Validate data size for packed Int4 (2 values per byte, rounded up)
         val expectedBytes = (shape.volume + 1) / 2  // Ceiling division for odd volumes
         require(data.size == expectedBytes) {
             "Data size mismatch: expected $expectedBytes bytes for packed Int4 data (${shape.volume} values), got ${data.size}"
         }
-        
+
         // Create CpuTensorInt4 instance with packed data
         return sk.ainet.core.tensor.backend.CpuTensorInt4.fromPackedByteArray(shape, data)
     }

skainet-core/skainet-tensors/src/commonMain/kotlin/sk/ainet/core/tensor/factory/Int8TensorFactory.kt

@@ -10,55 +10,29 @@ import sk.ainet.core.tensor.backend.CpuTensorInt8
  * Since Int8 tensors store byte values directly, no conversion is needed.
  */
 public object Int8TensorFactory : TensorFactoryRegistry.TensorFromBytesFactory<Int8, Byte> {
-    
+
     /**
      * Creates a CpuTensorInt8 from GGUF byte data with the specified shape.
      * @param shape The desired shape of the tensor
      * @param data The byte array containing the tensor data
+     * @param littleEndian true for little-endian, false for big-endian
+     *
      * @return A new CpuTensorInt8 instance
      * @throws IllegalArgumentException if data size doesn't match expected byte count
      */
-    override fun fromByteArray(shape: Shape, data: ByteArray): Tensor<Int8, Byte> {
+    override fun fromByteArray(shape: Shape, data: ByteArray, littleEndian: Boolean): Tensor<Int8, Byte> {
         // Validate input data size matches shape requirements
         val expectedByteCount = shape.volume
-        
+
         require(data.size == expectedByteCount) {
             "Input data size (${data.size} bytes) does not match expected size for shape $shape " +
-            "(expected $expectedByteCount bytes)"
+                    "(expected $expectedByteCount bytes)"
         }
-        
+
         // For Int8 tensors, we can use the byte array directly (no conversion needed)
         val byteArray = ByteArrayConverter.convertBytesToByteArray(data)
-        
+
         // Create and return the tensor using the existing factory method
         return CpuTensorInt8.fromArray(shape, byteArray)
     }
-    
-    /**
-     * Creates a tensor from byte data with validation.
-     * This method is identical to fromByteArray since Int8 doesn't need endianness conversion.
-     * @param shape The desired shape of the tensor
-     * @param data The byte array containing the tensor data
-     * @return A new CpuTensorInt8 instance
-     */
-    public fun fromByteData(shape: Shape, data: ByteArray): Tensor<Int8, Byte> {
-        return fromByteArray(shape, data)
-    }
-    
-    /**
-     * Validates that the input data size matches the expected tensor volume.
-     * @param shape The tensor shape
-     * @param data The input data
-     * @throws IllegalArgumentException if validation fails
-     */
-    private fun validateInput(shape: Shape, data: ByteArray) {
-        val expectedByteCount = shape.volume
-        require(data.size == expectedByteCount) {
-            "Input data size (${data.size} bytes) does not match expected size for shape $shape " +
-            "(expected $expectedByteCount bytes)"
-        }
-        require(data.isNotEmpty()) {
-            "Input data cannot be empty"
-        }
-    }
 }