Introduced ModelReader and TensorInfo in skainet-io-core as a common abstraction.

- Refactored `MemoryChunk` to support non-copying `slice` views for efficient memory mapping.
- Added dependency on `skainet-io-core` in `skainet-io-onnx`.
- Fixed alignment parsing and improved tensor data handling in `GGUFReader` tests.
- Increased test heap size to 8GB to support large model tests.

Author: michalharakal

build.gradle.kts

@@ -29,6 +29,10 @@ subprojects {
             jvmToolchain(21)
         }
     }
+
+    tasks.withType<Test>().configureEach {
+        maxHeapSize = "8192m"
+    }
 }
 
 kover {

skainet-io/skainet-io-core/src/commonMain/kotlin/sk/ainet/io/MemoryChunk.kt

@@ -0,0 +1,32 @@
+package sk.ainet.io
+
+interface MemoryChunk {
+    val size: Long
+    
+    fun readByte(offset: Long): Byte
+    fun readBytes(offset: Long, length: Int): ByteArray
+    
+    fun slice(offset: Long, length: Long): MemoryChunk
+}
+
+class ByteArrayMemoryChunk(
+    private val data: ByteArray,
+    private val offset: Int = 0,
+    override val size: Long = data.size.toLong()
+) : MemoryChunk {
+
+    override fun readByte(offset: Long): Byte {
+        require(offset >= 0 && offset < size) { "Offset out of bounds: $offset" }
+        return data[this.offset + offset.toInt()]
+    }
+
+    override fun readBytes(offset: Long, length: Int): ByteArray {
+        require(offset >= 0 && offset + length <= size) { "Range out of bounds: $offset + $length" }
+        return data.copyOfRange(this.offset + offset.toInt(), this.offset + offset.toInt() + length)
+    }
+
+    override fun slice(offset: Long, length: Long): MemoryChunk {
+        require(offset >= 0 && offset + length <= size) { "Slice out of bounds: $offset + $length" }
+        return ByteArrayMemoryChunk(data, this.offset + offset.toInt(), length)
+    }
+}

skainet-io/skainet-io-core/src/commonMain/kotlin/sk/ainet/io/ModelReader.kt

@@ -0,0 +1,21 @@
+package sk.ainet.io
+
+import sk.ainet.lang.tensor.Shape
+import sk.ainet.lang.types.DType
+import sk.ainet.lang.tensor.data.TensorData
+
+interface ModelReader : AutoCloseable {
+    val metadata: Map<String, Any>
+    val tensors: Map<String, TensorInfo>
+    
+    suspend fun loadTensor(name: String): TensorData<*, *>
+}
+
+data class TensorInfo(
+    val name: String,
+    val shape: Shape,
+    val dtype: String,
+    val offset: Long,
+    val size: Long, // Size in bytes
+    val extra: Map<String, Any> = emptyMap()
+)

skainet-io/skainet-io-gguf/src/commonMain/kotlin/sk/ainet/io/gguf/GGUFModelReader.kt

@@ -0,0 +1,21 @@
+package sk.ainet.io.gguf
+
+import sk.ainet.io.ModelReader
+import sk.ainet.io.TensorInfo
+import sk.ainet.lang.tensor.data.TensorData
+import sk.ainet.lang.tensor.Shape
+
+class GGUFModelReader : ModelReader {
+    override val metadata: Map<String, Any> = mutableMapOf()
+    override val tensors: Map<String, TensorInfo> = mutableMapOf()
+
+    override suspend fun loadTensor(name: String): TensorData<*, *> {
+        val info = tensors[name] ?: error("Tensor $name not found")
+        // Implementation will use MemoryMappedFileChunk or similar to slice the data
+        TODO("Not yet implemented: streaming tensor loading for GGUF")
+    }
+
+    override fun close() {
+        // Close underlying resources (e.g. mapped file)
+    }
+}

skainet-io/skainet-io-gguf/src/commonMain/kotlin/sk/ainet/io/gguf/GGUFReader.kt

@@ -133,7 +133,7 @@ class GGUFReader(source: Source, private val loadTensorData: Boolean = true) {
         offs = newOffs
         val newAlign = fields["general.alignment"]
         if (newAlign != null && newAlign.types == listOf(GGUFValueType.UINT32)) {
-            alignment = newAlign.parts.last()[0] as Int
+            alignment = (newAlign.parts.last()[0] as UInt).toInt()
         }
         val padding = offs % alignment
         if (padding != 0) {

skainet-io/skainet-io-gguf/src/jvmTest/kotlin/sk/ainet/io/gguf/GGUFMinimalSampleTest.kt

@@ -74,13 +74,12 @@ class GGUFMinimalSampleTest {
                     GGMLQuantizationType.F32 -> ctx.fromFloatArray<FP32, Float>(shape, FP32::class, (raw as List<Float>).toFloatArray())
                     GGMLQuantizationType.I32 -> ctx.fromIntArray<Int32, Int>(shape, Int32::class, (raw as List<Int>).toIntArray())
                     GGMLQuantizationType.I8 -> ctx.fromByteArray<Int8, Byte>(shape, Int8::class, (raw as List<Byte>).toByteArray())
-                    GGMLQuantizationType.F16 -> {
-                        // FP16 raw storage is not supported by reader; we mapped to FP16 class as a placeholder.
-                        // As a minimal sample, store as bytes to keep the payload without decoding.
-                        ctx.fromByteArray<Int8, Byte>(shape, Int8::class, (raw as List<UByte>).toUByteArray().toByteArray())
+                    else -> {
+                        // All other quantized/native types: keep raw bytes as Int8 tensor for this minimal sample
+                        // We use a flat shape to match the byte count if it doesn't match the element count (e.g. for F16)
+                        val byteData = (raw as List<UByte>).toUByteArray().toByteArray()
+                        ctx.fromByteArray<Int8, Byte>(Shape(byteData.size), Int8::class, byteData)
                     }
-                    // All other quantized/native types: keep raw bytes as Int8 tensor for this minimal sample
-                    else -> ctx.fromByteArray<Int8, Byte>(shape, Int8::class, (raw as List<UByte>).toUByteArray().toByteArray())
                 }
                 created[rt.name] = dslTensor
             }

skainet-io/skainet-io-onnx/build.gradle.kts

@@ -53,6 +53,7 @@ kotlin {
             dependencies {
                 implementation(project(":skainet-lang:skainet-lang-core"))
                 implementation(project(":skainet-compile:skainet-compile-dag"))
+                implementation(project(":skainet-io:skainet-io-core"))
                 implementation(libs.kotlinx.io.core)
                 implementation(libs.pbandk.runtime)
             }

skainet-io/skainet-io-onnx/src/commonMain/kotlin/sk/ainet/io/onnx/OnnxModelReader.kt

@@ -0,0 +1,20 @@
+package sk.ainet.io.onnx
+
+import sk.ainet.io.ModelReader
+import sk.ainet.io.TensorInfo
+import sk.ainet.lang.tensor.data.TensorData
+
+public class OnnxModelReader : ModelReader {
+    override val metadata: Map<String, Any> = emptyMap<String, Any>()
+    override val tensors: Map<String, TensorInfo> = emptyMap<String, TensorInfo>()
+
+    override suspend fun loadTensor(name: String): TensorData<*, *> {
+        val info = tensors[name] ?: error("Tensor $name not found")
+        // Implementation will handle external_data or embedded rawData offsets
+        TODO("Not yet implemented: streaming tensor loading for ONNX")
+    }
+
+    override fun close() {
+        // Close underlying resources
+    }
+}