Skip to content

Commit 4bd5548

Browse files
michalharakalmichalharakal
committed
Add MNIST downloader
Relate-To: #112, #113
1 parent b3a146e commit 4bd5548

14 files changed

Lines changed: 1035 additions & 5 deletions

File tree

‎gradle/libs.versions.toml‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,34 @@
11
[versions]
22
agp = "8.11.2"
33
kotlin = "2.2.20"
4-
kotlinx-coroutines = "1.10.2"
4+
kotlinxCoroutines = "1.10.2"
55
android-minSdk = "24"
66
android-compileSdk = "36"
7+
kotlinxSerializationJson = "1.9.0"
8+
ktorClientCore = "3.3.0"
9+
ktorClientPlugins = "3.1.1"
10+
logbackClassic = "1.5.18"
11+
712

813
[libraries]
9-
kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" }
14+
kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" }
15+
kotlinx-coroutines-core-jvm = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm", version.ref = "kotlinxCoroutines" }
16+
1017
kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" }
18+
kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
19+
20+
ktor-client-android = { module = "io.ktor:ktor-client-android", version.ref = "ktorClientCore" }
21+
ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktorClientCore" }
22+
ktor-client-content-negotiation = { module = "io.ktor:ktor-client-content-negotiation", version.ref = "ktorClientCore" }
23+
ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktorClientCore" }
24+
ktor-client-darwin = { module = "io.ktor:ktor-client-darwin", version.ref = "ktorClientCore" }
25+
ktor-client-js = { module = "io.ktor:ktor-client-js", version.ref = "ktorClientCore" }
26+
ktor-client-logging = { module = "io.ktor:ktor-client-logging", version.ref = "ktorClientCore" }
27+
ktor-client-plugins = { module = "io.ktor:ktor-client-plugins", version.ref = "ktorClientPlugins" }
28+
29+
30+
logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logbackClassic" }
31+
1132

1233
[plugins]
1334
androidLibrary = { id = "com.android.library", version.ref = "agp" }

‎skainet-data/skainet-data-simple/build.gradle.kts‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,13 @@ kotlin {
2020

2121
iosArm64()
2222
iosSimulatorArm64()
23-
macosArm64 ()
24-
linuxX64 ()
25-
linuxArm64 ()
23+
/*
24+
macosArm64()
25+
26+
linuxX64()
27+
linuxArm64()
28+
29+
*/
2630

2731
jvm()
2832

@@ -33,17 +37,44 @@ kotlin {
3337
}
3438

3539
sourceSets {
40+
androidMain.dependencies {
41+
implementation(libs.ktor.client.android)
42+
}
3643
val commonMain by getting {
3744
dependencies {
3845
implementation(project(":skainet-core:skainet-tensors-api"))
3946
implementation(project(":skainet-core:skainet-tensors-api"))
47+
implementation(libs.kotlinx.serialization.json)
48+
implementation(libs.ktor.client.core)
49+
implementation(libs.kotlinx.coroutines)
50+
4051
}
4152
}
4253

54+
jvmMain.dependencies {
55+
implementation(libs.ktor.client.cio)
56+
implementation(libs.ktor.client.plugins)
57+
implementation(libs.ktor.client.logging)
58+
implementation(libs.ktor.client.content.negotiation)
59+
implementation(libs.kotlinx.coroutines.core.jvm)
60+
implementation(libs.logback.classic) // For logging
61+
}
62+
4363
commonTest.dependencies {
4464
implementation(libs.kotlin.test)
4565
implementation(project(":skainet-core:skainet-performance"))
4666
}
67+
val wasmJsMain by getting {
68+
dependencies {
69+
implementation(libs.ktor.client.js)
70+
}
71+
}
72+
73+
val iosMain by creating {
74+
dependencies {
75+
implementation(libs.ktor.client.darwin)
76+
}
77+
}
4778
}
4879
}
4980

‎skainet-data/skainet-data-simple/src/androidMain/kotlin/sk/ainet/data/mnist/MNISTLoaderAndroid.kt‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package sk.ainet.data.mnist
2+
3+
import io.ktor.client.HttpClient
4+
import io.ktor.client.engine.android.Android
5+
import io.ktor.client.request.get
6+
import io.ktor.client.statement.HttpResponse
7+
import io.ktor.client.call.body
8+
import kotlinx.coroutines.Dispatchers
9+
import kotlinx.coroutines.withContext
10+
import java.io.File
11+
import java.io.FileInputStream
12+
import java.io.FileOutputStream
13+
import java.util.zip.GZIPInputStream
14+
15+
/**
16+
* Android implementation of the MNIST loader.
17+
*
18+
* @property config The configuration for the MNIST loader.
19+
*/
20+
public class MNISTLoaderAndroid(config: MNISTLoaderConfig) : MNISTLoaderCommon(config) {
21+
22+
/**
23+
* Downloads and caches a file.
24+
*
25+
* @param url The URL to download from.
26+
* @param filename The name of the file to save.
27+
* @return The bytes of the decompressed file.
28+
*/
29+
override suspend fun downloadAndCacheFile(url: String, filename: String): ByteArray = withContext(Dispatchers.IO) {
30+
val cacheDir = File(config.cacheDir)
31+
if (!cacheDir.exists()) {
32+
cacheDir.mkdirs()
33+
}
34+
35+
val gzipFile = File(cacheDir, filename)
36+
val decompressedFile = File(cacheDir, filename.removeSuffix(".gz"))
37+
38+
// Check if the decompressed file already exists in cache
39+
if (config.useCache && decompressedFile.exists()) {
40+
println("Using cached file: ${decompressedFile.path}")
41+
return@withContext decompressedFile.readBytes()
42+
}
43+
44+
// Check if the gzip file already exists in cache
45+
if (!gzipFile.exists() || !config.useCache) {
46+
println("Downloading file: $url")
47+
downloadFile(url, gzipFile.path)
48+
} else {
49+
println("Using cached gzip file: ${gzipFile.path}")
50+
}
51+
52+
// Decompress the gzip file
53+
println("Decompressing file: ${gzipFile.path}")
54+
decompressGzipFile(gzipFile.path, decompressedFile.path)
55+
56+
return@withContext decompressedFile.readBytes()
57+
}
58+
59+
/**
60+
* Downloads a file from a URL.
61+
*
62+
* @param url The URL to download from.
63+
* @param outputPath The path to save the file to.
64+
*/
65+
private suspend fun downloadFile(url: String, outputPath: String) {
66+
val client = HttpClient(Android) {
67+
// No plugins needed for basic functionality
68+
}
69+
70+
try {
71+
val file = File(outputPath)
72+
73+
val httpResponse: HttpResponse = client.get(url)
74+
val responseBody: ByteArray = httpResponse.body()
75+
file.writeBytes(responseBody)
76+
77+
println("File saved to ${file.path}")
78+
} finally {
79+
client.close()
80+
}
81+
}
82+
83+
/**
84+
* Decompresses a gzip file.
85+
*
86+
* @param gzipFilePath The path to the gzip file.
87+
* @param outputFilePath The path to save the decompressed file to.
88+
*/
89+
private fun decompressGzipFile(gzipFilePath: String, outputFilePath: String) {
90+
GZIPInputStream(FileInputStream(gzipFilePath)).use { gzipInputStream ->
91+
FileOutputStream(outputFilePath).use { outputStream ->
92+
val buffer = ByteArray(1024)
93+
var len: Int
94+
while (gzipInputStream.read(buffer).also { len = it } > 0) {
95+
outputStream.write(buffer, 0, len)
96+
}
97+
}
98+
}
99+
}
100+
101+
public companion object {
102+
/**
103+
* Creates a new instance of MNISTLoaderAndroid with the default configuration.
104+
*
105+
* @return A new instance of MNISTLoaderAndroid.
106+
*/
107+
public fun create(): MNISTLoaderAndroid {
108+
return MNISTLoaderAndroid(MNISTLoaderConfig())
109+
}
110+
111+
/**
112+
* Creates a new instance of MNISTLoaderAndroid with a custom cache directory.
113+
*
114+
* @param cacheDir The directory to use for caching.
115+
* @return A new instance of MNISTLoaderAndroid.
116+
*/
117+
public fun create(cacheDir: String): MNISTLoaderAndroid {
118+
return MNISTLoaderAndroid(MNISTLoaderConfig(cacheDir = cacheDir))
119+
}
120+
121+
/**
122+
* Creates a new instance of MNISTLoaderAndroid with a custom configuration.
123+
*
124+
* @param config The configuration to use.
125+
* @return A new instance of MNISTLoaderAndroid.
126+
*/
127+
public fun create(config: MNISTLoaderConfig): MNISTLoaderAndroid {
128+
return MNISTLoaderAndroid(config)
129+
}
130+
}
131+
}

‎skainet-data/skainet-data-simple/src/androidMain/kotlin/sk/ainet/data/mnist/MNISTLoaderFactory.kt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package sk.ainet.data.mnist
2+
3+
/**
4+
* Android implementation of the MNIST loader factory.
5+
*/
6+
public actual object MNISTLoaderFactory {
7+
/**
8+
* Creates a new MNIST loader with the default configuration.
9+
*
10+
* @return A new MNIST loader.
11+
*/
12+
public actual fun create(): MNISTLoader {
13+
return MNISTLoaderAndroid.create()
14+
}
15+
16+
/**
17+
* Creates a new MNIST loader with a custom cache directory.
18+
*
19+
* @param cacheDir The directory to use for caching.
20+
* @return A new MNIST loader.
21+
*/
22+
public actual fun create(cacheDir: String): MNISTLoader {
23+
return MNISTLoaderAndroid.create(cacheDir)
24+
}
25+
26+
/**
27+
* Creates a new MNIST loader with a custom configuration.
28+
*
29+
* @param config The configuration to use.
30+
* @return A new MNIST loader.
31+
*/
32+
public actual fun create(config: MNISTLoaderConfig): MNISTLoader {
33+
return MNISTLoaderAndroid.create(config)
34+
}
35+
}

‎skainet-data/skainet-data-simple/src/commonMain/kotlin/sk/ainet/data/mnist/MNISTData.kt‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package sk.ainet.data.mnist
2+
3+
import kotlinx.serialization.Serializable
4+
5+
/**
6+
* Represents a single MNIST image with its label.
7+
*
8+
* @property image The pixel data of the image as a ByteArray (28x28 pixels).
9+
* @property label The label of the image (0-9).
10+
*/
11+
@Serializable
12+
public data class MNISTImage(
13+
val image: ByteArray,
14+
val label: Byte
15+
) {
16+
override fun equals(other: Any?): Boolean {
17+
if (this === other) return true
18+
if (other == null || this::class != other::class) return false
19+
20+
other as MNISTImage
21+
22+
if (!image.contentEquals(other.image)) return false
23+
if (label != other.label) return false
24+
25+
return true
26+
}
27+
28+
override fun hashCode(): Int {
29+
var result = image.contentHashCode()
30+
result = 31 * result + label.toInt()
31+
return result
32+
}
33+
}
34+
35+
/**
36+
* Represents a dataset of MNIST images.
37+
*
38+
* @property images The list of MNIST images.
39+
*/
40+
@Serializable
41+
public data class MNISTDataset(
42+
val images: List<MNISTImage>
43+
) {
44+
/**
45+
* Returns the number of images in the dataset.
46+
*/
47+
val size: Int
48+
get() = images.size
49+
50+
/**
51+
* Returns a subset of the dataset.
52+
*
53+
* @param fromIndex The starting index (inclusive).
54+
* @param toIndex The ending index (exclusive).
55+
* @return A new MNISTDataset containing the specified range of images.
56+
*/
57+
public fun subset(fromIndex: Int, toIndex: Int): MNISTDataset {
58+
return MNISTDataset(images.subList(fromIndex, toIndex))
59+
}
60+
}
61+
62+
/**
63+
* Configuration for the MNIST loader.
64+
*
65+
* @property cacheDir The directory where downloaded files will be cached.
66+
* @property useCache Whether to use cached files if available.
67+
*/
68+
public data class MNISTLoaderConfig(
69+
val cacheDir: String = "mnist-data",
70+
val useCache: Boolean = true
71+
)
72+
73+
/**
74+
* Constants for the MNIST dataset.
75+
*/
76+
public object MNISTConstants {
77+
public const val IMAGE_SIZE: Int = 28
78+
public const val IMAGE_PIXELS: Int = IMAGE_SIZE * IMAGE_SIZE
79+
80+
public const val TRAIN_IMAGES_FILENAME: String = "train-images-idx3-ubyte.gz"
81+
public const val TRAIN_LABELS_FILENAME: String = "train-labels-idx1-ubyte.gz"
82+
public const val TEST_IMAGES_FILENAME: String = "t10k-images-idx3-ubyte.gz"
83+
public const val TEST_LABELS_FILENAME: String = "t10k-labels-idx1-ubyte.gz"
84+
85+
public const val TRAIN_IMAGES_URL: String =
86+
"https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz"
87+
public const val TRAIN_LABELS_URL: String =
88+
"https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz"
89+
public const val TEST_IMAGES_URL: String = "https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz"
90+
public const val TEST_LABELS_URL: String = "https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz"
91+
}
92+
93+
/**
94+
* Interface for the MNIST loader.
95+
*/
96+
public interface MNISTLoader {
97+
/**
98+
* Loads the MNIST training dataset.
99+
*
100+
* @return The MNIST training dataset.
101+
*/
102+
public suspend fun loadTrainingData(): MNISTDataset
103+
104+
/**
105+
* Loads the MNIST test dataset.
106+
*
107+
* @return The MNIST test dataset.
108+
*/
109+
public suspend fun loadTestData(): MNISTDataset
110+
}

0 commit comments

Comments
 (0)