Skip to content

Commit 73fd383

Browse files
michalharakalmichalharakal
committed
Add initial version of performance measurement
Related-To: #36
1 parent 7bd2ce3 commit 73fd383

6 files changed

Lines changed: 411 additions & 0 deletions

File tree

‎docs/modules/performance/nav.adoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
* xref:index.adoc[Performance Measurement]
2+
** xref:benchmark-framework.adoc[Benchmark Framework]
3+
** xref:performance-testing.adoc[Performance Testing Guide]
4+
** xref:metrics-analysis.adoc[Metrics and Analysis]

‎docs/modules/performance/pages/index.adoc‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
= Performance Measurement Framework
2+
:toc:
3+
:toclevels: 3
4+
:sectanchors:
5+
:sectlinks:
6+
7+
The SKaiNET Performance Measurement Framework provides comprehensive tools for benchmarking tensor operations and analyzing computational performance across different backends and configurations.
8+
9+
== Overview
10+
11+
The performance measurement framework is designed to:
12+
13+
* Measure execution time and throughput of tensor operations
14+
* Compare performance between different compute backends
15+
* Provide statistical analysis of benchmark results
16+
* Generate detailed performance reports
17+
* Support multiplatform benchmarking
18+
19+
== Key Components
20+
21+
=== BenchmarkRunner
22+
23+
The `BenchmarkRunner` class is the core component that executes performance measurements:
24+
25+
[source,kotlin]
26+
----
27+
val runner = BenchmarkRunner()
28+
val result = runner.benchmark(
29+
name = "Matrix Multiplication 256x256",
30+
warmupRuns = 10,
31+
measurementRuns = 100
32+
) {
33+
// Your operation to benchmark
34+
backend.matmul(matrixA, matrixB)
35+
}
36+
----
37+
38+
=== Benchmark Results
39+
40+
The framework provides comprehensive result analysis:
41+
42+
* **TimeStatistics**: Mean, median, standard deviation, min/max execution times
43+
* **MemoryStatistics**: Memory usage tracking (platform-dependent)
44+
* **BenchmarkResult**: Complete benchmark results with metadata
45+
* **BenchmarkReport**: Comprehensive reports for multiple operations
46+
47+
=== Performance Analysis
48+
49+
Advanced analysis capabilities include:
50+
51+
* Statistical analysis of execution times
52+
* Throughput calculations (operations per second)
53+
* Performance comparison between backends
54+
* Speedup analysis and regression detection
55+
56+
== Quick Start
57+
58+
. Add the performance module dependency to your test configuration
59+
. Create a `BenchmarkRunner` instance
60+
. Use the `benchmark()` method to measure your operations
61+
. Analyze results using the provided data classes
62+
63+
== Module Structure
64+
65+
The performance module (`skainet-core:skainet-performance`) contains:
66+
67+
* Core benchmarking infrastructure
68+
* Result analysis and reporting tools
69+
* Statistical utilities
70+
* Multiplatform compatibility layer

‎skainet-core/skainet-performance/build.gradle.kts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi
2+
import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
3+
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
4+
5+
plugins {
6+
alias(libs.plugins.kotlinMultiplatform)
7+
alias(libs.plugins.androidLibrary)
8+
alias(libs.plugins.vanniktech.mavenPublish)
9+
}
10+
11+
kotlin {
12+
explicitApi()
13+
14+
androidTarget {
15+
@OptIn(ExperimentalKotlinGradlePluginApi::class)
16+
compilerOptions {
17+
jvmTarget.set(JvmTarget.JVM_11)
18+
}
19+
}
20+
21+
iosArm64()
22+
iosSimulatorArm64()
23+
macosArm64 ()
24+
linuxX64 ()
25+
linuxArm64 ()
26+
27+
jvm()
28+
29+
@OptIn(ExperimentalWasmDsl::class)
30+
wasmJs {
31+
browser()
32+
binaries.executable()
33+
}
34+
35+
sourceSets {
36+
val commonMain by getting {
37+
dependencies {
38+
implementation(project(":skainet-core:skainet-tensors-api"))
39+
}
40+
}
41+
42+
commonTest.dependencies {
43+
implementation(libs.kotlin.test)
44+
}
45+
}
46+
}
47+
48+
android {
49+
namespace = "sk.ai.net.core.performance"
50+
compileSdk = libs.versions.android.compileSdk.get().toInt()
51+
52+
defaultConfig {
53+
minSdk = libs.versions.android.minSdk.get().toInt()
54+
}
55+
compileOptions {
56+
sourceCompatibility = JavaVersion.VERSION_11
57+
targetCompatibility = JavaVersion.VERSION_11
58+
}
59+
}

‎skainet-core/skainet-performance/gradle.properties‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
POM_ARTIFACT_ID=skainet-performance
2+
POM_NAME=SKaiNET Performance Measurement Framework
3+
POM_DESCRIPTION=Core performance measurement and benchmarking utilities for SKaiNET tensor operations

‎skainet-core/skainet-performance/src/commonMain/kotlin/sk/ainet/core/performance/BenchmarkResult.kt‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package sk.ainet.core.performance
2+
3+
/**
4+
* Comprehensive benchmark result containing timing statistics
5+
*/
6+
public data class BenchmarkResult(
7+
val name: String,
8+
val executionTime: TimeStatistics,
9+
val memoryUsage: MemoryStatistics,
10+
val throughput: Double, // operations per second
11+
val metadata: Map<String, Any>
12+
) {
13+
public fun prettyPrint(): String {
14+
return buildString {
15+
appendLine("$name:")
16+
appendLine(" Time: ${(executionTime.mean / 1000).format(3)}ms ± ${(executionTime.standardDeviation / 1000).format(3)}ms")
17+
appendLine(" Throughput: ${throughput.format(1)} ops/sec")
18+
appendLine(" Memory: ${formatBytes(memoryUsage.allocatedBytes)}")
19+
appendLine(" Range: ${(executionTime.min / 1000).format(3)}ms - ${(executionTime.max / 1000).format(3)}ms")
20+
}
21+
}
22+
23+
private fun formatBytes(bytes: Long): String {
24+
return when {
25+
bytes >= 1_000_000 -> "${(bytes / 1_000_000.0).format(1)}MB"
26+
bytes >= 1_000 -> "${(bytes / 1_000.0).format(1)}KB"
27+
else -> "${bytes}B"
28+
}
29+
}
30+
31+
private fun Double.format(decimals: Int): String {
32+
val multiplier = when (decimals) {
33+
0 -> 1.0
34+
1 -> 10.0
35+
2 -> 100.0
36+
3 -> 1000.0
37+
else -> 1000.0
38+
}
39+
val rounded = (this * multiplier).toInt() / multiplier
40+
return rounded.toString()
41+
}
42+
}
43+
44+
/**
45+
* Statistical analysis of execution times
46+
*/
47+
public data class TimeStatistics(
48+
val mean: Double, // microseconds
49+
val median: Double, // microseconds
50+
val standardDeviation: Double, // microseconds
51+
val min: Double, // microseconds
52+
val max: Double, // microseconds
53+
val percentile95: Double // microseconds
54+
)
55+
56+
/**
57+
* Memory usage statistics (placeholder for multiplatform compatibility)
58+
*/
59+
public data class MemoryStatistics(
60+
val allocatedBytes: Long,
61+
val peakHeapUsage: Long,
62+
val gcCollections: Int,
63+
val gcTime: Long // milliseconds
64+
)
65+
66+
/**
67+
* Comprehensive benchmark report containing all operation results
68+
*/
69+
public data class BenchmarkReport(
70+
val backendName: String,
71+
val matrixMultiplication: List<BenchmarkResult>,
72+
val elementwiseOperations: Map<String, List<BenchmarkResult>>,
73+
val dotProduct: List<BenchmarkResult>,
74+
val scalarOperations: Map<String, List<BenchmarkResult>>,
75+
val summary: BenchmarkSummary
76+
) {
77+
public fun prettyPrint(): String {
78+
return buildString {
79+
appendLine("=" .repeat(50))
80+
appendLine("TENSOR BENCHMARK REPORT - $backendName")
81+
appendLine("=" .repeat(50))
82+
appendLine()
83+
84+
appendLine("MATRIX MULTIPLICATION:")
85+
matrixMultiplication.forEach { result ->
86+
appendLine(result.prettyPrint())
87+
}
88+
appendLine()
89+
90+
appendLine("ELEMENT-WISE OPERATIONS:")
91+
elementwiseOperations.forEach { (opType, results) ->
92+
appendLine(" $opType:")
93+
results.forEach { result ->
94+
appendLine(" ${result.prettyPrint().prependIndent(" ")}")
95+
}
96+
}
97+
appendLine()
98+
99+
appendLine("SUMMARY:")
100+
appendLine(" Total Operations: ${summary.totalOperations}")
101+
appendLine(" Average Throughput: ${summary.averageThroughput.format(1)} ops/sec")
102+
appendLine(" Fastest: ${summary.fastestOperation}")
103+
appendLine(" Slowest: ${summary.slowestOperation}")
104+
}
105+
}
106+
107+
private fun Double.format(decimals: Int): String {
108+
val multiplier = when (decimals) {
109+
0 -> 1.0
110+
1 -> 10.0
111+
2 -> 100.0
112+
3 -> 1000.0
113+
else -> 1000.0
114+
}
115+
val rounded = (this * multiplier).toInt() / multiplier
116+
return rounded.toString()
117+
}
118+
}
119+
120+
/**
121+
* Summary statistics for a benchmark run
122+
*/
123+
public data class BenchmarkSummary(
124+
val totalOperations: Int,
125+
val averageThroughput: Double,
126+
val fastestOperation: String,
127+
val slowestOperation: String
128+
)
129+
130+
/**
131+
* Backend comparison report showing speedup analysis
132+
*/
133+
public data class ComparisonReport(
134+
val baselineBackend: String,
135+
val baselineResults: BenchmarkReport,
136+
val comparisonBackend: String,
137+
val comparisonResults: BenchmarkReport,
138+
val speedupAnalysis: SpeedupAnalysis
139+
) {
140+
public fun prettyPrint(): String {
141+
return buildString {
142+
appendLine("=" .repeat(60))
143+
appendLine("BACKEND COMPARISON: $baselineBackend vs $comparisonBackend")
144+
appendLine("=" .repeat(60))
145+
appendLine()
146+
147+
appendLine("SPEEDUP ANALYSIS:")
148+
appendLine(" Average Speedup: ${speedupAnalysis.averageSpeedup.format(2)}x")
149+
appendLine(" Best Case: ${speedupAnalysis.bestCaseSpeedup.format(2)}x")
150+
appendLine(" Worst Case: ${speedupAnalysis.worstCaseSpeedup.format(2)}x")
151+
appendLine()
152+
153+
appendLine("DETAILED SPEEDUPS:")
154+
speedupAnalysis.operationSpeedups.forEach { (operation, speedup) ->
155+
appendLine(" $operation: ${speedup.format(2)}x")
156+
}
157+
}
158+
}
159+
160+
private fun Double.format(decimals: Int): String {
161+
val multiplier = when (decimals) {
162+
0 -> 1.0
163+
1 -> 10.0
164+
2 -> 100.0
165+
3 -> 1000.0
166+
else -> 1000.0
167+
}
168+
val rounded = (this * multiplier).toInt() / multiplier
169+
return rounded.toString()
170+
}
171+
}
172+
173+
/**
174+
* Speedup analysis comparing two backends
175+
*/
176+
public data class SpeedupAnalysis(
177+
val operationSpeedups: Map<String, Double>,
178+
val averageSpeedup: Double,
179+
val bestCaseSpeedup: Double,
180+
val worstCaseSpeedup: Double,
181+
val speedupByTensorSize: Map<String, Double>
182+
)

0 commit comments

Comments
 (0)