Skip to content

Commit bb33723

Browse files
mudkipmemudkipme
authored
Merge pull request #342 from mudkipme/fix/upload-big-files
fix: stream uploading big files
2 parents 595a3d3 + 90db848 commit bb33723

18 files changed

Lines changed: 185 additions & 42 deletions

‎app/src/main/java/me/mudkip/moememos/data/api/MemosV1Api.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import androidx.core.net.toUri
55
import com.skydoves.sandwich.ApiResponse
66
import kotlinx.serialization.SerialName
77
import kotlinx.serialization.Serializable
8+
import okhttp3.RequestBody
89
import retrofit2.http.Body
910
import retrofit2.http.DELETE
1011
import retrofit2.http.GET
@@ -42,7 +43,7 @@ interface MemosV1Api {
4243
suspend fun listResources(): ApiResponse<ListResourceResponse>
4344

4445
@POST("api/v1/attachments")
45-
suspend fun createResource(@Body body: CreateResourceRequest): ApiResponse<MemosV1Resource>
46+
suspend fun createResource(@Body body: RequestBody): ApiResponse<MemosV1Resource>
4647

4748
@DELETE("api/v1/attachments/{id}")
4849
suspend fun deleteResource(@Path("id") resourceId: String): ApiResponse<Unit>

‎app/src/main/java/me/mudkip/moememos/data/local/FileStorage.kt‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import android.content.Context
44
import android.net.Uri
55
import dagger.hilt.android.qualifiers.ApplicationContext
66
import java.io.File
7+
import java.io.InputStream
78
import java.util.Base64
89
import javax.inject.Inject
910
import javax.inject.Singleton
@@ -20,8 +21,28 @@ class FileStorage @Inject constructor(
2021
}
2122

2223
fun saveFile(accountKey: String, content: ByteArray, filename: String): Uri {
24+
return saveFile(accountKey, filename) { output ->
25+
output.write(content)
26+
}
27+
}
28+
29+
fun saveFile(accountKey: String, sourceUri: Uri, filename: String): Uri {
30+
val inputStream = context.contentResolver.openInputStream(sourceUri)
31+
?: throw IllegalArgumentException("Unable to open URI for reading: $sourceUri")
32+
inputStream.use { input ->
33+
return saveFile(accountKey, input, filename)
34+
}
35+
}
36+
37+
fun saveFile(accountKey: String, input: InputStream, filename: String): Uri {
38+
return saveFile(accountKey, filename) { output ->
39+
input.copyTo(output)
40+
}
41+
}
42+
43+
private fun saveFile(accountKey: String, filename: String, writer: (java.io.OutputStream) -> Unit): Uri {
2344
val file = File(accountDir(accountKey), filename)
24-
file.writeBytes(content)
45+
file.outputStream().use(writer)
2546
return Uri.fromFile(file)
2647
}
2748

‎app/src/main/java/me/mudkip/moememos/data/repository/AbstractMemoRepository.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ abstract class AbstractMemoRepository {
2929
abstract suspend fun listTags(): ApiResponse<List<String>>
3030

3131
abstract suspend fun listResources(): ApiResponse<List<ResourceEntity>>
32-
abstract suspend fun createResource(filename: String, type: MediaType?, content: ByteArray, memoIdentifier: String? = null): ApiResponse<ResourceEntity>
32+
abstract suspend fun createResource(filename: String, type: MediaType?, contentUri: Uri, memoIdentifier: String? = null): ApiResponse<ResourceEntity>
3333
abstract suspend fun deleteResource(identifier: String): ApiResponse<Unit>
3434

3535
abstract suspend fun getCurrentUser(): ApiResponse<User>

‎app/src/main/java/me/mudkip/moememos/data/repository/LocalDatabaseRepository.kt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package me.mudkip.moememos.data.repository
22

3+
import android.net.Uri
34
import androidx.core.net.toUri
45
import com.skydoves.sandwich.ApiResponse
56
import kotlinx.coroutines.flow.Flow
@@ -214,13 +215,13 @@ class LocalDatabaseRepository(
214215
override suspend fun createResource(
215216
filename: String,
216217
type: MediaType?,
217-
content: ByteArray,
218+
contentUri: Uri,
218219
memoIdentifier: String?
219220
): ApiResponse<ResourceEntity> {
220221
return try {
221222
val uri = fileStorage.saveFile(
222223
accountKey = accountKey,
223-
content = content,
224+
sourceUri = contentUri,
224225
filename = UUID.randomUUID().toString() + "_" + filename
225226
)
226227
val resource = ResourceEntity(

‎app/src/main/java/me/mudkip/moememos/data/repository/MemosV0Repository.kt‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import me.mudkip.moememos.data.model.Resource
1919
import me.mudkip.moememos.data.model.User
2020
import okhttp3.MediaType
2121
import okhttp3.MultipartBody
22-
import okhttp3.RequestBody.Companion.toRequestBody
2322
import java.time.Instant
2423

2524
class MemosV0Repository (
@@ -149,10 +148,15 @@ class MemosV0Repository (
149148
override suspend fun createResource(
150149
filename: String,
151150
type: MediaType?,
152-
content: ByteArray,
151+
contentLength: Long?,
152+
openInputStream: () -> java.io.InputStream,
153153
memoRemoteId: String?
154154
): ApiResponse<Resource> {
155-
val file = MultipartBody.Part.createFormData("file", filename, content.toRequestBody(type))
155+
val file = MultipartBody.Part.createFormData(
156+
"file",
157+
filename,
158+
InputStreamRequestBody(type, contentLength, openInputStream)
159+
)
156160
return memosApi.uploadResource(file).mapSuccess {
157161
convertResource(this)
158162
}

‎app/src/main/java/me/mudkip/moememos/data/repository/MemosV1Repository.kt‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import com.skydoves.sandwich.onSuccess
77
import kotlinx.coroutines.async
88
import kotlinx.coroutines.awaitAll
99
import kotlinx.coroutines.coroutineScope
10-
import me.mudkip.moememos.data.api.CreateResourceRequest
1110
import me.mudkip.moememos.data.api.MemosV1Api
1211
import me.mudkip.moememos.data.api.MemosV1CreateMemoRequest
1312
import me.mudkip.moememos.data.api.MemosV1Memo
@@ -22,7 +21,6 @@ import me.mudkip.moememos.data.model.MemoVisibility
2221
import me.mudkip.moememos.data.model.Resource
2322
import me.mudkip.moememos.data.model.User
2423
import okhttp3.MediaType
25-
import okio.ByteString.Companion.toByteString
2624
import java.time.Instant
2725

2826
private const val PAGE_SIZE = 200
@@ -176,15 +174,18 @@ class MemosV1Repository(
176174
override suspend fun createResource(
177175
filename: String,
178176
type: MediaType?,
179-
content: ByteArray,
177+
contentLength: Long?,
178+
openInputStream: () -> java.io.InputStream,
180179
memoRemoteId: String?
181180
): ApiResponse<Resource> {
182-
return memosApi.createResource(CreateResourceRequest(
181+
val requestBody = StreamingBase64JsonRequestBody(
183182
filename = filename,
184183
type = type?.toString() ?: "application/octet-stream",
185-
content = content.toByteString().base64(),
186-
memo = memoRemoteId?.let { getName(it) }
187-
)).mapSuccess { convertResource(this) }
184+
memo = memoRemoteId?.let { getName(it) },
185+
contentLength = contentLength,
186+
openInputStream = openInputStream
187+
)
188+
return memosApi.createResource(requestBody).mapSuccess { convertResource(this) }
188189
}
189190

190191
override suspend fun deleteResource(remoteId: String): ApiResponse<Unit> {

‎app/src/main/java/me/mudkip/moememos/data/repository/RemoteRepository.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import me.mudkip.moememos.data.model.MemoVisibility
66
import me.mudkip.moememos.data.model.Resource
77
import me.mudkip.moememos.data.model.User
88
import okhttp3.MediaType
9+
import java.io.InputStream
910
import java.time.Instant
1011

1112
abstract class RemoteRepository {
@@ -39,7 +40,8 @@ abstract class RemoteRepository {
3940
abstract suspend fun createResource(
4041
filename: String,
4142
type: MediaType?,
42-
content: ByteArray,
43+
contentLength: Long?,
44+
openInputStream: () -> InputStream,
4345
memoRemoteId: String? = null
4446
): ApiResponse<Resource>
4547

‎app/src/main/java/me/mudkip/moememos/data/repository/StreamingRequestBodies.kt‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package me.mudkip.moememos.data.repository
2+
3+
import android.util.Base64
4+
import android.util.Base64OutputStream
5+
import kotlinx.serialization.encodeToString
6+
import kotlinx.serialization.json.Json
7+
import okhttp3.MediaType
8+
import okhttp3.MediaType.Companion.toMediaType
9+
import okhttp3.RequestBody
10+
import okio.BufferedSink
11+
import okio.source
12+
import java.io.InputStream
13+
import java.io.OutputStream
14+
15+
class InputStreamRequestBody(
16+
private val mediaType: MediaType?,
17+
private val length: Long?,
18+
private val openInputStream: () -> InputStream
19+
) : RequestBody() {
20+
override fun contentType(): MediaType? = mediaType
21+
22+
override fun contentLength(): Long {
23+
return length ?: -1L
24+
}
25+
26+
override fun writeTo(sink: BufferedSink) {
27+
openInputStream().use { input ->
28+
sink.writeAll(input.source())
29+
}
30+
}
31+
}
32+
33+
class StreamingBase64JsonRequestBody(
34+
filename: String,
35+
type: String,
36+
memo: String?,
37+
private val contentLength: Long?,
38+
private val openInputStream: () -> InputStream
39+
) : RequestBody() {
40+
private val prefix: String
41+
private val suffix: String
42+
43+
init {
44+
val quotedFilename = Json.encodeToString(filename)
45+
val quotedType = Json.encodeToString(type)
46+
prefix = buildString {
47+
append('{')
48+
append("\"filename\":")
49+
append(quotedFilename)
50+
append(',')
51+
append("\"type\":")
52+
append(quotedType)
53+
append(',')
54+
append("\"content\":\"")
55+
}
56+
suffix = buildString {
57+
append('\"')
58+
if (memo != null) {
59+
append(',')
60+
append("\"memo\":")
61+
append(Json.encodeToString(memo))
62+
}
63+
append('}')
64+
}
65+
}
66+
67+
override fun contentType(): MediaType {
68+
return "application/json; charset=utf-8".toMediaType()
69+
}
70+
71+
override fun contentLength(): Long {
72+
val rawLength = contentLength
73+
if (rawLength == null || rawLength < 0L) {
74+
return -1L
75+
}
76+
val encodedLength = ((rawLength + 2L) / 3L) * 4L
77+
return prefix.encodeToByteArray().size.toLong() + encodedLength + suffix.encodeToByteArray().size.toLong()
78+
}
79+
80+
override fun writeTo(sink: BufferedSink) {
81+
sink.writeUtf8(prefix)
82+
openInputStream().use { input ->
83+
val stream = Base64OutputStream(NoCloseOutputStream(sink.outputStream()), Base64.NO_WRAP)
84+
stream.use { base64 ->
85+
input.copyTo(base64)
86+
}
87+
}
88+
sink.writeUtf8(suffix)
89+
}
90+
}
91+
92+
private class NoCloseOutputStream(
93+
private val delegate: OutputStream
94+
) : OutputStream() {
95+
override fun write(b: Int) {
96+
delegate.write(b)
97+
}
98+
99+
override fun write(b: ByteArray) {
100+
delegate.write(b)
101+
}
102+
103+
override fun write(b: ByteArray, off: Int, len: Int) {
104+
delegate.write(b, off, len)
105+
}
106+
107+
override fun flush() {
108+
delegate.flush()
109+
}
110+
111+
override fun close() {
112+
flush()
113+
}
114+
}

‎app/src/main/java/me/mudkip/moememos/data/repository/SyncingRepository.kt‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@ import kotlinx.coroutines.flow.update
1919
import kotlinx.coroutines.launch
2020
import kotlinx.coroutines.sync.Mutex
2121
import kotlinx.coroutines.sync.withLock
22+
import me.mudkip.moememos.data.constant.MoeMemosException
2223
import me.mudkip.moememos.data.local.FileStorage
2324
import me.mudkip.moememos.data.local.dao.MemoDao
2425
import me.mudkip.moememos.data.local.entity.MemoEntity
2526
import me.mudkip.moememos.data.local.entity.MemoWithResources
2627
import me.mudkip.moememos.data.local.entity.ResourceEntity
27-
import me.mudkip.moememos.data.constant.MoeMemosException
2828
import me.mudkip.moememos.data.model.Account
2929
import me.mudkip.moememos.data.model.Memo
3030
import me.mudkip.moememos.data.model.MemoVisibility
@@ -264,13 +264,13 @@ class SyncingRepository(
264264
override suspend fun createResource(
265265
filename: String,
266266
type: MediaType?,
267-
content: ByteArray,
267+
contentUri: Uri,
268268
memoIdentifier: String?
269269
): ApiResponse<ResourceEntity> {
270270
return try {
271271
val uri = fileStorage.saveFile(
272272
accountKey = accountKey,
273-
content = content,
273+
sourceUri = contentUri,
274274
filename = UUID.randomUUID().toString() + "_" + filename
275275
)
276276
val resource = ResourceEntity(
@@ -355,7 +355,7 @@ class SyncingRepository(
355355

356356
val canonical = fileStorage.saveFile(
357357
accountKey = accountKey,
358-
content = sourceFile.readBytes(),
358+
input = sourceFile.inputStream(),
359359
filename = "${resource.identifier}_${resource.filename}"
360360
).toString()
361361

@@ -699,7 +699,8 @@ class SyncingRepository(
699699
val uploaded = remoteRepository.createResource(
700700
filename = resource.filename,
701701
type = resource.mimeType?.toMediaTypeOrNull(),
702-
content = file.readBytes(),
702+
contentLength = file.length(),
703+
openInputStream = { file.inputStream() },
703704
memoRemoteId = memoRemoteId
704705
)
705706

‎app/src/main/java/me/mudkip/moememos/data/service/AccountService.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ import com.skydoves.sandwich.getOrNull
88
import com.skydoves.sandwich.getOrThrow
99
import com.skydoves.sandwich.retrofit.adapters.ApiResponseCallAdapterFactory
1010
import dagger.hilt.android.qualifiers.ApplicationContext
11+
import kotlinx.coroutines.CompletableDeferred
1112
import kotlinx.coroutines.CoroutineScope
1213
import kotlinx.coroutines.Dispatchers
1314
import kotlinx.coroutines.SupervisorJob
14-
import kotlinx.coroutines.CompletableDeferred
1515
import kotlinx.coroutines.flow.first
1616
import kotlinx.coroutines.flow.map
1717
import kotlinx.coroutines.launch
@@ -35,8 +35,8 @@ import me.mudkip.moememos.data.repository.MemosV0Repository
3535
import me.mudkip.moememos.data.repository.MemosV1Repository
3636
import me.mudkip.moememos.data.repository.RemoteRepository
3737
import me.mudkip.moememos.data.repository.SyncingRepository
38-
import me.mudkip.moememos.ext.string
3938
import me.mudkip.moememos.ext.settingsDataStore
39+
import me.mudkip.moememos.ext.string
4040
import net.swiftzer.semver.SemVer
4141
import okhttp3.HttpUrl
4242
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull

0 commit comments

Comments
 (0)