Skip to content

Commit e89ab02

Browse files
frettfrett
authored
Merge pull request #4424 from CruGlobal/cdnResources
GT-2601: Download translation files from CDN with SHA-256 verification
2 parents 9493111 + 82e7fb2 commit e89ab02

8 files changed

Lines changed: 239 additions & 25 deletions

File tree

‎app/build.gradle.kts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ android {
4646
applicationIdSuffix = ".stage"
4747

4848
buildConfigField("String", "MOBILE_CONTENT_API", "\"$URI_MOBILE_CONTENT_API_STAGE\"")
49+
buildConfigField("String", "MOBILE_CONTENT_CDN", "\"https://mobilecontent-stage.cru.org\"")
4950

5051
// Facebook
5152
resValue("string", "facebook_app_id", "448969905944197")
@@ -61,6 +62,7 @@ android {
6162
}
6263
named("production") {
6364
buildConfigField("String", "MOBILE_CONTENT_API", "\"$URI_MOBILE_CONTENT_API_PRODUCTION\"")
65+
buildConfigField("String", "MOBILE_CONTENT_CDN", "\"https://mobilecontent.cru.org\"")
6466

6567
// Facebook
6668
resValue("string", "facebook_app_id", "2236701616451487")

‎app/src/main/kotlin/org/cru/godtools/dagger/ConfigModule.kt‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,15 @@ import dagger.Module
44
import dagger.Provides
55
import dagger.hilt.InstallIn
66
import dagger.hilt.components.SingletonComponent
7-
import javax.inject.Named
87
import org.cru.godtools.BuildConfig
9-
import org.cru.godtools.api.ApiModule
8+
import org.cru.godtools.api.ApiConfig
109

1110
@Module
1211
@InstallIn(SingletonComponent::class)
1312
object ConfigModule {
1413
@get:Provides
15-
@get:Named(ApiModule.MOBILE_CONTENT_API_URL)
16-
val mobileContentApiBaseUrl = BuildConfig.MOBILE_CONTENT_API
14+
val apiConfig = ApiConfig(
15+
mobileContentApiUrl = BuildConfig.MOBILE_CONTENT_API,
16+
cdnUrl = BuildConfig.MOBILE_CONTENT_CDN
17+
)
1718
}

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ facebook = "18.2.3"
2121
facebook-flipper = "0.273.0"
2222
firebase-crashlytics = "20.0.6"
2323
firebase-perf = "22.0.5"
24-
godtoolsShared = "1.3.3-SNAPSHOT"
24+
godtoolsShared = "1.4.0-SNAPSHOT"
2525
gtoSupport = "4.5.1-SNAPSHOT"
2626
kotlin = "2.3.21"
2727
kotlinCoroutines = "1.11.0"

‎library/api/src/main/kotlin/org/cru/godtools/api/ApiConfig.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
package org.cru.godtools.api
2+
3+
data class ApiConfig(val mobileContentApiUrl: String, val cdnUrl: String)

‎library/api/src/main/kotlin/org/cru/godtools/api/ApiModule.kt‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,19 +79,18 @@ object ApiModule {
7979
.build()
8080

8181
// region mobile-content-api APIs
82-
const val MOBILE_CONTENT_API_URL = "MOBILE_CONTENT_API_BASE_URL"
8382
private const val MOBILE_CONTENT_API = "MOBILE_CONTENT_API"
8483
private const val MOBILE_CONTENT_API_AUTHENTICATED = "MOBILE_CONTENT_API_AUTHENTICATED"
8584

8685
@Provides
8786
@Reusable
8887
@Named(MOBILE_CONTENT_API)
8988
fun mobileContentApiRetrofit(
90-
@Named(MOBILE_CONTENT_API_URL) baseUrl: String,
89+
apiConfig: ApiConfig,
9190
jsonApiConverter: JsonApiConverter,
9291
okhttp: OkHttpClient,
9392
): Retrofit = Retrofit.Builder()
94-
.baseUrl(baseUrl)
93+
.baseUrl(apiConfig.mobileContentApiUrl)
9594
.addConverterFactory(LocaleConverterFactory)
9695
.addConverterFactory(JsonApiConverterFactory(jsonApiConverter))
9796
.callFactory(okhttp)
@@ -165,14 +164,16 @@ object ApiModule {
165164
@Provides
166165
@Reusable
167166
fun actionCableScarlet(
168-
@Named(MOBILE_CONTENT_API_URL) baseUrl: String,
167+
apiConfig: ApiConfig,
169168
app: Application,
170169
jsonApi: JsonApiConverter,
171170
okhttp: OkHttpClient,
172171
referenceLifecycle: ReferenceLifecycle,
173172
) = Scarlet.Builder()
174173
.forceDefaultPlatform()
175-
.webSocketFactory(okhttp.newWebSocketFactory(ActionCableRequestFactory("${baseUrl}cable")))
174+
.webSocketFactory(
175+
okhttp.newWebSocketFactory(ActionCableRequestFactory("${apiConfig.mobileContentApiUrl}cable"))
176+
)
176177
.addMessageAdapterFactory(
177178
ActionCableMessageAdapterFactory.Builder()
178179
.addMessageAdapterFactory(JsonApiMessageAdapterFactory(jsonApi))
@@ -195,4 +196,13 @@ object ApiModule {
195196
.callFactory(okhttp)
196197
.build().create<CampaignFormsApi>()
197198
// endregion Adobe APIs
199+
200+
// region CDN APIs
201+
@Provides
202+
@Reusable
203+
fun cdnApi(okhttp: OkHttpClient, apiConfig: ApiConfig): CdnApi = Retrofit.Builder().baseUrl(apiConfig.cdnUrl)
204+
.callFactory(okhttp)
205+
.build()
206+
.create()
207+
// endregion CDN APIs
198208
}

‎library/api/src/main/kotlin/org/cru/godtools/api/CdnApi.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package org.cru.godtools.api
2+
3+
import okhttp3.ResponseBody
4+
import retrofit2.Response
5+
import retrofit2.http.GET
6+
import retrofit2.http.Path
7+
import retrofit2.http.Streaming
8+
9+
interface CdnApi {
10+
@Streaming
11+
@GET("translations/files/{filename}")
12+
suspend fun downloadPublishedFile(@Path("filename") name: String): Response<ResponseBody>
13+
}

‎library/download-manager/src/main/kotlin/org/cru/godtools/downloadmanager/GodToolsDownloadManager.kt‎

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,18 @@ import kotlinx.coroutines.flow.transformLatest
4343
import kotlinx.coroutines.launch
4444
import kotlinx.coroutines.sync.withLock
4545
import kotlinx.coroutines.withContext
46+
import okhttp3.ResponseBody
47+
import okio.ByteString.Companion.decodeHex
48+
import okio.HashingSource.Companion.sha256
49+
import okio.buffer
50+
import okio.sink
4651
import org.ccci.gto.android.common.kotlin.coroutines.MutexMap
4752
import org.ccci.gto.android.common.kotlin.coroutines.ReadWriteMutex
4853
import org.ccci.gto.android.common.kotlin.coroutines.flow.EmptyStateFlow
4954
import org.ccci.gto.android.common.kotlin.coroutines.flow.combineTransformLatest
5055
import org.ccci.gto.android.common.kotlin.coroutines.withLock
5156
import org.cru.godtools.api.AttachmentsApi
57+
import org.cru.godtools.api.CdnApi
5258
import org.cru.godtools.api.TranslationsApi
5359
import org.cru.godtools.base.Settings
5460
import org.cru.godtools.base.ToolFileSystem
@@ -66,6 +72,7 @@ import org.cru.godtools.model.Translation
6672
import org.cru.godtools.model.TranslationKey
6773
import org.cru.godtools.shared.tool.parser.ManifestParser
6874
import org.cru.godtools.shared.tool.parser.ParserResult
75+
import retrofit2.Response
6976

7077
@VisibleForTesting
7178
internal const val CLEANUP_DELAY = 30_000L
@@ -74,6 +81,7 @@ internal const val CLEANUP_DELAY = 30_000L
7481
class GodToolsDownloadManager @VisibleForTesting internal constructor(
7582
private val attachmentsApi: AttachmentsApi,
7683
private val attachmentsRepository: AttachmentsRepository,
84+
private val cdnApi: CdnApi,
7785
private val downloadedFilesRepository: DownloadedFilesRepository,
7886
private val fs: ToolFileSystem,
7987
private val manifestParser: ManifestParser,
@@ -87,6 +95,7 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
8795
internal constructor(
8896
attachmentsApi: AttachmentsApi,
8997
attachmentsRepository: AttachmentsRepository,
98+
cdnApi: CdnApi,
9099
downloadedFilesRepository: DownloadedFilesRepository,
91100
fs: ToolFileSystem,
92101
manifestParser: ManifestParser,
@@ -96,6 +105,7 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
96105
) : this(
97106
attachmentsApi,
98107
attachmentsRepository,
108+
cdnApi,
99109
downloadedFilesRepository,
100110
fs,
101111
manifestParser,
@@ -280,7 +290,7 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
280290
private suspend fun downloadTranslationFiles(translation: Translation): Boolean = filesystemMutex.read.withLock {
281291
// download manifest if necessary
282292
val manifestFileName = translation.manifestFileName ?: return false
283-
if (!downloadTranslationFileIfNecessary(manifestFileName)) return false
293+
if (!downloadPublishedFileIfNecessary(manifestFileName)) return false
284294

285295
// parse manifest
286296
val parserResult = manifestParser.parseManifest(
@@ -294,9 +304,14 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
294304
val relatedFiles = manifest.relatedFiles
295305
val completedFiles = AtomicLong(0)
296306
val successful = coroutineScope {
297-
relatedFiles.map {
307+
relatedFiles.map { file ->
298308
async {
299-
downloadTranslationFileIfNecessary(it).also {
309+
val src = file.src ?: return@async true
310+
downloadPublishedFileIfNecessary(
311+
fileName = src,
312+
sha256 = file.checksumSha256,
313+
size = file.size?.toLong()
314+
).also {
300315
do {
301316
val completed = completedFiles.get()
302317
updateProgress(key, completed + 1, relatedFiles.size.toLong())
@@ -309,23 +324,61 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
309324

310325
// record the translation as downloaded
311326
downloadedFilesRepository.insertOrIgnore(DownloadedTranslationFile(translation, manifestFileName))
312-
relatedFiles.forEach { downloadedFilesRepository.insertOrIgnore(DownloadedTranslationFile(translation, it)) }
327+
relatedFiles.mapNotNull { it.src }.forEach {
328+
downloadedFilesRepository.insertOrIgnore(DownloadedTranslationFile(translation, it))
329+
}
313330
translationsRepository.markTranslationDownloaded(translation.id, true)
314331

315332
return true
316333
}
317334

318-
private suspend fun downloadTranslationFileIfNecessary(fileName: String): Boolean = filesMutex.withLock(fileName) {
335+
private suspend fun downloadPublishedFileIfNecessary(
336+
fileName: String,
337+
sha256: String? = null,
338+
size: Long? = null,
339+
): Boolean = filesMutex[fileName].withLock {
319340
if (downloadedFilesRepository.findDownloadedFile(fileName) != null) return true
320-
try {
321-
val body = translationsApi.downloadFile(fileName).takeIf { it.isSuccessful }?.body() ?: return false
322-
val downloadedFile = DownloadedFile(fileName)
323-
withContext(Dispatchers.IO) { body.byteStream().copyTo(downloadedFile) }
324-
downloadedFilesRepository.insertOrIgnore(downloadedFile)
325-
return true
326-
} catch (e: IOException) {
341+
342+
withContext(ioDispatcher) {
343+
downloadPublishedFileFromCdn(fileName, sha256, size) ||
344+
downloadPublishedFileFromApi(fileName, sha256, size)
345+
}
346+
}
347+
348+
private suspend fun downloadPublishedFileFromCdn(fileName: String, sha256: String?, size: Long?) = try {
349+
cdnApi.downloadPublishedFile(fileName).storeFile(fileName, sha256, size)
350+
} catch (_: IOException) {
351+
false
352+
}
353+
354+
private suspend fun downloadPublishedFileFromApi(fileName: String, sha256: String?, size: Long?) = try {
355+
translationsApi.downloadFile(fileName).storeFile(fileName, sha256, size)
356+
} catch (_: IOException) {
357+
false
358+
}
359+
360+
private suspend fun Response<ResponseBody>.storeFile(fileName: String, sha256: String?, size: Long?): Boolean {
361+
val body = body().takeIf { isSuccessful } ?: return false
362+
363+
val downloadedFile = DownloadedFile(fileName)
364+
val valid = downloadedFile.bufferedSink().use { sink ->
365+
body.source().use { source ->
366+
val digest = sha256(source)
367+
val bytesWritten = sink.writeAll(digest)
368+
when {
369+
size != null && size != bytesWritten -> false
370+
sha256 != null && sha256.decodeHex() != digest.hash -> false
371+
else -> true
372+
}
373+
}
374+
}
375+
if (!valid) {
376+
downloadedFile.getFile(fs).delete()
327377
return false
328378
}
379+
380+
downloadedFilesRepository.insertOrIgnore(downloadedFile)
381+
return true
329382
}
330383

331384
private suspend fun downloadTranslationZip(translation: Translation) = try {
@@ -450,6 +503,8 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
450503
}
451504
// endregion Cleanup
452505

506+
private suspend fun DownloadedFile.bufferedSink() = withContext(Dispatchers.IO) { getFile(fs).sink().buffer() }
507+
453508
private suspend fun InputStream.copyTo(file: DownloadedFile) = withContext(Dispatchers.IO) {
454509
file.getFile(fs).outputStream().use { copyTo(it) }
455510
}

0 commit comments

Comments
 (0)