Merge pull request #3801 from kiwix/Fix#3511

Fixed: Not able to upload 512MB+ custom apps anymore.
kiwix · Jun 15, 2024 · ab5fb35 · ab5fb35
2 parents d099876 + 14b782a
commit ab5fb35
Show file tree

Hide file tree

Showing 12 changed files with 113 additions and 38 deletions.
diff --git a/app/src/androidTest/java/org/kiwix/kiwixmobile/mimetype/MimeTypeTest.kt b/app/src/androidTest/java/org/kiwix/kiwixmobile/mimetype/MimeTypeTest.kt
@@ -84,7 +84,7 @@ class MimeTypeTest : BaseActivityTest() {
     val archive = Archive(zimFile.canonicalPath)
     val zimFileReader = ZimFileReader(
       zimFile,
-      null,
+      emptyList(),
       null,
       archive,
       NightModeConfig(SharedPreferenceUtil(context), context),

diff --git a/app/src/androidTest/java/org/kiwix/kiwixmobile/reader/EncodedUrlTest.kt b/app/src/androidTest/java/org/kiwix/kiwixmobile/reader/EncodedUrlTest.kt
@@ -92,7 +92,7 @@ class EncodedUrlTest : BaseActivityTest() {
     val archive = Archive(zimFile.canonicalPath)
     val zimFileReader = ZimFileReader(
       zimFile,
-      null,
+      emptyList(),
       null,
       archive,
       NightModeConfig(SharedPreferenceUtil(context), context),

diff --git a/...src/androidTest/java/org/kiwix/kiwixmobile/reader/ZimFileReaderWithSplittedZimFileTest.kt b/...src/androidTest/java/org/kiwix/kiwixmobile/reader/ZimFileReaderWithSplittedZimFileTest.kt
@@ -121,7 +121,7 @@ class ZimFileReaderWithSplittedZimFileTest : BaseActivityTest() {
       val archive = Archive(zimFile.canonicalPath)
       val zimFileReader = ZimFileReader(
         zimFile,
-        null,
+        emptyList(),
         null,
         archive,
         NightModeConfig(SharedPreferenceUtil(context), context),

diff --git a/app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt b/app/src/main/java/org/kiwix/kiwixmobile/nav/destination/reader/KiwixReaderFragment.kt
@@ -199,7 +199,7 @@ class KiwixReaderFragment : CoreReaderFragment() {
   override fun onResume() {
     super.onResume()
     if (zimReaderContainer?.zimFile == null &&
-      zimReaderContainer?.zimFileReader?.assetFileDescriptor == null
+      zimReaderContainer?.zimFileReader?.assetFileDescriptorList?.isEmpty() == true
     ) {
       exitBook()
     }

diff --git a/buildSrc/src/main/kotlin/custom/CustomApps.kt b/buildSrc/src/main/kotlin/custom/CustomApps.kt
@@ -48,8 +48,6 @@ fun ProductFlavors.create(customApps: List<CustomApp>) {
       buildConfigField("String", "ENFORCED_LANG", "\"${customApp.enforcedLanguage}\"")
       buildConfigField("String", "ABOUT_APP_URL", "\"${customApp.aboutAppUrl}\"")
       buildConfigField("String", "SUPPORT_URL", "\"${customApp.supportUrl}\"")
-      // Add asset file name in buildConfig file, we will use later to receive the zim file.
-      buildConfigField("String", "PLAY_ASSET_FILE", "\"${customApp.name}.zim\"")
       buildConfigField("Boolean", "DISABLE_SIDEBAR", "${customApp.disableSideBar}")
       buildConfigField("Boolean", "DISABLE_TABS", "${customApp.disableTabs}")
       buildConfigField("Boolean", "DISABLE_READ_ALOUD", "${customApp.disableReadAloud}")

diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/main/CoreReaderFragment.kt
@@ -156,8 +156,8 @@ import org.kiwix.kiwixmobile.core.utils.TAG_FILE_SEARCHED_NEW_TAB
 import org.kiwix.kiwixmobile.core.utils.TAG_KIWIX
 import org.kiwix.kiwixmobile.core.utils.UpdateUtils.reformatProviderUrl
 import org.kiwix.kiwixmobile.core.utils.dialog.DialogShower
-import org.kiwix.kiwixmobile.core.utils.dialog.UnsupportedMimeTypeHandler
 import org.kiwix.kiwixmobile.core.utils.dialog.KiwixDialog
+import org.kiwix.kiwixmobile.core.utils.dialog.UnsupportedMimeTypeHandler
 import org.kiwix.kiwixmobile.core.utils.files.FileUtils.deleteCachedFiles
 import org.kiwix.kiwixmobile.core.utils.files.FileUtils.readFile
 import org.kiwix.kiwixmobile.core.utils.files.Log
@@ -1554,7 +1554,7 @@ abstract class CoreReaderFragment :
   protected fun openZimFile(
     file: File?,
     isCustomApp: Boolean = false,
-    assetFileDescriptor: AssetFileDescriptor? = null,
+    assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
     filePath: String? = null
   ) {
     if (hasPermission(Manifest.permission.READ_EXTERNAL_STORAGE) || isCustomApp) {
@@ -1564,10 +1564,10 @@ abstract class CoreReaderFragment :
         reopenBook()
         openAndSetInContainer(file = file)
         updateTitle()
-      } else if (assetFileDescriptor != null) {
+      } else if (assetFileDescriptorList.isNotEmpty()) {
         reopenBook()
         openAndSetInContainer(
-          assetFileDescriptor = assetFileDescriptor,
+          assetFileDescriptorList = assetFileDescriptorList,
           filePath = filePath
         )
         updateTitle()
@@ -1602,7 +1602,7 @@ abstract class CoreReaderFragment :
 
   private fun openAndSetInContainer(
     file: File? = null,
-    assetFileDescriptor: AssetFileDescriptor? = null,
+    assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
     filePath: String? = null
   ) {
     try {
@@ -1613,9 +1613,9 @@ abstract class CoreReaderFragment :
       e.printStackTrace()
     }
     zimReaderContainer?.let { zimReaderContainer ->
-      if (assetFileDescriptor != null) {
+      if (assetFileDescriptorList.isNotEmpty()) {
         zimReaderContainer.setZimFileDescriptor(
-          assetFileDescriptor,
+          assetFileDescriptorList,
           filePath = filePath
         )
       } else {

diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimFileReader.kt
@@ -38,6 +38,7 @@ import org.kiwix.kiwixmobile.core.utils.files.Log
 import org.kiwix.libkiwix.JNIKiwixException
 import org.kiwix.libzim.Archive
 import org.kiwix.libzim.DirectAccessInfo
+import org.kiwix.libzim.FdInput
 import org.kiwix.libzim.Item
 import org.kiwix.libzim.SuggestionSearch
 import org.kiwix.libzim.SuggestionSearcher
@@ -56,7 +57,7 @@ private const val TAG = "ZimFileReader"
 @Suppress("LongParameterList")
 class ZimFileReader constructor(
   val zimFile: File?,
-  val assetFileDescriptor: AssetFileDescriptor? = null,
+  val assetFileDescriptorList: List<AssetFileDescriptor> = emptyList(),
   val assetDescriptorFilePath: String? = null,
   val jniKiwixReader: Archive,
   private val nightModeConfig: NightModeConfig,
@@ -65,7 +66,7 @@ class ZimFileReader constructor(
   interface Factory {
     suspend fun create(file: File): ZimFileReader?
     suspend fun create(
-      assetFileDescriptor: AssetFileDescriptor,
+      assetFileDescriptorList: List<AssetFileDescriptor>,
       filePath: String? = null
     ): ZimFileReader?
 
@@ -91,18 +92,19 @@ class ZimFileReader constructor(
         }
 
       override suspend fun create(
-        assetFileDescriptor: AssetFileDescriptor,
+        assetFileDescriptorList: List<AssetFileDescriptor>,
         filePath: String?
       ): ZimFileReader? = withContext(Dispatchers.IO) { // Bug Fix #3805
         try {
-          val archive = Archive(
-            assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor,
-            assetFileDescriptor.startOffset,
-            assetFileDescriptor.length
-          )
+          val fdInputArray = getFdInputArrayFromAssetFileDescriptorList(assetFileDescriptorList)
+          val archive = if (fdInputArray.size == 1) {
+            Archive(fdInputArray[0])
+          } else {
+            Archive(fdInputArray)
+          }
           ZimFileReader(
             null,
-            assetFileDescriptor,
+            assetFileDescriptorList,
             assetDescriptorFilePath = filePath,
             nightModeConfig = nightModeConfig,
             jniKiwixReader = archive,
@@ -116,6 +118,17 @@ class ZimFileReader constructor(
           null
         }
       }
+
+      private fun getFdInputArrayFromAssetFileDescriptorList(
+        assetFileDescriptorList: List<AssetFileDescriptor>
+      ): Array<FdInput> =
+        assetFileDescriptorList.map {
+          FdInput(
+            it.parcelFileDescriptor.dup().fileDescriptor,
+            it.startOffset,
+            it.length
+          )
+        }.toTypedArray()
     }
   }
 

diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/reader/ZimReaderContainer.kt
@@ -46,12 +46,14 @@ class ZimReaderContainer @Inject constructor(private val zimFileReaderFactory: F
   }
 
   fun setZimFileDescriptor(
-    assetFileDescriptor: AssetFileDescriptor,
+    assetFileDescriptorList: List<AssetFileDescriptor>,
     filePath: String? = null
   ) {
     zimFileReader = runBlocking {
-      if (assetFileDescriptor.parcelFileDescriptor.dup().fileDescriptor.valid())
-        zimFileReaderFactory.create(assetFileDescriptor, filePath)
+      if (assetFileDescriptorList.isNotEmpty() &&
+        assetFileDescriptorList[0].parcelFileDescriptor.dup().fileDescriptor.valid()
+      )
+        zimFileReaderFactory.create(assetFileDescriptorList, filePath)
       else null
     }
   }

diff --git a/core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt b/core/src/main/java/org/kiwix/kiwixmobile/core/webserver/KiwixServer.kt
@@ -51,7 +51,8 @@ class KiwixServer @Inject constructor(
             // Determine whether to create an Archive from an asset or a file path
             val archive = if (path == getDemoFilePathForCustomApp(context)) {
               // For custom apps using a demo file, create an Archive with FileDescriptor
-              val assetFileDescriptor = zimReaderContainer.zimFileReader?.assetFileDescriptor
+              val assetFileDescriptor =
+                zimReaderContainer.zimFileReader?.assetFileDescriptorList?.get(0)
               val startOffset = assetFileDescriptor?.startOffset ?: 0L
               val size = assetFileDescriptor?.length ?: 0L
               Archive(

diff --git a/custom/build.gradle.kts b/custom/build.gradle.kts
@@ -116,6 +116,41 @@ fun writeZimFileData(responseBody: ResponseBody, file: File) {
   }
 }
 
+fun writeZimFileDataInChunk(
+  responseBody: ResponseBody,
+  file: File,
+  chunkSize: Long = 500 * 1024 * 1024 // create a chunk of 500MB
+) {
+  var outputStream: FileOutputStream? = null
+  val buffer = ByteArray(4096)
+  var bytesRead: Int
+  var totalBytesWritten = 0L
+  var chunkNumber = 0
+
+  responseBody.byteStream().use { inputStream ->
+    while (inputStream.read(buffer).also { bytesRead = it } != -1) {
+      if (outputStream == null || totalBytesWritten >= chunkSize) {
+        // Close the current chunk and open a new one
+        outputStream?.flush()
+        outputStream?.close()
+        chunkNumber++
+        val nextChunkFile = File(file.parent, "chunk$chunkNumber.zim")
+        nextChunkFile.createNewFile()
+        outputStream = FileOutputStream(nextChunkFile)
+        totalBytesWritten = 0 // Reset totalBytesWritten for the new chunk
+      }
+
+      // Write data to the output stream
+      outputStream?.write(buffer, 0, bytesRead)
+      totalBytesWritten += bytesRead
+    }
+  }
+
+  // Close the last chunk (if any)
+  outputStream?.flush()
+  outputStream?.close()
+}
+
 fun ProductFlavor.createDownloadTaskForPlayAssetDelivery(
   file: File
 ): Task {
@@ -134,7 +169,7 @@ fun ProductFlavor.createDownloadTaskForPlayAssetDelivery(
       OkHttpClient().newCall(fetchRequest()).execute().use { response ->
         if (response.isSuccessful) {
           response.body?.let { responseBody ->
-            writeZimFileData(responseBody, file)
+            writeZimFileDataInChunk(responseBody, file)
           }
         } else {
           throw RuntimeException(

diff --git a/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt b/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomFileValidator.kt
@@ -21,9 +21,9 @@ package org.kiwix.kiwixmobile.custom.main
 import android.content.Context
 import android.content.pm.PackageManager
 import android.content.res.AssetFileDescriptor
+import android.content.res.AssetManager
 import androidx.core.content.ContextCompat
 import org.kiwix.kiwixmobile.core.utils.files.Log
-import org.kiwix.kiwixmobile.custom.BuildConfig
 import org.kiwix.kiwixmobile.custom.main.ValidationState.HasBothFiles
 import org.kiwix.kiwixmobile.custom.main.ValidationState.HasFile
 import org.kiwix.kiwixmobile.custom.main.ValidationState.HasNothing
@@ -37,16 +37,18 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
     when (val installationState = detectInstallationState()) {
       is HasBothFiles,
       is HasFile -> onFilesFound(installationState)
+
       HasNothing -> onNoFilesFound()
     }
 
   private fun detectInstallationState(
     obbFiles: List<File> = obbFiles(),
     zimFiles: List<File> = zimFiles(),
-    assetFileDescriptor: AssetFileDescriptor? = getAssetFileDescriptorFromPlayAssetDelivery()
+    assetFileDescriptorList: List<AssetFileDescriptor> =
+      getAssetFileDescriptorListFromPlayAssetDelivery()
   ): ValidationState {
     return when {
-      assetFileDescriptor != null -> HasFile(null, assetFileDescriptor)
+      assetFileDescriptorList.isNotEmpty() -> HasFile(null, assetFileDescriptorList)
       obbFiles.isNotEmpty() && zimFiles().isNotEmpty() -> HasBothFiles(obbFiles[0], zimFiles[0])
       obbFiles.isNotEmpty() -> HasFile(obbFiles[0])
       zimFiles.isNotEmpty() -> HasFile(zimFiles[0])
@@ -55,11 +57,15 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
   }
 
   @Suppress("MagicNumber")
-  private fun getAssetFileDescriptorFromPlayAssetDelivery(): AssetFileDescriptor? {
+  private fun getAssetFileDescriptorListFromPlayAssetDelivery(): List<AssetFileDescriptor> {
     try {
-      val context = context.createPackageContext(context.packageName, 0)
-      val assetManager = context.assets
-      return assetManager.openFd(BuildConfig.PLAY_ASSET_FILE)
+      val assetManager = context.createPackageContext(context.packageName, 0).assets
+      val assetFileDescriptorList: ArrayList<AssetFileDescriptor> = arrayListOf()
+      getChunksList(assetManager).forEach {
+        assetFileDescriptorList.add(assetManager.openFd(it))
+      }
+
+      return assetFileDescriptorList
     } catch (packageNameNotFoundException: PackageManager.NameNotFoundException) {
       Log.w(
         "ASSET_PACKAGE_DELIVERY",
@@ -68,7 +74,24 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
     } catch (ioException: IOException) {
       Log.w("ASSET_PACKAGE_DELIVERY", "Unable to copy the content of asset $ioException")
     }
-    return null
+    return emptyList()
+  }
+
+  private fun getChunksList(assetManager: AssetManager): List<String> {
+    val chunkFiles = mutableListOf<String>()
+
+    try {
+      // List of all files in the asset directory
+      val assets = assetManager.list("") ?: emptyArray()
+
+      // Filter and count chunk files.
+      assets.filterTo(chunkFiles) { it.startsWith("chunk") && it.endsWith(".zim") }
+      chunkFiles.sortBy { it.substringAfter("chunk").substringBefore(".zim").toInt() }
+    } catch (ioException: IOException) {
+      ioException.printStackTrace()
+    }
+
+    return chunkFiles
   }
 
   private fun obbFiles() = scanDirs(ContextCompat.getObbDirs(context), "obb")
@@ -104,7 +127,10 @@ class CustomFileValidator @Inject constructor(private val context: Context) {
 
 sealed class ValidationState {
   data class HasBothFiles(val obbFile: File, val zimFile: File) : ValidationState()
-  data class HasFile(val file: File?, val assetFileDescriptor: AssetFileDescriptor? = null) :
+  data class HasFile(
+    val file: File?,
+    val assetFileDescriptorList: List<AssetFileDescriptor> = emptyList()
+  ) :
     ValidationState()
 
   object HasNothing : ValidationState()

diff --git a/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt b/custom/src/main/java/org/kiwix/kiwixmobile/custom/main/CustomReaderFragment.kt
@@ -181,8 +181,8 @@ class CustomReaderFragment : CoreReaderFragment() {
       onFilesFound = {
         when (it) {
           is ValidationState.HasFile -> {
-            if (it.assetFileDescriptor != null) {
-              openZimFile(null, true, it.assetFileDescriptor)
+            if (it.assetFileDescriptorList.isNotEmpty()) {
+              openZimFile(null, true, it.assetFileDescriptorList)
             } else {
               openZimFile(it.file, true)
             }