feat: Add HEIC to Image Formats (#71)

* fix: Fix build because of missing `allowGpu`

* feat: Add `HEIC` to `ImageFormat`s

* Add `supportsHeicLoading`

* Make quality go from 0...100

* Use it on Android

* fix: Fix `toByteArray`

Author: mrousavy

README.md

@@ -112,12 +112,22 @@ const image = useImage(require('./image.png'))
 
 #### `ArrayBuffer`
 
-The `Image` type can be converted to- and from- an `ArrayBuffer`, which gives you access to the raw pixel data in ARGB format:
+The `Image` type can be converted to- and from- an `ArrayBuffer`, which gives you access to the raw pixel data in an RGB format:
 
 ```ts
 const image           = // ...
-const arrayBuffer     = await image.toArrayBufferAsync()
-const sameImageCopied = await Images.loadFromArrayBufferAsync(arrayBuffer)
+const pixelData       = await image.toRawPixelData()
+const sameImageCopied = await Images.loadFromRawPixelData(pixelData)
+```
+
+#### `EncodedImageData`
+
+The `Image` type can be encoded to- and decoded from- an `ArrayBuffer` using a container format like `jpg`, `png` or `heic`:
+
+```ts
+const image           = // ...
+const imageData       = await image.toEncodedImageData('jpg', 90)
+const sameImageCopied = await Images.loadFromEncodedImageData(imageData)
 ```
 
 #### Resizing
@@ -143,8 +153,37 @@ const smaller  = await webImage.cropAsync(100, 100, 50, 50)
 An in-memory `Image` object can also be written/saved to a file:
 
 ```ts
-const smaller  = ...
-const path     = await smaller.saveToTemporaryFileAsync('jpg', 90)
+const image  = ...
+const path   = await image.saveToTemporaryFileAsync('jpg', 90)
+```
+
+#### Compressing
+
+Images can be compressed using the `jpg` container format - either in-memory or when writing to a file:
+
+```ts
+const image      = ...
+const path       = await image.saveToTemporaryFileAsync('jpg', 50) // 50% compression
+const compressed = await image.toEncodedImageData('jpg', 50)       // 50% compression
+```
+
+#### HEIC/HEIF
+
+NitroImage supports `HEIC`/`HEIF` format if the host OS natively supports it.
+
+|              | iOS            | Android        |
+|--------------|----------------|----------------|
+| Loading HEIC | ✅             | ✅ (>= SDK 28) |
+| Writing HEIC | ✅ (>= iOS 17) | ❌             |
+
+You can check whether your OS supports `HEIC` via NitroImage:
+
+```ts
+import { supportsHeicWriting } from 'react-native-nitro-modules'
+
+const image  = ...
+const format = supportsHeicWriting ? 'heic' : 'jpg'
+const path   = await image.saveToTemporaryFileAsync(format, 100)
 ```
 
 ### Hooks

packages/react-native-nitro-image/android/src/main/java/com/margelo/nitro/image/ArrayBuffer+toByteArray.kt

@@ -7,7 +7,15 @@ fun ArrayBuffer.toByteArray(): ByteArray {
     val buffer = this.getBuffer(false)
     if (buffer.hasArray()) {
         // It's a CPU-backed array - we can return this directly
-        return buffer.array()
+        val array = buffer.array()
+        if (array.size == this.size) {
+            // The CPU-backed array is exactly the view we have in our ArrayBuffer.
+            // Return as is!
+            return array
+        }
+        // we had a CPU-backed array, but it's size differs from our ArrayBuffer size.
+        // This might be because the ArrayBuffer has a smaller view of the data, so we need
+        // to resort back to a good ol' copy.
     }
     // It's not a CPU-backed array (e.g. HardwareBuffer) - we need to copy to the CPU
     val copy = ByteBuffer.allocate(buffer.capacity())

packages/react-native-nitro-image/android/src/main/java/com/margelo/nitro/image/Bitmap+compressInMemory.kt

@@ -0,0 +1,24 @@
+package com.margelo.nitro.image
+
+import android.graphics.Bitmap
+import java.nio.ByteBuffer
+
+fun Bitmap.compressInMemory(format: ImageFormat, quality: Int): ByteBuffer {
+    if (quality < 0 || quality > 100) {
+        throw Error("Image quality has to be between 0 and 100! (Received: $quality)")
+    }
+    val estimatedByteSize = when (format) {
+        ImageFormat.JPG -> (width * height) / 2
+        ImageFormat.PNG -> width * height
+        ImageFormat.HEIC -> width * height
+    }
+
+    FastByteArrayOutputStream(estimatedByteSize).use { out ->
+        val successful = this.compress(format.toBitmapFormat(), quality, out)
+        if (!successful) {
+            throw Error("Failed to compress the Bitmap into EncodedImageData! (Format: ${format.name}, " +
+                    "Quality: ${quality}, Written Bytes: ${out.count})")
+        }
+        return out.toByteBuffer()
+    }
+}

packages/react-native-nitro-image/android/src/main/java/com/margelo/nitro/image/Bitmap+saveToFile.kt

@@ -8,15 +8,24 @@ fun ImageFormat.toBitmapFormat(): Bitmap.CompressFormat {
     return when (this) {
         ImageFormat.JPG -> Bitmap.CompressFormat.JPEG
         ImageFormat.PNG -> Bitmap.CompressFormat.PNG
+        ImageFormat.HEIC -> {
+            throw Error("Saving Images as HEIC is not yet supported on Android!")
+        }
     }
 }
 
 fun Bitmap.saveToFile(path: String, format: ImageFormat, quality: Int) {
+    if (quality < 0 || quality > 100) {
+        throw Error("Image quality has to be between 0 and 100! (Received: $quality)")
+    }
     // 1. Make sure all parent directories exist
     File(path).parentFile?.mkdirs()
     // 2. Create a file output stream
     FileOutputStream(path).use { out ->
         val bitmapFormat = format.toBitmapFormat()
-        this.compress(bitmapFormat, quality, out)
+        val successful = this.compress(bitmapFormat, quality, out)
+        if (!successful) {
+            throw Error("Failed to compress ${width}x${height} Image to file (\"$path\")! (Format: $format, Quality: $quality)")
+        }
     }
 }

packages/react-native-nitro-image/android/src/main/java/com/margelo/nitro/image/HybridImage.kt

@@ -56,19 +56,9 @@ class HybridImage: HybridImageSpec {
     }
 
     override fun toEncodedImageData(format: ImageFormat, quality: Double?): EncodedImageData {
-        val quality = quality ?: 1.0
-        val estimatedByteSize = when (format) {
-            ImageFormat.JPG -> (width * height) / 2
-            ImageFormat.PNG -> width * height
-        }
-        val outputStream = FastByteArrayOutputStream(estimatedByteSize.toInt())
-        val successful = bitmap.compress(format.toBitmapFormat(), quality.toInt(), outputStream)
-        if (!successful) {
-            throw Error("Failed to compress the Bitmap into EncodedImageData! (Format: ${format.name}, " +
-                    "Quality: ${quality}, Written Bytes: ${outputStream.count})")
-        }
-        val byteBuffer = outputStream.toByteBuffer()
-        val arrayBuffer = ArrayBuffer.wrap(byteBuffer)
+        val quality = quality ?: 100.0
+        val byteBuffer = bitmap.compressInMemory(format, quality.toInt())
+        val arrayBuffer = ArrayBuffer.copy(byteBuffer)
         return EncodedImageData(arrayBuffer, width, height, format)
     }
     override fun toEncodedImageDataAsync(
@@ -125,16 +115,17 @@ class HybridImage: HybridImageSpec {
         format: ImageFormat,
         quality: Double?
     ): Promise<Unit> {
-        val quality = quality ?: 1.0
+        val quality = quality ?: 100.0
         return Promise.async {
             bitmap.saveToFile(path, format, quality.toInt())
         }
     }
 
     override fun saveToTemporaryFileAsync(format: ImageFormat, quality: Double?): Promise<String> {
+        val quality = quality ?: 100.0
         return Promise.async {
             val tempFile = File.createTempFile("nitro_image_", format.name)
-            this.saveToFileAsync(tempFile.path, format, quality)
+            bitmap.saveToFile(tempFile.path, format, quality.toInt())
             return@async tempFile.path
         }
     }

packages/react-native-nitro-image/android/src/main/java/com/margelo/nitro/image/HybridImageFactory.kt

@@ -4,8 +4,8 @@ import android.annotation.SuppressLint
 import android.graphics.Bitmap
 import android.graphics.BitmapFactory
 import androidx.annotation.Keep
+import androidx.core.graphics.createBitmap
 import com.facebook.common.internal.DoNotStrip
-import com.facebook.react.bridge.ReactApplicationContext
 import com.madebyevan.thumbhash.ThumbHash
 import com.margelo.nitro.NitroModules
 import com.margelo.nitro.core.ArrayBuffer
@@ -56,6 +56,9 @@ class HybridImageFactory: HybridImageFactorySpec() {
 
     private fun loadFromEncodedBytes(bytes: ByteArray): HybridImageSpec {
         val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
+        if (bitmap == null) {
+            throw Error("Failed to decode EncodedImageData to an Image! (Bytes: ${bytes.size})")
+        }
         return HybridImage(bitmap)
     }
     override fun loadFromEncodedImageData(data: EncodedImageData): HybridImageSpec {
@@ -69,6 +72,9 @@ class HybridImageFactory: HybridImageFactorySpec() {
 
     override fun loadFromFile(filePath: String): HybridImageSpec {
         val bitmap = BitmapFactory.decodeFile(filePath)
+        if (bitmap == null) {
+            throw Error("Failed to load Image from file! (Path: $filePath)")
+        }
         return HybridImage(bitmap)
     }
 
@@ -79,7 +85,7 @@ class HybridImageFactory: HybridImageFactorySpec() {
     private fun loadFromThumbHash(thumbHashBytes: ByteArray): HybridImage {
         val rgba = ThumbHash.thumbHashToRGBA(thumbHashBytes)
 
-        val bitmap = Bitmap.createBitmap(rgba.width, rgba.height, Bitmap.Config.ARGB_8888)
+        val bitmap = createBitmap(rgba.width, rgba.height, Bitmap.Config.ARGB_8888)
         val buffer = ByteBuffer.wrap(rgba.rgba)
         bitmap.copyPixelsFromBuffer(buffer)
         return HybridImage(bitmap)

packages/react-native-nitro-image/android/src/main/java/com/margelo/nitro/image/HybridImageUtils.kt

@@ -1,5 +1,6 @@
 package com.margelo.nitro.image
 
+import android.os.Build
 import androidx.annotation.Keep
 import com.facebook.common.internal.DoNotStrip
 import com.margelo.nitro.core.ArrayBuffer
@@ -10,6 +11,19 @@ import kotlin.io.encoding.ExperimentalEncodingApi
 @DoNotStrip
 @Keep
 class HybridImageUtils: HybridImageUtilsSpec() {
+    override val supportsHeicLoading: Boolean
+        get() {
+            // Since Android 10, HEIF/HEIC is standard.
+            // https://source.android.com/docs/core/camera/heif
+            return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P
+        }
+    override val supportsHeicWriting: Boolean
+        get() {
+            // Android does not support saving HEIF data yet
+            return false
+        }
+
+
     @OptIn(ExperimentalEncodingApi::class)
     override fun thumbHashToBase64String(thumbhash: ArrayBuffer): String {
         val buffer = thumbhash.toByteArray()

packages/react-native-nitro-image/ios/HybridImageFactory.swift

@@ -7,6 +7,7 @@
 
 import Foundation
 import NitroModules
+import UniformTypeIdentifiers
 
 class HybridImageFactory: HybridImageFactorySpec {
   /**
@@ -55,19 +56,19 @@ class HybridImageFactory: HybridImageFactorySpec {
   /**
    * Load Image from the given raw ArrayBuffer data
    */
-  func loadFromRawPixelData(data: RawPixelData) throws -> any HybridImageSpec {
+  func loadFromRawPixelData(data: RawPixelData, allowGpu _ : Bool?) throws -> any HybridImageSpec {
     let uiImage = try UIImage(fromRawPixelData: data)
     return HybridImage(uiImage: uiImage)
   }
 
-  func loadFromRawPixelDataAsync(data: RawPixelData) throws -> Promise<any HybridImageSpec> {
+  func loadFromRawPixelDataAsync(data: RawPixelData, allowGpu: Bool?) throws -> Promise<any HybridImageSpec> {
     let dataCopy = data.buffer.isOwner ? data.buffer : ArrayBuffer.copy(of: data.buffer)
     let newData = RawPixelData(buffer: dataCopy,
                                width: data.width,
                                height: data.height,
                                pixelFormat: data.pixelFormat)
     return Promise.async {
-      return try self.loadFromRawPixelData(data: newData)
+      return try self.loadFromRawPixelData(data: newData, allowGpu: allowGpu)
     }
   }
 

packages/react-native-nitro-image/ios/HybridImageLoaderFactory.swift

@@ -31,7 +31,7 @@ class HybridImageLoaderFactory: HybridImageLoaderFactorySpec {
 
   func createRawPixelDataImageLoader(data: RawPixelData) throws -> any HybridImageLoaderSpec {
     return HybridImageLoader(load: {
-      try self.imageFactory.loadFromRawPixelDataAsync(data: data)
+      try self.imageFactory.loadFromRawPixelDataAsync(data: data, allowGpu: false)
     })
   }
 

packages/react-native-nitro-image/ios/HybridImageUtils.swift

@@ -7,8 +7,23 @@
 
 import Foundation
 import NitroModules
+import UniformTypeIdentifiers
 
 class HybridImageUtils: HybridImageUtilsSpec {
+  var supportsHeicLoading: Bool {
+    // Check if the type is supported by the OS
+    let types = CGImageDestinationCopyTypeIdentifiers() as! [String]
+    return types.contains(UTType.heic.identifier)
+  }
+  var supportsHeicWriting: Bool {
+    // HEIC .heicData() is only available on iOS 17
+    if #available(iOS 17.0, *) {
+      return true
+    } else {
+      return false
+    }
+  }
+  
   func thumbHashToBase64String(thumbhash: ArrayBuffer) throws -> String {
     let data = thumbhash.toData(copyIfNeeded: false)
     return data.base64EncodedString()