Reject invalid depth in avifImageAllocatePlanes() (AOMediaCodec#3038)

Also reject >16-bit in avifParsePixelInformationProperty() to avoid
returning the wrong AVIF_RESULT_OUT_OF_MEMORY error code later.
Also reject depth 0 in avifParsePixelInformationProperty().
Add test.

Author: y-guyon

src/avif.c

@@ -426,7 +426,7 @@ avifResult avifImageAddUUIDProperty(avifImage * image, const uint8_t uuid[16], c
 
 avifResult avifImageAllocatePlanes(avifImage * image, avifPlanesFlags planes)
 {
-    if (image->width == 0 || image->height == 0) {
+    if (image->width == 0 || image->height == 0 || image->depth == 0 || image->depth > 16) {
         return AVIF_RESULT_INVALID_ARGUMENT;
     }
     const uint32_t channelSize = avifImageUsesU16(image) ? 2 : 1;

src/read.c

@@ -2708,12 +2708,14 @@ static avifResult avifParsePixelInformationProperty(avifProperty * prop, const u
     }
     for (uint8_t i = 0; i < pixi->planeCount; ++i) {
         AVIF_CHECKERR(avifROStreamRead(&s, &pixi->planeDepths[i], 1), AVIF_RESULT_BMFF_PARSE_FAILED); // unsigned int (8) bits_per_channel;
-#if defined(AVIF_ENABLE_EXPERIMENTAL_EXTENDED_PIXI)
         if (pixi->planeDepths[i] == 0) {
             avifDiagnosticsPrintf(diag, "Box[pixi] plane depth shall not be 0 for channel %u", i);
             return AVIF_RESULT_BMFF_PARSE_FAILED;
         }
-#endif // AVIF_ENABLE_EXPERIMENTAL_EXTENDED_PIXI
+        if (pixi->planeDepths[i] > 16) {
+            avifDiagnosticsPrintf(diag, "Box[pixi] plane depth %d is not supported", (int)pixi->planeDepths[i]);
+            return AVIF_RESULT_NOT_IMPLEMENTED;
+        }
         if (pixi->planeDepths[i] != pixi->planeDepths[0]) {
             avifDiagnosticsPrintf(diag,
                                   "Box[pixi] contains unsupported mismatched plane depths [%u != %u]",

tests/gtest/avif16bittest.cc

@@ -2,6 +2,8 @@
 // SPDX-License-Identifier: BSD-2-Clause
 
 #include <cstring>
+#include <filesystem>
+#include <fstream>
 
 #include "avif/avif.h"
 #include "avif/avif_cxx.h"
@@ -376,6 +378,72 @@ INSTANTIATE_TEST_SUITE_P(
 
 //------------------------------------------------------------------------------
 
+TEST(Avif16bitTest, SampleTransformWithOtherBitDepths) {
+  if (!testutil::Av1DecoderAvailable()) {
+    GTEST_SKIP() << "AV1 Codec unavailable, skip test.";
+  }
+
+  const std::string file_path =
+      std::string(data_path) + "weld_sato_12B_8B_q0.avif";
+  std::vector<uint8_t> encoded_16bit(std::filesystem::file_size(file_path));
+  std::ifstream(file_path, std::ios::binary)
+      .read(reinterpret_cast<char*>(encoded_16bit.data()),
+            encoded_16bit.size());
+
+  ImagePtr reference(avifImageCreateEmpty());
+  ASSERT_NE(reference, nullptr);
+  DecoderPtr decoder_reference(avifDecoderCreate());
+  ASSERT_NE(decoder_reference, nullptr);
+  decoder_reference->imageContentToDecode |=
+      AVIF_IMAGE_CONTENT_SAMPLE_TRANSFORMS;
+  ASSERT_EQ(avifDecoderReadMemory(decoder_reference.get(), reference.get(),
+                                  encoded_16bit.data(), encoded_16bit.size()),
+            AVIF_RESULT_OK);
+
+  for (uint8_t num_bits = 0; num_bits <= 32; ++num_bits) {
+    if (num_bits == reference->depth) {
+      continue;
+    }
+    std::vector<uint8_t> encoded(encoded_16bit);
+    // Replace 'pixi' 3-channel 16-bit by another bit depth.
+    bool found_subsequence_to_replace = false;
+    for (size_t i = 0; i + 4 <= encoded.size(); ++i) {
+      if (!std::memcmp(&encoded[i],
+                       "pixi"       // PixelInformationProperty 4CC
+                       "\0\0\0\0"   // version and flags
+                       "\3"         // num_channels
+                       "\20\20\20"  // bits_per_channels (16 is 20 in octal)
+                       ,
+                       4 + 4 + 1 + 3)) {
+        encoded[i + 9] = encoded[i + 10] = encoded[i + 11] = num_bits;
+        found_subsequence_to_replace = true;
+        break;
+      }
+    }
+    ASSERT_TRUE(found_subsequence_to_replace);
+
+    ImagePtr decoded(avifImageCreateEmpty());
+    ASSERT_NE(decoded, nullptr);
+    DecoderPtr decoder(avifDecoderCreate());
+    ASSERT_NE(decoder, nullptr);
+    decoder->imageContentToDecode |= AVIF_IMAGE_CONTENT_SAMPLE_TRANSFORMS;
+    const avifResult result = avifDecoderReadMemory(
+        decoder.get(), decoded.get(), encoded.data(), encoded.size());
+    const avifResult expected_result =
+        num_bits == 0   ? AVIF_RESULT_BMFF_PARSE_FAILED
+        : num_bits > 16 ? AVIF_RESULT_NOT_IMPLEMENTED
+                        : AVIF_RESULT_OK;
+    ASSERT_EQ(result, expected_result) << "bits_per_channels " << num_bits;
+    if (result == AVIF_RESULT_OK) {
+      // The output image should be highly distorted because of the pixel value
+      // clamping to (1<<num_bits)-1.
+      EXPECT_LE(testutil::GetPsnr(*reference, *decoded), 10.0);
+    }
+  }
+}
+
+//------------------------------------------------------------------------------
+
 }  // namespace
 }  // namespace avif