Fix heap buffer overflow in libopenapv encoder

Add bounds checking to BSW_FLUSH_4BYTE and BSW_FLUSH_8BYTE macros
in oapv_vlc.c to prevent writing past the end of the bitstream buffer
during VLC encoding.

Add bounds checking in enc_frame in oapv.c to ensure the cumulative
tile bitstream size does not exceed the target bitstream buffer end
before copying tile bitstreams.

These changes prevent heap buffer overflows in the encoder.

Bug: 501452526
Test: Manual verification with PoC binaries on Cuttlefish
Flag: EXEMPT CVE_FIX
Change-Id: Ib40bc500096b6fda93e5802d97b306e4320ba6eb

Author: fkyslov

src/oapv.c

@@ -1171,6 +1171,7 @@ static int enc_frame(oapve_ctx_t *ctx, oapv_bs_t *bs)
     /****************************************************/
 
     for(int i = 0; i < ctx->num_tiles; i++) {
+        oapv_assert_gv(bs_tile_pos + ctx->tile[i].bs_size <= bs->end, ret, OAPV_ERR_OUT_OF_BS_BUF, ERR);
         oapv_mcpy(bs_tile_pos, ctx->tile[i].bs_buf, ctx->tile[i].bs_size);
         bs_tile_pos = bs_tile_pos + ctx->tile[i].bs_size;
         ctx->fh.tile_size[i] = ctx->tile[i].bs_size - OAPV_TILE_SIZE_LEN;

src/oapv_vlc.c

@@ -37,23 +37,31 @@
 #if ENABLE_ENCODER
 ///////////////////////////////////////////////////////////////////////////////
 #define BSW_FLUSH_4BYTE(bs) {                     \
-        *(bs)->cur++ = ((bs)->code >> 24) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 16) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 8) & 0xFF;  \
-        *(bs)->cur++ = ((bs)->code) & 0xFF;       \
+        if ((bs)->cur + 4 <= (bs)->end) {         \
+            *(bs)->cur++ = ((bs)->code >> 24) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 16) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 8) & 0xFF;  \
+            *(bs)->cur++ = ((bs)->code) & 0xFF;       \
+        } else {                                  \
+            (bs)->cur += 4;                       \
+        }                                         \
         (bs)->code = 0;                           \
         (bs)->leftbits = 32;                      \
     }
 
 #define BSW_FLUSH_8BYTE(bs) {                     \
-        *(bs)->cur++ = ((bs)->code >> 56) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 48) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 40) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 32) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 24) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 16) & 0xFF; \
-        *(bs)->cur++ = ((bs)->code >> 8) & 0xFF;  \
-        *(bs)->cur++ = ((bs)->code) & 0xFF;       \
+        if ((bs)->cur + 8 <= (bs)->end) {         \
+            *(bs)->cur++ = ((bs)->code >> 56) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 48) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 40) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 32) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 24) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 16) & 0xFF; \
+            *(bs)->cur++ = ((bs)->code >> 8) & 0xFF;  \
+            *(bs)->cur++ = ((bs)->code) & 0xFF;       \
+        } else {                                  \
+            (bs)->cur += 8;                       \
+        }                                         \
         (bs)->code = 0;                           \
         (bs)->leftbits = 64;                      \
     }