Merge pull request #211 from Botts-Innovative-Research/ffmpeg-support-for-h265

Add support for H264/HEVC in FFMPEG

Author: drewbotts

sensors/video/sensorhub-driver-ffmpeg/src/main/java/org/sensorhub/impl/sensor/ffmpeg/outputs/VideoOutput.java

@@ -36,6 +36,7 @@
 public class VideoOutput<T extends ISensorModule<?>> extends AbstractSensorOutput<T> implements DataBufferListener {
     private static final String CODEC_MJPEG = "JPEG";
     private static final String CODEC_H264 = "H264";
+    private static final String CODEC_H265 = "H265";
     private static final Logger logger = LoggerFactory.getLogger(VideoOutput.class.getSimpleName());
     private static final int MAX_NUM_TIMING_SAMPLES = 10;
 
@@ -83,11 +84,13 @@ public VideoOutput(T parentSensor, int[] videoFrameDimensions, String codecName,
             this.codecName = CODEC_MJPEG;
         } else if (codecName.equalsIgnoreCase("h264")) {
             this.codecName = CODEC_H264;
+        } else if (codecName.equalsIgnoreCase("hevc") || codecName.equalsIgnoreCase("h265")) {
+            this.codecName = CODEC_H265;
         } else {
             this.codecName = codecName;
-            logger.warn("Codec {} is neither H264 nor MJPEG. Compression set using FFmpeg name.", codecName);
+            logger.warn("Codec {} is not one of H264, H265, or MJPEG. Compression set using FFmpeg name.", codecName);
             ((AbstractModule<?>) parentSensor).reportStatus("Video codec = " + codecName.toUpperCase() + ". Use the transcoder process module" +
-                    " with H264 or MJPEG output for compatibility with OSH visualization and serialization.");
+                    " with H264, H265, or MJPEG output for compatibility with OSH visualization and serialization.");
         }
 
         logger.debug("Video output created.");

sensors/video/sensorhub-driver-ffmpeg/src/main/java/org/sensorhub/mpegts/StreamContext.java

@@ -154,10 +154,16 @@ public void openCodecContext(AVFormatContext avFormatContext) throws IllegalStat
         setCodecName(codec.name().getString());
         setCodecId(codec.id());
 
-        if (isInjectingExtradata && codecId == avcodec.AV_CODEC_ID_H264) {
+        if (isInjectingExtradata) {
             BytePointer extra = params.extradata();
             int extraLen = params.extradata_size();
-            extraData = getAnnexBExtradata(extra, extraLen);
+            if (codecId == avcodec.AV_CODEC_ID_H264) {
+                extraData = getAnnexBExtradata(extra, extraLen);
+            } else if (codecId == avcodec.AV_CODEC_ID_HEVC) {
+                extraData = getHvccAnnexBExtradata(extra, extraLen);
+            } else {
+                extraData = null;
+            }
         } else {
             extraData = null;
         }
@@ -178,7 +184,9 @@ public void processPacket(AVPacket avPacket) {
         byte[] dataBuffer = new byte[avPacket.size()];
         avPacket.data().get(dataBuffer);
 
-        // Add extradata if the packet has an h264 keyframe
+        // Prepend cached parameter sets ahead of every keyframe so late-join
+        // decoders have the VPS/SPS/PPS they need. Applies to both H.264 (SPS/PPS)
+        // and HEVC (VPS/SPS/PPS) when extradata injection is enabled.
         if (extraData != null && (avPacket.flags() & avcodec.AV_PKT_FLAG_KEY) != 0) {
             getDataBufferListener().onDataBuffer(new DataBufferRecord(avPacket.pts() * getStreamTimeBase(), extraData));
         }
@@ -187,8 +195,9 @@ public void processPacket(AVPacket avPacket) {
     }
 
     /**
-     * Converts the given extradata from AVCC format to Annex B format.
-     * If the data is already in Annex B format, it is returned directly.
+     * Converts the given H.264 extradata from AVCC format (AVCDecoderConfigurationRecord,
+     * ISO/IEC 14496-15 §5.2.4.1) to Annex B format. If the data is already in Annex B format,
+     * it is returned directly.
      *
      * @param extradata A BytePointer containing the codec extradata. Must not be null.
      * @param size The size of the extradata in bytes.
@@ -235,4 +244,66 @@ private byte[] getAnnexBExtradata(BytePointer extradata, int size) {
         }
         return out.toByteArray();
     }
+
+    /**
+     * Converts HEVC extradata from HVCC format (HEVCDecoderConfigurationRecord,
+     * ISO/IEC 14496-15 §8.3.3.1.2) to Annex B format. Walks the {@code numOfArrays}
+     * table and emits every contained NAL (typically VPS=32, SPS=33, PPS=34) prefixed
+     * with the 4-byte 0x00000001 start code so that downstream pipelines can decode
+     * without out-of-band parameter sets.
+     * <p>
+     * If the data is already in Annex B format, it is returned directly.
+     *
+     * @param extradata A BytePointer containing the codec extradata. Must not be null.
+     * @param size The size of the extradata in bytes.
+     * @return A byte array containing the Annex B formatted VPS/SPS/PPS NALs, or
+     *         {@code null} if the data is invalid, too short, or cannot be parsed.
+     */
+    private byte[] getHvccAnnexBExtradata(BytePointer extradata, int size) {
+        // HVCC has a 22-byte fixed header followed by numOfArrays (1 byte),
+        // so the minimum useful size is 23 bytes.
+        if (extradata == null || size < 23) return null;
+
+        byte[] data = new byte[size];
+        extradata.get(data);
+
+        // If already Annex B, pass straight through.
+        if (data[0] == 0x00 && data[1] == 0x00 && (data[2] == 0x01 || (data[2] == 0x00 && data[3] == 0x01))) {
+            return data;
+        }
+
+        // configurationVersion must be 1 per ISO/IEC 14496-15.
+        if ((data[0] & 0xFF) != 1) {
+            logger.warn("Unsupported HVCC configurationVersion: {}", data[0] & 0xFF);
+            return null;
+        }
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try {
+            int pos = 22;
+            int numOfArrays = data[pos++] & 0xFF;
+
+            for (int i = 0; i < numOfArrays; i++) {
+                if (pos + 3 > data.length) return null;
+
+                pos++;
+                int numNalus = ((data[pos++] & 0xFF) << 8) | (data[pos++] & 0xFF);
+
+                for (int j = 0; j < numNalus; j++) {
+                    if (pos + 2 > data.length) return null;
+                    int nalLen = ((data[pos++] & 0xFF) << 8) | (data[pos++] & 0xFF);
+                    if (nalLen <= 0 || pos + nalLen > data.length) return null;
+                    out.write(new byte[]{0x00, 0x00, 0x00, 0x01});
+                    out.write(data, pos, nalLen);
+                    pos += nalLen;
+                }
+            }
+        } catch (Exception e) {
+            logger.error("Error extracting VPS/SPS/PPS from HVCC extradata", e);
+            return null;
+        }
+
+        byte[] result = out.toByteArray();
+        return result.length == 0 ? null : result;
+    }
 }