[TIKA-4777] Expose presentation start of delayed QuickTime timed metadata tracks

QuickTime timed metadata tracks (mebx sample descriptions, ISO 14496-12
boxed metadata) declare their key names in the file, and the track's
presentation start is defined by its edit list: a leading empty edit
delays the media by its duration. Parse the mebx key declarations
(keys/keyd) and emit, for every declared key of a delayed track, the
presentation start in microseconds as <key name>.track-start-us.
Undelayed tracks (start 0) are not reported, their start carries no
information.

This is deliberately generic, no key names are hardcoded. The main use
case are Apple Live Photos, which mark the moment the paired still
image was captured as the single one-tick sample of the
still-image-time track, shifted to that moment by an empty edit. The
sample value itself is a constant -1 marker, so the moov boxes alone
are sufficient and mdat is never touched.

The handler stays active for timed metadata tracks (handler type
'meta') instead of switching to the extraction-free Mp4MetaHandler,
which would keep it from seeing the track's stsd.

Verified against a real Live Photo video (iPhone 15 Pro, iOS 18.5):
com.apple.quicktime.still-image-time.track-start-us = 1233333 (1.2333s,
matching the empty edit of 740/600s), likewise the transform keys of
the same track; the undelayed video-orientation and live-photo-info
tracks are not reported.

Author: dschmidt

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java

@@ -52,13 +52,24 @@ public class TikaMp4BoxHandler extends Mp4BoxHandler {
     private static final Pattern ISO6709_PATTERN =
             Pattern.compile("([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)?");
 
+    //suffix under which the presentation start of a timed metadata track is
+    //emitted for each of the track's keys, in microseconds (e.g. the Live Photo
+    //still moment appears as com.apple.quicktime.still-image-time.track-start-us)
+    private static final String TRACK_START_SUFFIX = ".track-start-us";
+
     org.apache.tika.metadata.Metadata tikaMetadata;
     final XHTMLContentHandler xhtml;
 
     //key names for the current 'meta' box, filled from its 'keys' box and consumed
     //by the following 'ilst' box (e.g. com.apple.quicktime.content.identifier)
     private final List<String> quickTimeMetadataKeys = new ArrayList<>();
 
+    //movie timescale from 'mvhd'; edit list durations are expressed in it
+    private long movieTimescale = 0;
+    //duration of the current track's leading empty edit ('elst' entry with
+    //media time -1), in movie timescale units; -1 if the track has none
+    private long emptyEditDuration = -1;
+
     public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata tikaMetadata,
                              XHTMLContentHandler xhtml) {
         super(metadata);
@@ -68,14 +79,22 @@ public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata ti
 
     @Override
     public boolean shouldAcceptBox(@NotNull String box) {
-        if (box.equals("udta") || box.equals("keys") || box.equals("ilst")) {
+        if (box.equals("udta") || box.equals("keys") || box.equals("ilst")
+                || box.equals("elst") || box.equals("stsd")) {
             return true;
         }
         return super.shouldAcceptBox(box);
     }
 
     @Override
     public boolean shouldAcceptContainer(@NotNull String box) {
+        //edts/minf/stbl are needed to reach the edit list and sample description
+        //of QuickTime timed metadata tracks (handler type 'mdta', for which the
+        //base handler keeps this handler active). Media tracks switch to their
+        //own handlers at 'hdlr', so their boxes never reach this handler.
+        if (box.equals("edts") || box.equals("minf") || box.equals("stbl")) {
+            return true;
+        }
         return super.shouldAcceptContainer(box);
     }
 
@@ -91,6 +110,27 @@ public Mp4Handler<?> processBox(@NotNull String box, @Nullable byte[] payload,
         } else if (box.equals("ilst")) {
             processQuickTimeItemList(payload);
             return this;
+        } else if (box.equals("mvhd")) {
+            processMovieTimescale(payload);
+            //fall through, the base handler extracts the other mvhd fields
+        } else if (box.equals("tkhd")) {
+            //a new track starts; forget the previous track's edit list
+            emptyEditDuration = -1;
+            //fall through to the base handler
+        } else if (box.equals("elst")) {
+            processEditList(payload);
+            return this;
+        } else if (box.equals("stsd")) {
+            processSampleDescription(payload);
+            return this;
+        } else if (box.equals("hdlr") && payload != null && payload.length >= 12
+                && payload[8] == 'm' && payload[9] == 'e'
+                && payload[10] == 't' && payload[11] == 'a') {
+            //timed metadata track (e.g. the Live Photo still-image-time track).
+            //The base handler would switch to Mp4MetaHandler, which extracts
+            //nothing, but would keep this handler from seeing the track's 'stsd'.
+            //Stay active instead.
+            return this;
         }
 
         return super.processBox(box, payload, size, context);
@@ -180,6 +220,137 @@ private void processQuickTimeItemList(@Nullable byte[] payload) {
         }
     }
 
+    private void processMovieTimescale(@Nullable byte[] payload) {
+        if (payload == null || payload.length < 16) {
+            return;
+        }
+        //1 byte version + 3 bytes flags, then creation/modification time,
+        //which are 4 bytes each in version 0 and 8 bytes each in version 1
+        int offset = payload[0] == 1 ? 20 : 12;
+        if (payload.length >= offset + 4) {
+            movieTimescale = readUInt32(payload, offset);
+        }
+    }
+
+    /**
+     * Parses an 'elst' edit list and remembers the duration of a leading empty edit
+     * (media time -1, expressed in movie timescale units). Apple writes the Live Photo
+     * still moment as such an empty edit shifting the single one-tick sample of the
+     * still-image-time track.
+     */
+    private void processEditList(@Nullable byte[] payload) {
+        if (payload == null || payload.length < 8) {
+            return;
+        }
+        int version = payload[0];
+        long entryCount = readUInt32(payload, 4);
+        int pos = 8;
+        int entrySize = version == 1 ? 20 : 12;
+        for (long i = 0; i < entryCount && pos + entrySize <= payload.length; i++) {
+            long segmentDuration;
+            long mediaTime;
+            if (version == 1) {
+                segmentDuration = readInt64(payload, pos);
+                mediaTime = readInt64(payload, pos + 8);
+            } else {
+                segmentDuration = readUInt32(payload, pos);
+                mediaTime = (int) readUInt32(payload, pos + 4);
+            }
+            if (mediaTime == -1) {
+                emptyEditDuration = segmentDuration;
+                return;
+            }
+            pos += entrySize;
+        }
+    }
+
+    /**
+     * Parses an 'stsd' sample description looking for timed metadata entries
+     * ('mebx', ISO 14496-12 boxed metadata). For every key the entry declares,
+     * the presentation start of the track is emitted in microseconds under
+     * {@code <key name>.track-start-us}. Only tracks delayed by a leading empty
+     * edit are reported; an undelayed track (start 0) carries no information.
+     * The main use case are Apple Live Photo videos, which place the single
+     * sample of the {@code com.apple.quicktime.still-image-time} track at the
+     * moment the paired still image was captured. See TIKA-4777.
+     */
+    private void processSampleDescription(@Nullable byte[] payload) {
+        if (payload == null || payload.length < 8 || movieTimescale <= 0
+                || emptyEditDuration <= 0) {
+            return;
+        }
+        long trackStartUs = emptyEditDuration * 1_000_000L / movieTimescale;
+        long entryCount = readUInt32(payload, 4);
+        int pos = 8;
+        for (long i = 0; i < entryCount && pos + 16 <= payload.length; i++) {
+            long entrySize = readUInt32(payload, pos);
+            if (entrySize < 16 || pos + entrySize > payload.length) {
+                return;
+            }
+            if ("mebx".equals(boxType(payload, pos + 4))) {
+                //the entry body starts after the 8 byte box header and the
+                //8 byte sample entry header (6 bytes reserved, 2 bytes dref)
+                for (String key : parseMebxKeyNames(payload, pos + 16,
+                        (int) (pos + entrySize))) {
+                    tikaMetadata.set(key + TRACK_START_SUFFIX,
+                            Long.toString(trackStartUs));
+                }
+            }
+            pos += (int) entrySize;
+        }
+    }
+
+    /**
+     * Extracts the key names declared by a 'mebx' sample entry: a 'keys' box
+     * containing one child box per key (typed by the local key id), each of
+     * which holds a 'keyd' key declaration of namespace plus key name.
+     */
+    private static List<String> parseMebxKeyNames(byte[] b, int start, int end) {
+        List<String> keyNames = new ArrayList<>();
+        int pos = start;
+        while (pos + 8 <= end) {
+            long size = readUInt32(b, pos);
+            if (size < 8 || pos + size > end) {
+                break;
+            }
+            if ("keys".equals(boxType(b, pos + 4))) {
+                int keyPos = pos + 8;
+                int keysEnd = (int) (pos + size);
+                while (keyPos + 8 <= keysEnd) {
+                    long keySize = readUInt32(b, keyPos);
+                    if (keySize < 8 || keyPos + keySize > keysEnd) {
+                        break;
+                    }
+                    int declPos = keyPos + 8;
+                    int keyEnd = (int) (keyPos + keySize);
+                    while (declPos + 8 <= keyEnd) {
+                        long declSize = readUInt32(b, declPos);
+                        if (declSize < 8 || declPos + declSize > keyEnd) {
+                            break;
+                        }
+                        //'keyd' payload: 4 bytes namespace (e.g. mdta), then the name
+                        if ("keyd".equals(boxType(b, declPos + 4)) && declSize > 12) {
+                            keyNames.add(new String(b, declPos + 12,
+                                    (int) declSize - 12, StandardCharsets.UTF_8));
+                        }
+                        declPos += (int) declSize;
+                    }
+                    keyPos += (int) keySize;
+                }
+            }
+            pos += (int) size;
+        }
+        return keyNames;
+    }
+
+    private static String boxType(byte[] b, int off) {
+        return new String(b, off, 4, StandardCharsets.ISO_8859_1);
+    }
+
+    private static long readInt64(byte[] b, int off) {
+        return (readUInt32(b, off) << 32) | readUInt32(b, off + 4);
+    }
+
     /**
      * Maps an ISO 6709 location string (latitude, longitude, optional altitude) to the
      * standard {@code geo:lat}/{@code geo:long}/{@code geo:alt} properties, in addition to

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java

@@ -307,6 +307,11 @@ public void testQuickTimeMetadataKeys() throws Exception {
                 metadata.get("com.apple.quicktime.camera.focal_length.35mm_equivalent"));
         assertEquals("1.5",
                 metadata.get("com.apple.quicktime.full-frame-rate-playback-intent"));
+
+        //the Live Photo still moment: presentation start of the timed metadata
+        //track declaring still-image-time (mebx, empty edit of 740/600s). TIKA-4777
+        assertEquals("1233333",
+                metadata.get("com.apple.quicktime.still-image-time.track-start-us"));
     }
 
     @Test