TIKA-4777: Expose presentation start of delayed QuickTime timed metadata tracks

tika-core/src/main/java/org/apache/tika/metadata/QuickTime.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata;
+
+/**
+ * Metadata properties specific to the QuickTime container family (.mov,
+ * .mp4, .m4a and friends). The container's com.apple.quicktime.* item-list
+ * keys pass through under their own names; the properties defined here are
+ * values Tika derives from structures beyond that key list.
+ */
+public interface QuickTime {
+
+    /**
+     * Presentation start of the com.apple.quicktime.still-image-time timed
+     * metadata track, in microseconds. In an Apple Live Photo video this is
+     * the moment the paired still image was captured. A value of 0 means the
+     * still is the very first frame; the property is absent when the file
+     * declares no such track. Only set when the track holds exactly one
+     * sample (the Live Photo shape): the sample marks a point in time, its
+     * own one-tick duration is filler, so there is deliberately no
+     * corresponding end property.
+     */
+    Property STILL_IMAGE_TIME = Property.internalReal("quicktime:still-image-time");
+}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java

@@ -32,6 +32,7 @@
 import com.drew.metadata.Metadata;
 import com.drew.metadata.mp4.Mp4BoxHandler;
 import com.drew.metadata.mp4.Mp4Context;
+import com.drew.metadata.mp4.Mp4Directory;
 import org.xml.sax.SAXException;
 
 import org.apache.tika.metadata.TikaCoreProperties;
@@ -52,13 +53,18 @@ public class TikaMp4BoxHandler extends Mp4BoxHandler {
     private static final Pattern ISO6709_PATTERN =
             Pattern.compile("([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)?");
 
+
     org.apache.tika.metadata.Metadata tikaMetadata;
     final XHTMLContentHandler xhtml;
 
     //key names for the current 'meta' box, filled from its 'keys' box and consumed
     //by the following 'ilst' box (e.g. com.apple.quicktime.content.identifier)
     private final List<String> quickTimeMetadataKeys = new ArrayList<>();
 
+    //duration of the current track's leading empty edit(s) ('elst' entries
+    //with media time -1), in movie timescale units; -1 if the track has none
+    private long emptyEditDuration = -1;
+
     public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata tikaMetadata,
                              XHTMLContentHandler xhtml) {
         super(metadata);
@@ -68,29 +74,52 @@ public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata ti
 
     @Override
     public boolean shouldAcceptBox(@NotNull String box) {
-        if (box.equals("udta") || box.equals("keys") || box.equals("ilst")) {
+        if (box.equals("udta") || box.equals("keys") || box.equals("ilst")
+                || box.equals("elst")) {
             return true;
         }
         return super.shouldAcceptBox(box);
     }
 
     @Override
     public boolean shouldAcceptContainer(@NotNull String box) {
+        //edts is needed to reach a track's edit list, which the base handler
+        //skips; the edit list defines the presentation start of the track
+        if (box.equals("edts")) {
+            return true;
+        }
         return super.shouldAcceptContainer(box);
     }
 
     @Override
     public Mp4Handler<?> processBox(@NotNull String box, @Nullable byte[] payload,
                                     long size, Mp4Context context)
             throws IOException {
-        if (box.equals("udta")) {
+        if (payload == null && box.equals("trak")) {
+            //a new track starts (containers route through here with a null
+            //payload); forget the previous track's edit list
+            emptyEditDuration = -1;
+        } else if (box.equals("udta")) {
             return processUserData(box, payload, context);
         } else if (box.equals("keys")) {
             processQuickTimeKeys(payload);
             return this;
         } else if (box.equals("ilst")) {
             processQuickTimeItemList(payload);
             return this;
+        } else if (box.equals("elst")) {
+            processEditList(payload);
+            return this;
+        } else if (box.equals("hdlr") && payload != null && payload.length >= 12
+                && payload[8] == 'm' && payload[9] == 'e'
+                && payload[10] == 't' && payload[11] == 'a') {
+            //timed metadata track (e.g. the Live Photo still-image-time track):
+            //hand over to our handler, which extracts the mebx key declarations
+            //(the base Mp4MetaHandler accepts the track's boxes but extracts
+            //nothing from them)
+            Long movieTimescale = directory.getLongObject(Mp4Directory.TAG_TIME_SCALE);
+            return new TikaMp4MetaHandler(metadata, context, tikaMetadata,
+                    emptyEditDuration, movieTimescale == null ? 0 : movieTimescale);
         }
 
         return super.processBox(box, payload, size, context);
@@ -180,6 +209,45 @@ private void processQuickTimeItemList(@Nullable byte[] payload) {
         }
     }
 
+
+    /**
+     * Parses an 'elst' edit list and remembers the total duration of the leading
+     * empty edits (media time -1, expressed in movie timescale units), which is
+     * what delays the track's presentation start. Empty edits after the first
+     * normal entry do not delay the track and are ignored. Apple writes the Live
+     * Photo still moment as such an empty edit shifting the single one-tick
+     * sample of the still-image-time track.
+     */
+    private void processEditList(@Nullable byte[] payload) {
+        if (payload == null || payload.length < 8) {
+            return;
+        }
+        int version = payload[0];
+        long entryCount = readUInt32(payload, 4);
+        int pos = 8;
+        int entrySize = version == 1 ? 20 : 12;
+        long leadingEmptyEdits = 0;
+        for (long i = 0; i < entryCount && pos + entrySize <= payload.length; i++) {
+            long segmentDuration;
+            long mediaTime;
+            if (version == 1) {
+                segmentDuration = readInt64(payload, pos);
+                mediaTime = readInt64(payload, pos + 8);
+            } else {
+                segmentDuration = readUInt32(payload, pos);
+                mediaTime = (int) readUInt32(payload, pos + 4);
+            }
+            if (mediaTime != -1) {
+                break;
+            }
+            leadingEmptyEdits += segmentDuration;
+            pos += entrySize;
+        }
+        if (leadingEmptyEdits > 0) {
+            emptyEditDuration = leadingEmptyEdits;
+        }
+    }
+
     /**
      * Maps an ISO 6709 location string (latitude, longitude, optional altitude) to the
      * standard {@code geo:lat}/{@code geo:long}/{@code geo:alt} properties, in addition to
@@ -228,4 +296,8 @@ private static long readUInt32(byte[] b, int off) {
         return ((b[off] & 0xFFL) << 24) | ((b[off + 1] & 0xFFL) << 16)
                 | ((b[off + 2] & 0xFFL) << 8) | (b[off + 3] & 0xFFL);
     }
+
+    private static long readInt64(byte[] b, int off) {
+        return (readUInt32(b, off) << 32) | readUInt32(b, off + 4);
+    }
 }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4MetaHandler.java

@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.mp4;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.drew.lang.SequentialReader;
+import com.drew.metadata.Metadata;
+import com.drew.metadata.mp4.Mp4Context;
+import com.drew.metadata.mp4.media.Mp4MetaHandler;
+
+import org.apache.tika.metadata.QuickTime;
+
+/**
+ * Handles a QuickTime timed metadata track (handler type 'meta'). The base
+ * {@link Mp4MetaHandler} accepts the track's 'stsd' and 'stts' boxes but
+ * extracts nothing from them; this subclass reads the key names declared by
+ * 'mebx' sample descriptions (ISO 14496-12 boxed metadata) and, for a
+ * single-sample track declaring {@code com.apple.quicktime.still-image-time},
+ * emits the presentation time of that sample as
+ * {@link QuickTime#STILL_IMAGE_TIME}.
+ * <p>
+ * Apple Live Photo videos mark the moment the paired still image was
+ * captured this way: a leading empty edit delays the track's single one-tick
+ * sample to the still moment. Without a leading empty edit the sample
+ * presents at 0, so 0 is emitted (the still is the first frame); with more
+ * than one sample the track's start is not the time of any single sample and
+ * nothing is emitted. The sample value itself is a constant -1 marker, so
+ * the moov boxes alone are sufficient and mdat is never read. See TIKA-4777.
+ */
+class TikaMp4MetaHandler extends Mp4MetaHandler {
+
+    static final String STILL_IMAGE_TIME_KEY = "com.apple.quicktime.still-image-time";
+
+    private final org.apache.tika.metadata.Metadata tikaMetadata;
+    //duration of the track's leading empty edit in movie timescale units,
+    //or -1 if the track is not delayed
+    private final long emptyEditDuration;
+    private final long movieTimescale;
+    private final List<String> keyNames = new ArrayList<>();
+    private long sampleCount = -1;
+
+    TikaMp4MetaHandler(Metadata metadata, Mp4Context context,
+                       org.apache.tika.metadata.Metadata tikaMetadata,
+                       long emptyEditDuration, long movieTimescale) {
+        super(metadata, context);
+        this.tikaMetadata = tikaMetadata;
+        this.emptyEditDuration = emptyEditDuration;
+        this.movieTimescale = movieTimescale;
+    }
+
+    @Override
+    protected void processSampleDescription(SequentialReader reader) throws IOException {
+        reader.skip(4); //1 byte version + 3 bytes flags
+        long entryCount = reader.getUInt32();
+        for (long i = 0; i < entryCount; i++) {
+            long entrySize = reader.getUInt32();
+            String format = reader.getString(4, StandardCharsets.ISO_8859_1);
+            //also reject sizes beyond the remaining payload: a crafted size
+            //like 0xFFFFFFFF would turn negative in the int cast below
+            if (entrySize < 16 || entrySize - 8 > reader.available()) {
+                return;
+            }
+            byte[] entry = reader.getBytes((int) entrySize - 8);
+            if ("mebx".equals(format)) {
+                //6 bytes reserved + 2 bytes data reference index, then the boxes
+                parseMebxKeyNames(entry, 8, entry.length, keyNames);
+            }
+        }
+        maybeEmit();
+    }
+
+    @Override
+    protected void processTimeToSample(SequentialReader reader, Mp4Context context)
+            throws IOException {
+        reader.skip(4); //1 byte version + 3 bytes flags
+        long entryCount = reader.getUInt32();
+        long samples = 0;
+        for (long i = 0; i < entryCount; i++) {
+            samples += reader.getUInt32(); //sample count
+            reader.skip(4);                //sample delta
+        }
+        sampleCount = samples;
+        maybeEmit();
+    }
+
+    /**
+     * Emits once the sample description and the time-to-sample table have both
+     * been seen (their order within 'stbl' is not fixed), and only when the
+     * conditions from the class comment hold.
+     */
+    private void maybeEmit() {
+        if (sampleCount != 1 || !keyNames.contains(STILL_IMAGE_TIME_KEY)) {
+            return;
+        }
+        if (emptyEditDuration <= 0) {
+            //no leading empty edit: the sample presents at 0, the still is
+            //the first frame
+            tikaMetadata.set(QuickTime.STILL_IMAGE_TIME, 0L);
+        } else if (movieTimescale > 0
+                && emptyEditDuration <= Long.MAX_VALUE / 1_000_000L) {
+            tikaMetadata.set(QuickTime.STILL_IMAGE_TIME,
+                    emptyEditDuration * 1_000_000L / movieTimescale);
+        }
+        keyNames.clear();
+    }
+
+    /**
+     * Extracts the key names declared by a 'mebx' sample entry: a 'keys' box
+     * containing one child box per key (typed by the local key id), each of
+     * which holds a 'keyd' key declaration of namespace plus key name.
+     */
+    private static void parseMebxKeyNames(byte[] b, int start, int end, List<String> keyNames) {
+        int pos = start;
+        while (pos + 8 <= end) {
+            long size = readUInt32(b, pos);
+            if (size < 8 || pos + size > end) {
+                break;
+            }
+            if ("keys".equals(boxType(b, pos + 4))) {
+                int keyPos = pos + 8;
+                int keysEnd = (int) (pos + size);
+                while (keyPos + 8 <= keysEnd) {
+                    long keySize = readUInt32(b, keyPos);
+                    if (keySize < 8 || keyPos + keySize > keysEnd) {
+                        break;
+                    }
+                    int declPos = keyPos + 8;
+                    int keyEnd = (int) (keyPos + keySize);
+                    while (declPos + 8 <= keyEnd) {
+                        long declSize = readUInt32(b, declPos);
+                        if (declSize < 8 || declPos + declSize > keyEnd) {
+                            break;
+                        }
+                        //'keyd' payload: 4 bytes namespace (e.g. mdta), then the name
+                        if ("keyd".equals(boxType(b, declPos + 4)) && declSize > 12) {
+                            keyNames.add(new String(b, declPos + 12,
+                                    (int) declSize - 12, StandardCharsets.UTF_8));
+                        }
+                        declPos += (int) declSize;
+                    }
+                    keyPos += (int) keySize;
+                }
+            }
+            pos += (int) size;
+        }
+    }
+
+    private static String boxType(byte[] b, int off) {
+        return new String(b, off, 4, StandardCharsets.ISO_8859_1);
+    }
+
+    private static long readUInt32(byte[] b, int off) {
+        return ((b[off] & 0xFFL) << 24) | ((b[off + 1] & 0xFFL) << 16)
+                | ((b[off + 2] & 0xFFL) << 8) | (b[off + 3] & 0xFFL);
+    }
+}