Fix DataSource.seek implementation, fix thumbnails upside down

Author: natario1

lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt

@@ -97,7 +97,7 @@ internal class DefaultThumbnailsEngine(
             Seeker(source, positions) { it == stubs.firstOrNull()?.localizedUs } +
                     Reader(source, type) +
                     Decoder(source.getTrackFormat(type)!!, continuous = false) +
-                    VideoRenderer(source.orientation, rotation, outputFormat) +
+                    VideoRenderer(source.orientation, rotation, outputFormat, flipY = true) +
                     VideoSnapshots(outputFormat, positions, 50 * 1000) { pos, bitmap ->
                         val stub = stubs.removeFirst()
                         stub.actualLocalizedUs = pos

lib/src/main/java/com/otaliastudios/transcoder/internal/video/FrameDrawer.java

@@ -40,6 +40,7 @@ class FrameDrawer {
     private float mScaleX = 1F;
     private float mScaleY = 1F;
     private int mRotation = 0;
+    private boolean mFlipY = false;
 
     @GuardedBy("mFrameAvailableLock")
     private boolean mFrameAvailable;
@@ -95,6 +96,10 @@ public void setRotation(int rotation) {
         mRotation = rotation;
     }
 
+    public void setFlipY(boolean flipY) {
+        mFlipY = flipY;
+    }
+
     /**
      * Returns a Surface to draw onto.
      * @return the output surface
@@ -169,10 +174,14 @@ private void drawNewFrame() {
         Matrix.translateM(mProgram.getTextureTransform(), 0, glTranslX, glTranslY, 0);
         // Scale.
         Matrix.scaleM(mProgram.getTextureTransform(), 0, glScaleX, glScaleY, 1);
-        // Apply rotation.
+        // Apply rotation and flip.
         Matrix.translateM(mProgram.getTextureTransform(), 0, 0.5F, 0.5F, 0);
         Matrix.rotateM(mProgram.getTextureTransform(), 0, mRotation, 0, 0, 1);
+        if (mFlipY) {
+            Matrix.scaleM(mProgram.getTextureTransform(), 0, 1F, -1F, 1F);
+        }
         Matrix.translateM(mProgram.getTextureTransform(), 0, -0.5F, -0.5F, 0);
+
         // Draw.
         mProgram.draw(mDrawable);
     }

lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoRenderer.kt

@@ -16,14 +16,19 @@ internal class VideoRenderer(
         private val sourceRotation: Int, // intrinsic source rotation
         private val extraRotation: Int, // any extra rotation in TranscoderOptions
         private val targetFormat: MediaFormat,
+        flipY: Boolean = false
 ): Step<DecoderData, DecoderChannel, Long, Channel>, DecoderChannel {
 
     private val log = Logger("VideoRenderer")
 
     override val channel = this
 
     // frame drawer needs EGL context which is not created by us, so let's use by lazy.
-    private val frameDrawer by lazy { FrameDrawer() }
+    private val frameDrawer by lazy {
+        val drawer = FrameDrawer()
+        drawer.setFlipY(flipY)
+        drawer
+    }
 
     private lateinit var frameDropper: FrameDropper
 

lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java

@@ -43,6 +43,7 @@ public abstract class DefaultDataSource implements DataSource {
 
     @Override
     public void initialize() {
+        LOG.i("initialize(): initializing...");
         mExtractor = new MediaExtractor();
         try {
             initializeExtractor(mExtractor);
@@ -67,13 +68,27 @@ public void initialize() {
         // This is very important to have a timebase e.g. for seeks that happen before any read.
         for (int i = 0; i < mExtractor.getTrackCount(); i++) mExtractor.selectTrack(i);
         mOriginUs = mExtractor.getSampleTime();
+        LOG.v("initialize(): found origin=" + mOriginUs);
         for (int i = 0; i < mExtractor.getTrackCount(); i++) mExtractor.unselectTrack(i);
         mInitialized = true;
+
+        // Debugging mOriginUs issues.
+        /* LOG.v("initialize(): origin after unselect is" + mExtractor.getSampleTime());
+        if (getTrackFormat(TrackType.VIDEO) != null) {
+            mExtractor.selectTrack(mIndex.getVideo());
+            LOG.v("initialize(): video only origin is" + mExtractor.getSampleTime());
+            mExtractor.unselectTrack(mIndex.getVideo());
+        }
+        if (getTrackFormat(TrackType.AUDIO) != null) {
+            mExtractor.selectTrack(mIndex.getAudio());
+            LOG.v("initialize(): audio only origin is" + mExtractor.getSampleTime());
+            mExtractor.unselectTrack(mIndex.getAudio());
+        } */
     }
 
     @Override
     public void deinitialize() {
-        LOG.i("deinitialize(): releasing...");
+        LOG.i("deinitialize(): deinitializing...");
         try {
             mExtractor.release();
         } catch (Exception e) {
@@ -124,6 +139,7 @@ public long seekTo(long desiredPositionUs) {
         boolean hasAudio = mSelectedTracks.contains(TrackType.AUDIO);
         LOG.i("seekTo(): seeking to " + (mOriginUs + desiredPositionUs)
                 + " originUs=" + mOriginUs
+                + " extractorUs=" + mExtractor.getSampleTime()
                 + " externalUs=" + desiredPositionUs
                 + " hasVideo=" + hasVideo
                 + " hasAudio=" + hasAudio);
@@ -140,17 +156,18 @@ public long seekTo(long desiredPositionUs) {
         }
         mDontRenderRangeStart = mExtractor.getSampleTime();
         mDontRenderRangeEnd = mOriginUs + desiredPositionUs;
-
         if (mDontRenderRangeStart > mDontRenderRangeEnd) {
-            throw new IllegalStateException("The dontRenderRange has unexpected values! " +
-                    "start=" + mDontRenderRangeStart + ", " +
-                    "end=" + mDontRenderRangeEnd);
-        } else {
-            LOG.i("seekTo(): dontRenderRange=" +
-                    mDontRenderRangeStart + ".." +
-                    mDontRenderRangeEnd + " (" +
-                    (mDontRenderRangeEnd - mDontRenderRangeStart) + "us)");
+            // Extractor jumped beyond the requested point!
+            // This can happen in edge cases because we compute mOriginUs with both tracks selected,
+            // while source can later be used with a single track. E.g. audio track starts at 0,
+            // video track starts at 20000, mOriginUs is 0. A seekTo(0) will give range=20000..0.
+            // In this case, range should just be empty.
+            mDontRenderRangeStart = mDontRenderRangeEnd; // 0..0
         }
+        LOG.i("seekTo(): dontRenderRange=" +
+                mDontRenderRangeStart + ".." +
+                mDontRenderRangeEnd + " (" +
+                (mDontRenderRangeEnd - mDontRenderRangeStart) + "us)");
         return mExtractor.getSampleTime() - mOriginUs;
     }