Skip to content

Commit d88bb6f

Browse files
clainclycopybara-github
claincly
authored and
copybara-github
committed
Add GlEffectProcessor
PiperOrigin-RevId: 917373651
1 parent 2bf62ec commit d88bb6f

3 files changed

Lines changed: 743 additions & 0 deletions

File tree

‎libraries/effect/src/main/java/androidx/media3/effect/GlShaderProgramAdapter.java‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
/*
2+
* Copyright 2026 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package androidx.media3.effect;
17+
18+
import static com.google.common.base.Preconditions.checkNotNull;
19+
import static com.google.common.base.Preconditions.checkState;
20+
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
21+
22+
import androidx.annotation.Nullable;
23+
import androidx.media3.common.Format;
24+
import androidx.media3.common.GlObjectsProvider;
25+
import androidx.media3.common.GlTextureInfo;
26+
import androidx.media3.common.VideoFrameProcessingException;
27+
import androidx.media3.common.util.Consumer;
28+
import androidx.media3.common.util.ExperimentalApi;
29+
import androidx.media3.effect.GlShaderProgram.InputListener;
30+
import androidx.media3.effect.GlShaderProgram.OutputListener;
31+
import androidx.media3.effect.GlTextureFrameConsumer.GlTextureFrameProcessor;
32+
import java.util.ArrayDeque;
33+
import java.util.HashMap;
34+
import java.util.Map;
35+
import java.util.Queue;
36+
import java.util.concurrent.Executor;
37+
import org.checkerframework.checker.initialization.qual.Initialized;
38+
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
39+
40+
/**
41+
* Bridges a legacy {@link GlShaderProgram} to {@link GlTextureFrameProcessor}.
42+
*
43+
* <p>Methods must be called on the dedicated OpenGL thread.
44+
*
45+
* <p>The underlying {@link GlShaderProgram} must not alter frame presentation times.
46+
*
47+
* <p>Currently, only supports {@link GlShaderProgram GlShaderPrograms} that output the same amount
48+
* of frames as input.
49+
*/
50+
@ExperimentalApi // TODO: b/505721737 Remove once FrameProcessor is production ready.
51+
/* package */ final class GlShaderProgramAdapter
52+
implements GlTextureFrameProcessor, InputListener, OutputListener {
53+
54+
private final GlShaderProgram glShaderProgram;
55+
private final GlObjectsProvider glObjectsProvider;
56+
// Map from texId -> input Frame
57+
private final Map<Integer, GlTextureFrame> inFlightFrames;
58+
private final Map<Long, Format> inputFormats;
59+
private final Queue<GlTextureFrame> pendingOutputFrames;
60+
private final Executor glExecutor;
61+
private final Consumer<VideoFrameProcessingException> errorConsumer;
62+
63+
private @MonotonicNonNull GlTextureFrameConsumer downstreamConsumer;
64+
@Nullable private Runnable pendingWakeupListener;
65+
@Nullable private Executor pendingWakeupExecutor;
66+
private int shaderInputCapacity;
67+
68+
public GlShaderProgramAdapter(
69+
GlShaderProgram glShaderProgram,
70+
GlObjectsProvider glObjectsProvider,
71+
Executor glExecutor,
72+
Consumer<VideoFrameProcessingException> errorConsumer) {
73+
this.glShaderProgram = glShaderProgram;
74+
this.glObjectsProvider = glObjectsProvider;
75+
this.glExecutor = glExecutor;
76+
this.errorConsumer = errorConsumer;
77+
@SuppressWarnings("nullness:assignment")
78+
@Initialized
79+
GlShaderProgramAdapter thisRef = this;
80+
this.glShaderProgram.setInputListener(thisRef);
81+
this.glShaderProgram.setOutputListener(thisRef);
82+
this.glShaderProgram.setErrorListener(directExecutor(), errorConsumer::accept);
83+
inFlightFrames = new HashMap<>();
84+
pendingOutputFrames = new ArrayDeque<>();
85+
inputFormats = new HashMap<>();
86+
}
87+
88+
@Override
89+
public void setOutput(GlTextureFrameConsumer downstreamConsumer) {
90+
checkState(inFlightFrames.isEmpty());
91+
this.downstreamConsumer = downstreamConsumer;
92+
}
93+
94+
@Override
95+
public boolean queue(GlTextureFrame frame, Executor listenerExecutor, Runnable wakeupListener) {
96+
// TODO: b/505721737 - Support frame dropping and repeating.
97+
inputFormats.put(frame.presentationTimeUs, frame.format);
98+
if (shaderInputCapacity > 0) {
99+
shaderInputCapacity--;
100+
inFlightFrames.put(frame.glTextureInfo.texId, frame);
101+
glShaderProgram.queueInputFrame(
102+
glObjectsProvider, frame.glTextureInfo, frame.presentationTimeUs);
103+
return true;
104+
} else {
105+
// We keep only the last wakeup, if queue() is called repetitively
106+
pendingWakeupListener = wakeupListener;
107+
pendingWakeupExecutor = listenerExecutor;
108+
return false;
109+
}
110+
}
111+
112+
@Override
113+
public void onReadyToAcceptInputFrame() {
114+
shaderInputCapacity++;
115+
if (pendingWakeupListener != null && pendingWakeupExecutor != null) {
116+
pendingWakeupExecutor.execute(pendingWakeupListener);
117+
pendingWakeupListener = null;
118+
pendingWakeupExecutor = null;
119+
}
120+
}
121+
122+
@Override
123+
public void onInputFrameProcessed(GlTextureInfo inputTexture) {
124+
// TODO: b/505721737 - GlTextureFrame inherit from Frame, simplify the release method.
125+
checkNotNull(inFlightFrames.remove(inputTexture.texId)).release(/* releaseFence= */ null);
126+
}
127+
128+
@Override
129+
public void onFlush() {
130+
// TODO: b/505721737 - Support seeking.
131+
throw new UnsupportedOperationException();
132+
}
133+
134+
@Override
135+
public void onOutputFrameAvailable(GlTextureInfo outputTexture, long presentationTimeUs) {
136+
// We don't support GlShaderPrograms that modify presentationTimeUs.
137+
Format inputFormat = checkNotNull(inputFormats.remove(presentationTimeUs));
138+
GlTextureFrame outputGlTextureFrame =
139+
new GlTextureFrame.Builder(
140+
outputTexture,
141+
// We are always on the GL thread
142+
directExecutor(),
143+
/* releaseTextureCallback= */ glShaderProgram::releaseOutputFrame)
144+
// TODO: b/505721737 - Set sequence presentation time.
145+
.setPresentationTimeUs(presentationTimeUs)
146+
.setFormat(
147+
inputFormat
148+
.buildUpon()
149+
// TODO: b/505721737 - set color info for tone mapping.
150+
.setWidth(outputTexture.width)
151+
.setHeight(outputTexture.height)
152+
.build())
153+
.build();
154+
155+
pendingOutputFrames.add(outputGlTextureFrame);
156+
tryQueueFramesDownstream();
157+
}
158+
159+
@Override
160+
public void onCurrentOutputStreamEnded() {
161+
checkNotNull(downstreamConsumer).signalEndOfStream();
162+
}
163+
164+
/** Called on the GL thread. */
165+
private void tryQueueFramesDownstream() {
166+
checkNotNull(downstreamConsumer);
167+
while (!pendingOutputFrames.isEmpty()) {
168+
GlTextureFrame frame = checkNotNull(pendingOutputFrames.peek());
169+
try {
170+
boolean queued =
171+
downstreamConsumer.queue(
172+
frame, glExecutor, /* wakeupListener= */ this::tryQueueFramesDownstream);
173+
if (queued) {
174+
pendingOutputFrames.remove();
175+
} else {
176+
break;
177+
}
178+
} catch (VideoFrameProcessingException e) {
179+
errorConsumer.accept(e);
180+
return;
181+
}
182+
}
183+
}
184+
185+
@Override
186+
public void signalEndOfStream() {
187+
glShaderProgram.signalEndOfCurrentInputStream();
188+
}
189+
190+
@Override
191+
public void close() throws VideoFrameProcessingException {
192+
glShaderProgram.release();
193+
if (!inFlightFrames.isEmpty()) {
194+
for (GlTextureFrame frame : inFlightFrames.values()) {
195+
frame.release(/* releaseFence= */ null);
196+
}
197+
inFlightFrames.clear();
198+
}
199+
while (!pendingOutputFrames.isEmpty()) {
200+
pendingOutputFrames.remove().release(/* releaseFence= */ null);
201+
}
202+
inputFormats.clear();
203+
}
204+
}

‎libraries/effect/src/main/java/androidx/media3/effect/GlTextureFrameConsumer.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright 2026 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package androidx.media3.effect;
17+
18+
import androidx.media3.common.VideoFrameProcessingException;
19+
import androidx.media3.common.util.ExperimentalApi;
20+
import androidx.media3.common.video.FrameProcessor;
21+
import java.util.concurrent.Executor;
22+
23+
/**
24+
* Consumes a {@link GlTextureFrame}.
25+
*
26+
* <p>This interface aims at resembling {@link FrameProcessor}.
27+
*/
28+
@ExperimentalApi // TODO: b/505721737 Remove once FrameProcessor is production ready.
29+
public interface GlTextureFrameConsumer extends AutoCloseable {
30+
31+
/** Allowing outputting {@link GlTextureFrame} from the {@link GlTextureFrameConsumer}. */
32+
interface GlTextureFrameProcessor extends GlTextureFrameConsumer {
33+
/** Sets the {@link GlTextureFrameConsumer} to which this processor outputs. */
34+
void setOutput(GlTextureFrameConsumer downstream);
35+
}
36+
37+
// TODO: b/505721737 - Consider extending to multiple frame input for compositor implementation.
38+
/**
39+
* Attempts to queue a frame for processing.
40+
*
41+
* <p>If the consumer is at capacity, it returns {@code false} and the {@code wakeupListener} will
42+
* be invoked on the {@code listenerExecutor} when capacity becomes available.
43+
*
44+
* @param frame The input frame to process.
45+
* @param listenerExecutor The executor to run the wakeupListener on.
46+
* @param wakeupListener The callback invoked when capacity is freed.
47+
* @return true if queued successfully, false if at capacity.
48+
*/
49+
boolean queue(GlTextureFrame frame, Executor listenerExecutor, Runnable wakeupListener)
50+
throws VideoFrameProcessingException;
51+
52+
/** Notifies the current stream has ended. */
53+
void signalEndOfStream();
54+
55+
@Override
56+
void close() throws VideoFrameProcessingException;
57+
}

0 commit comments

Comments
 (0)