[android] Refine video render algorithm (#5551)

jasonzhangxx · web-flow · commit fc769cbc35a3 · 2025-11-03T11:01:29.000-08:00
1. Allow VideoRenderAlgorithm release first frame as soon as possible.
2. Besides the first frame, other frames will be released after any
   audio frame has been consumed.

Bug: 186660620
diff --git a/starboard/android/shared/audio_track_audio_sink_type.cc b/starboard/android/shared/audio_track_audio_sink_type.cc
@@ -15,7 +15,9 @@
 #include "starboard/android/shared/audio_track_audio_sink_type.h"
 
 #include <unistd.h>
+
 #include <algorithm>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -27,6 +29,7 @@
 #include "starboard/common/scoped_timer.h"
 #include "starboard/common/string.h"
 #include "starboard/common/time.h"
+#include "starboard/shared/starboard/features.h"
 #include "starboard/shared/starboard/media/media_util.h"
 #include "starboard/shared/starboard/player/filter/common.h"
 #include "starboard/thread.h"
@@ -215,6 +218,9 @@ void AudioTrackAudioSink::AudioThreadFunc() {
 
   int last_playback_head_position = 0;
 
+  bool release_frames_after_audio_starts = features::FeatureList::IsEnabled(
+      features::kReleaseVideoFramesAfterAudioStarts);
+
   while (!quit_) {
     int playback_head_position = 0;
     int64_t frames_consumed_at = 0;
@@ -282,6 +288,12 @@ void AudioTrackAudioSink::AudioThreadFunc() {
       last_playback_head_event_at = -1;
       ScopedTimer timer("Play");
       bridge_.Play();
+      if (release_frames_after_audio_starts) {
+        // To promptly re-evaluate and update audio state, we restart the loop
+        // after calling AudioTrack.play() on Android, as this operation often
+        // takes hundreds of milliseconds.
+        continue;
+      }
     }
 
     if (!is_playing || frames_in_buffer == 0) {
diff --git a/starboard/android/shared/video_render_algorithm.cc b/starboard/android/shared/video_render_algorithm.cc
@@ -15,12 +15,13 @@
 #include "starboard/android/shared/video_render_algorithm.h"
 
 #include <algorithm>
+#include <list>
 
+#include "cobalt/android/jni_headers/VideoFrameReleaseTimeHelper_jni.h"
 #include "starboard/android/shared/media_common.h"
 #include "starboard/common/check_op.h"
 #include "starboard/common/log.h"
-
-#include "cobalt/android/jni_headers/VideoFrameReleaseTimeHelper_jni.h"
+#include "starboard/shared/starboard/features.h"
 
 namespace starboard {
 
@@ -43,7 +44,10 @@ jlong GetSystemNanoTime() {
 VideoRenderAlgorithmAndroid::VideoRenderAlgorithmAndroid(
     MediaCodecVideoDecoder* video_decoder,
     VideoFrameTracker* frame_tracker)
-    : video_decoder_(video_decoder), frame_tracker_(frame_tracker) {
+    : video_decoder_(video_decoder),
+      frame_tracker_(frame_tracker),
+      release_frames_after_audio_starts_(features::FeatureList::IsEnabled(
+          features::kReleaseVideoFramesAfterAudioStarts)) {
   SB_CHECK(video_decoder_);
   video_decoder_->SetPlaybackRate(playback_rate_);
 }
@@ -70,9 +74,22 @@ void VideoRenderAlgorithmAndroid::Render(
     double playback_rate;
     int64_t playback_time = media_time_provider->GetCurrentMediaTime(
         &is_audio_playing, &is_audio_eos_played, &is_underflow, &playback_rate);
-    if (!is_audio_playing) {
-      break;
+
+    if (release_frames_after_audio_starts_) {
+      // After the first frame, stop rendering if audio isn't playing, or if
+      // audio playback hasn't advanced past the current seek_to_time (or
+      // initial time 0). This ensures video doesn't run ahead if audio is
+      // stalled or hasn't consumed frames yet.
+      if (first_frame_released_ &&
+          (!is_audio_playing || playback_time == seek_to_time_)) {
+        break;
+      }
+    } else {
+      if (!is_audio_playing) {
+        break;
+      }
     }
+
     if (playback_rate != playback_rate_) {
       playback_rate_ = playback_rate;
       video_decoder_->SetPlaybackRate(playback_rate);
@@ -116,6 +133,7 @@ void VideoRenderAlgorithmAndroid::Render(
           draw_frame_cb(frames->front(), adjusted_release_time_ns);
       SB_DCHECK_EQ(status, VideoRendererSink::kReleased);
       frames->pop_front();
+      first_frame_released_ = true;
     } else {
       break;
     }
@@ -126,6 +144,8 @@ void VideoRenderAlgorithmAndroid::Seek(int64_t seek_to_time) {
   if (frame_tracker_) {
     frame_tracker_->Seek(seek_to_time);
   }
+  first_frame_released_ = false;
+  seek_to_time_ = seek_to_time;
 }
 
 int VideoRenderAlgorithmAndroid::GetDroppedFrames() {
diff --git a/starboard/android/shared/video_render_algorithm.h b/starboard/android/shared/video_render_algorithm.h
@@ -53,6 +53,10 @@ class VideoRenderAlgorithmAndroid : public VideoRenderAlgorithm {
   double playback_rate_ = 1.0;
   VideoFrameReleaseTimeHelper video_frame_release_time_helper_;
   int dropped_frames_ = 0;
+  bool first_frame_released_ = false;
+  int64_t seek_to_time_ = 0;  // microseconds
+
+  const bool release_frames_after_audio_starts_;
 };
 
 }  // namespace starboard
diff --git a/starboard/extension/feature_config.h b/starboard/extension/feature_config.h
@@ -122,6 +122,13 @@ STARBOARD_FEATURE(kForceResetAudioDecoder, "ForceResetAudioDecoder", false)
 // enabling tunnel mode on all playbacks.
 STARBOARD_FEATURE(kForceTunnelMode, "ForceTunnelMode", false)
 
+// Cobalt VideoRenderAlgorithm used to release video frames immediately after
+// playback starts. Set the following variable to true to make it release video
+// frames until the underlying audio sink actually starts.
+STARBOARD_FEATURE(kReleaseVideoFramesAfterAudioStarts,
+                  "ReleaseVideoFramesAfterAudioStarts",
+                  false)
+
 // By default, set the following to true to use stub decoder as audio/video
 // decoder.
 STARBOARD_FEATURE(kUseStubAudioDecoder, "UseStubAudioDecoder", false)
@@ -135,6 +142,7 @@ STARBOARD_FEATURE(kUseStubVideoDecoder, "UseStubVideoDecoder", false)
 STARBOARD_FEATURE(kVideoDecoderDelayUsecOverride,
                   "VideoDecoderDelayUsecOverride",
                   false)
+
 #endif  // BUILDFLAG(IS_ANDROID) && (SB_API_VERSION >= 17)
 FEATURE_LIST_END
 
diff --git a/starboard/shared/starboard/player/player_internal.h b/starboard/shared/starboard/player/player_internal.h
@@ -79,7 +79,11 @@ class SbPlayerPrivateImpl final : public SbPlayerPrivate {
   void WriteEndOfStream(SbMediaType stream_type) final;
   void SetBounds(int z_index, int x, int y, int width, int height) final;
   void GetInfo(SbPlayerInfo* out_player_info) final;
+
+  // TODO (b/456786219): as SbPlayer doesn't support pause, we should remove
+  // unused pause related functions.
   void SetPause(bool pause) final;
+
   void SetPlaybackRate(double playback_rate) final;
   void SetVolume(double volume) final;
 
