From edf85d39c6497f33bf8501e202fa4b7eddc98646 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Sun, 9 Apr 2023 12:08:49 -0400 Subject: LibWeb: Port HTMLVideoElement to play videos with Video::PlaybackManager This has several advantages over the current manual demuxing currently being performed. PlaybackManager hides the specific demuxer being used, which will allow more codecs to be added transparently to LibWeb. It also provides buffering and controls playback rate for us. Further, it will allow us to much more easily implement the "media timeline" to render a timestamp and implement seeking. --- .../Libraries/LibWeb/HTML/HTMLMediaElement.cpp | 37 +++++++-- .../Libraries/LibWeb/HTML/HTMLVideoElement.cpp | 32 +++----- Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h | 3 +- Userland/Libraries/LibWeb/HTML/VideoTrack.cpp | 89 +++++++++------------- Userland/Libraries/LibWeb/HTML/VideoTrack.h | 19 ++--- 5 files changed, 87 insertions(+), 93 deletions(-) (limited to 'Userland') diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp index f155456d16..26dfe6740f 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp @@ -6,7 +6,7 @@ */ #include -#include +#include #include #include #include @@ -26,10 +26,31 @@ #include #include #include +#include #include namespace Web::HTML { +class MediaElementPlaybackTimer final : public Video::PlaybackTimer { +public: + static ErrorOr> create(int interval_ms, Function timeout_handler) + { + auto timer = Platform::Timer::create_single_shot(interval_ms, move(timeout_handler)); + return adopt_nonnull_own_or_enomem(new (nothrow) MediaElementPlaybackTimer(move(timer))); + } + + virtual void start() override { m_timer->start(); } + virtual void start(int interval_ms) override { m_timer->start(interval_ms); } + +private: + explicit MediaElementPlaybackTimer(NonnullRefPtr timer) + : m_timer(move(timer)) + { + } + + NonnullRefPtr m_timer; +}; + HTMLMediaElement::HTMLMediaElement(DOM::Document& document, DOM::QualifiedName qualified_name) : HTMLElement(document, move(qualified_name)) , m_pending_play_promises(heap()) @@ -579,10 +600,13 @@ WebIDL::ExceptionOr HTMLMediaElement::process_media_data(Function auto& realm = this->realm(); auto& vm = realm.vm(); + auto playback_manager = Video::PlaybackManager::from_data(m_media_data, [](auto interval_ms, auto timeout_handler) { + return MediaElementPlaybackTimer::create(interval_ms, move(timeout_handler)); + }); + // -> If the media data cannot be fetched at all, due to network errors, causing the user agent to give up trying to fetch the resource // -> If the media data can be fetched but is found by inspection to be in an unsupported format, or can otherwise not be rendered at all - auto demuxer = Video::Matroska::MatroskaDemuxer::from_data(m_media_data); - if (demuxer.is_error()) { + if (playback_manager.is_error()) { // 1. The user agent should cancel the fetching process. m_fetch_controller->terminate(); @@ -595,7 +619,7 @@ WebIDL::ExceptionOr HTMLMediaElement::process_media_data(Function JS::GCPtr video_track; // -> If the media resource is found to have an audio track - if (auto audio_tracks = demuxer.value()->get_tracks_for_type(Video::TrackType::Audio); !audio_tracks.is_error() && !audio_tracks.value().is_empty()) { + { // FIXME: 1. Create an AudioTrack object to represent the audio track. // FIXME: 2. Update the media element's audioTracks attribute's AudioTrackList object with the new AudioTrack object. // FIXME: 3. Let enable be unknown. @@ -609,9 +633,10 @@ WebIDL::ExceptionOr HTMLMediaElement::process_media_data(Function } // -> If the media resource is found to have a video track - if (auto video_tracks = demuxer.value()->get_tracks_for_type(Video::TrackType::Video); !video_tracks.is_error() && !video_tracks.value().is_empty()) { + // NOTE: Creating a Video::PlaybackManager above will have failed if there was not a video track. + { // 1. Create a VideoTrack object to represent the video track. - video_track = TRY(vm.heap().allocate(realm, realm, *this, demuxer.release_value(), video_tracks.value()[0])); + video_track = TRY(vm.heap().allocate(realm, realm, *this, playback_manager.release_value())); // 2. Update the media element's videoTracks attribute's VideoTrackList object with the new VideoTrack object. TRY_OR_THROW_OOM(vm, m_video_tracks->add_track({}, *video_track)); diff --git a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp index 6f50bb5b55..d0ac5449ec 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp +++ b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp @@ -11,13 +11,9 @@ #include #include #include -#include namespace Web::HTML { -// FIXME: Determine a reasonable framerate somehow. For now, this is roughly 24fps. -static constexpr int s_frame_delay_ms = 42; - HTMLVideoElement::HTMLVideoElement(DOM::Document& document, DOM::QualifiedName qualified_name) : HTMLMediaElement(document, move(qualified_name)) { @@ -81,32 +77,28 @@ void HTMLVideoElement::set_video_track(JS::GCPtr video_track) set_needs_style_update(true); document().set_needs_layout(); - if (m_video_timer) - m_video_timer->stop(); + if (m_video_track) + m_video_track->pause_video({}); m_video_track = video_track; } +void HTMLVideoElement::set_current_frame(Badge, RefPtr frame) +{ + m_current_frame = move(frame); + layout_node()->set_needs_display(); +} + void HTMLVideoElement::on_playing() { - if (!m_video_timer) { - m_video_timer = Platform::Timer::create_repeating(s_frame_delay_ms, [this]() { - if (auto frame = m_video_track->next_frame()) - m_current_frame = move(frame); - else - m_video_timer->stop(); - - layout_node()->set_needs_display(); - }); - } - - m_video_timer->start(); + if (m_video_track) + m_video_track->play_video({}); } void HTMLVideoElement::on_paused() { - if (m_video_timer) - m_video_timer->stop(); + if (m_video_track) + m_video_track->pause_video({}); } } diff --git a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h index 51012a2b84..2c1c82cb47 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h +++ b/Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h @@ -28,6 +28,8 @@ public: u32 video_height() const; void set_video_track(JS::GCPtr); + + void set_current_frame(Badge, RefPtr frame); RefPtr const& current_frame() const { return m_current_frame; } private: @@ -42,7 +44,6 @@ private: virtual void on_paused() override; JS::GCPtr m_video_track; - RefPtr m_video_timer; RefPtr m_current_frame; u32 m_video_width { 0 }; diff --git a/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp b/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp index 72dc4b5c72..c3e23dbbfc 100644 --- a/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp +++ b/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -21,12 +23,23 @@ namespace Web::HTML { static IDAllocator s_video_track_id_allocator; -VideoTrack::VideoTrack(JS::Realm& realm, JS::NonnullGCPtr media_element, NonnullOwnPtr demuxer, Video::Track track) +VideoTrack::VideoTrack(JS::Realm& realm, JS::NonnullGCPtr media_element, NonnullOwnPtr playback_manager) : PlatformObject(realm) , m_media_element(media_element) - , m_demuxer(move(demuxer)) - , m_track(track) + , m_playback_manager(move(playback_manager)) { + m_playback_manager->on_video_frame = [this](auto frame) { + if (is(*m_media_element)) + verify_cast(*m_media_element).set_current_frame({}, move(frame)); + }; + + m_playback_manager->on_decoder_error = [](auto) { + // FIXME: Propagate this error to HTMLMediaElement's error attribute. + }; + + m_playback_manager->on_fatal_playback_error = [](auto) { + // FIXME: Propagate this error to HTMLMediaElement's error attribute. + }; } VideoTrack::~VideoTrack() @@ -54,63 +67,29 @@ void VideoTrack::visit_edges(Cell::Visitor& visitor) visitor.visit(m_video_track_list); } -RefPtr VideoTrack::next_frame() +void VideoTrack::play_video(Badge) { - auto frame_sample = m_demuxer->get_next_video_sample_for_track(m_track); - if (frame_sample.is_error()) { - if (frame_sample.error().category() != Video::DecoderErrorCategory::EndOfStream) - dbgln("VideoTrack: Error getting next video sample: {}", frame_sample.error().description()); - return {}; - } - - OwnPtr decoded_frame; - - while (!decoded_frame) { - auto result = m_decoder.receive_sample(frame_sample.value()->data()); - if (result.is_error()) { - dbgln("VideoTrack: Error receiving video sample data: {}", result.error().description()); - return {}; - } - - while (true) { - auto frame_result = m_decoder.get_decoded_frame(); - if (frame_result.is_error()) { - if (frame_result.error().category() == Video::DecoderErrorCategory::NeedsMoreInput) - break; - - dbgln("VideoTrack: Error decoding video frame: {}", frame_result.error().description()); - return {}; - } + m_playback_manager->resume_playback(); +} - decoded_frame = frame_result.release_value(); - VERIFY(decoded_frame); - } - } +void VideoTrack::pause_video(Badge) +{ + m_playback_manager->pause_playback(); +} - auto& cicp = decoded_frame->cicp(); - cicp.adopt_specified_values(frame_sample.value()->container_cicp()); - cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::VideoFullRangeFlag::Studio }); - - // BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications - // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by - // handling those as sRGB instead, which causes no transfer function change in the output, - // unless display color management is later implemented. - switch (cicp.transfer_characteristics()) { - case Video::TransferCharacteristics::BT601: - case Video::TransferCharacteristics::BT709: - case Video::TransferCharacteristics::BT2020BitDepth10: - case Video::TransferCharacteristics::BT2020BitDepth12: - cicp.set_transfer_characteristics(Video::TransferCharacteristics::SRGB); - break; - default: - break; - } +Time VideoTrack::duration() const +{ + return m_playback_manager->selected_video_track().video_data().duration; +} - auto bitmap = decoded_frame->to_bitmap(); - if (bitmap.is_error()) - return {}; +u64 VideoTrack::pixel_width() const +{ + return m_playback_manager->selected_video_track().video_data().pixel_width; +} - return bitmap.release_value(); +u64 VideoTrack::pixel_height() const +{ + return m_playback_manager->selected_video_track().video_data().pixel_height; } // https://html.spec.whatwg.org/multipage/media.html#dom-videotrack-selected diff --git a/Userland/Libraries/LibWeb/HTML/VideoTrack.h b/Userland/Libraries/LibWeb/HTML/VideoTrack.h index 5009271705..77e37cecf6 100644 --- a/Userland/Libraries/LibWeb/HTML/VideoTrack.h +++ b/Userland/Libraries/LibWeb/HTML/VideoTrack.h @@ -9,9 +9,7 @@ #include #include #include -#include -#include -#include +#include #include namespace Web::HTML { @@ -24,11 +22,12 @@ public: void set_video_track_list(Badge, JS::GCPtr video_track_list) { m_video_track_list = video_track_list; } - RefPtr next_frame(); + void play_video(Badge); + void pause_video(Badge); - Time duration() const { return m_track.video_data().duration; } - u64 pixel_width() const { return m_track.video_data().pixel_width; } - u64 pixel_height() const { return m_track.video_data().pixel_height; } + Time duration() const; + u64 pixel_width() const; + u64 pixel_height() const; String const& id() const { return m_id; } String const& kind() const { return m_kind; } @@ -39,7 +38,7 @@ public: void set_selected(bool selected); private: - explicit VideoTrack(JS::Realm&, JS::NonnullGCPtr, NonnullOwnPtr, Video::Track); + VideoTrack(JS::Realm&, JS::NonnullGCPtr, NonnullOwnPtr); virtual JS::ThrowCompletionOr initialize(JS::Realm&) override; virtual void visit_edges(Cell::Visitor&) override; @@ -62,9 +61,7 @@ private: JS::NonnullGCPtr m_media_element; JS::GCPtr m_video_track_list; - NonnullOwnPtr m_demuxer; - Video::VP9::Decoder m_decoder; - Video::Track m_track; + NonnullOwnPtr m_playback_manager; }; } -- cgit v1.2.3