summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--AK/Debug.h.in4
-rw-r--r--Meta/CMake/all_the_debug_macros.cmake1
-rw-r--r--Userland/Libraries/LibVideo/CMakeLists.txt1
-rw-r--r--Userland/Libraries/LibVideo/PlaybackManager.cpp244
-rw-r--r--Userland/Libraries/LibVideo/PlaybackManager.h180
5 files changed, 430 insertions, 0 deletions
diff --git a/AK/Debug.h.in b/AK/Debug.h.in
index c07eba6544..b5f3d95933 100644
--- a/AK/Debug.h.in
+++ b/AK/Debug.h.in
@@ -342,6 +342,10 @@
#cmakedefine01 PDF_DEBUG
#endif
+#ifndef PLAYBACK_MANAGER_DEBUG
+#cmakedefine01 PLAYBACK_MANAGER_DEBUG
+#endif
+
#ifndef PNG_DEBUG
#cmakedefine01 PNG_DEBUG
#endif
diff --git a/Meta/CMake/all_the_debug_macros.cmake b/Meta/CMake/all_the_debug_macros.cmake
index a36d64fcce..577f40ca46 100644
--- a/Meta/CMake/all_the_debug_macros.cmake
+++ b/Meta/CMake/all_the_debug_macros.cmake
@@ -141,6 +141,7 @@ set(PATA_DEBUG ON)
set(PATH_DEBUG ON)
set(PCI_DEBUG ON)
set(PDF_DEBUG ON)
+set(PLAYBACK_MANAGER_DEBUG ON)
set(PNG_DEBUG ON)
set(POLL_SELECT_DEBUG ON)
set(PORTABLE_IMAGE_LOADER_DEBUG ON)
diff --git a/Userland/Libraries/LibVideo/CMakeLists.txt b/Userland/Libraries/LibVideo/CMakeLists.txt
index 5f4500e0b1..ab1a5192ef 100644
--- a/Userland/Libraries/LibVideo/CMakeLists.txt
+++ b/Userland/Libraries/LibVideo/CMakeLists.txt
@@ -4,6 +4,7 @@ set(SOURCES
Color/TransferCharacteristics.cpp
MatroskaDemuxer.cpp
MatroskaReader.cpp
+ PlaybackManager.cpp
VideoFrame.cpp
VP9/BitStream.cpp
VP9/Decoder.cpp
diff --git a/Userland/Libraries/LibVideo/PlaybackManager.cpp b/Userland/Libraries/LibVideo/PlaybackManager.cpp
new file mode 100644
index 0000000000..2999fe3df5
--- /dev/null
+++ b/Userland/Libraries/LibVideo/PlaybackManager.cpp
@@ -0,0 +1,244 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Format.h>
+#include <LibCore/Timer.h>
+#include <LibVideo/MatroskaReader.h>
+#include <LibVideo/VP9/Decoder.h>
+
+#include "MatroskaDemuxer.h"
+#include "PlaybackManager.h"
+
+namespace Video {
+
+// We post DecoderErrors to the event queue to be handled, since some will occur off the main thread.
+#define TRY_OR_POST_ERROR_AND_RETURN(expression, return_value) \
+ ({ \
+ auto _temporary_result = ((expression)); \
+ if (_temporary_result.is_error()) { \
+ dbgln("Playback error encountered: {}", _temporary_result.error().string_literal()); \
+ m_main_loop.post_event(*this, make<DecoderErrorEvent>(_temporary_result.release_error())); \
+ return return_value; \
+ } \
+ _temporary_result.release_value(); \
+ })
+
+#define TRY_OR_POST_ERROR(expression) TRY_OR_POST_ERROR_AND_RETURN(expression, )
+
+DecoderErrorOr<NonnullRefPtr<PlaybackManager>> PlaybackManager::from_file(Object* event_handler, StringView filename)
+{
+ NonnullOwnPtr<Demuxer> demuxer = TRY(MatroskaDemuxer::from_file(filename));
+ auto video_tracks = demuxer->get_tracks_for_type(TrackType::Video);
+ if (video_tracks.is_empty())
+ return DecoderError::with_description(DecoderErrorCategory::Invalid, "No video track is present"sv);
+ auto track = video_tracks[0];
+
+ dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier());
+
+ NonnullOwnPtr<VideoDecoder> decoder = make<VP9::Decoder>();
+ return PlaybackManager::construct(event_handler, demuxer, track, decoder);
+}
+
+PlaybackManager::PlaybackManager(Object* event_handler, NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>& decoder)
+ : Object(event_handler)
+ , m_main_loop(Core::EventLoop::current())
+ , m_demuxer(move(demuxer))
+ , m_selected_video_track(video_track)
+ , m_decoder(move(decoder))
+ , m_frame_queue(make<VideoFrameQueue>())
+ , m_present_timer(Core::Timer::construct())
+ , m_decode_timer(Core::Timer::construct())
+{
+ m_present_timer->set_single_shot(true);
+ m_present_timer->set_interval(0);
+ m_present_timer->on_timeout = [&] { update_presented_frame(); };
+
+ m_decode_timer->set_single_shot(true);
+ m_decode_timer->set_interval(0);
+ m_decode_timer->on_timeout = [&] { on_decode_timer(); };
+}
+
+void PlaybackManager::set_playback_status(PlaybackStatus status)
+{
+ if (status != m_status) {
+ auto old_status = m_status;
+ m_status = status;
+ dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set playback status from {} to {}", playback_status_to_string(old_status), playback_status_to_string(m_status));
+
+ if (status == PlaybackStatus::Playing) {
+ if (old_status == PlaybackStatus::Stopped) {
+ restart_playback();
+ m_frame_queue->clear();
+ m_skipped_frames = 0;
+ }
+ m_last_present_in_real_time = Time::now_monotonic();
+ m_present_timer->start();
+ } else {
+ m_last_present_in_media_time = current_playback_time();
+ m_last_present_in_real_time = Time::zero();
+ m_present_timer->stop();
+ }
+
+ m_main_loop.post_event(*this, make<PlaybackStatusChangeEvent>(status, old_status));
+ }
+}
+
+void PlaybackManager::event(Core::Event& event)
+{
+ if (event.type() == DecoderErrorOccurred) {
+ auto& error_event = static_cast<DecoderErrorEvent&>(event);
+ VERIFY(error_event.error().category() != DecoderErrorCategory::EndOfStream);
+ }
+
+ // Allow events to bubble up in all cases.
+ event.ignore();
+}
+
+void PlaybackManager::resume_playback()
+{
+ set_playback_status(PlaybackStatus::Playing);
+}
+
+void PlaybackManager::pause_playback()
+{
+ set_playback_status(PlaybackStatus::Paused);
+}
+
+bool PlaybackManager::prepare_next_frame()
+{
+ if (m_next_frame.has_value())
+ return true;
+ if (m_frame_queue->is_empty())
+ return false;
+ auto frame_item = m_frame_queue->dequeue();
+ m_next_frame.emplace(frame_item);
+ m_decode_timer->start();
+ return true;
+}
+
+Time PlaybackManager::current_playback_time()
+{
+ if (is_playing())
+ return m_last_present_in_media_time + (Time::now_monotonic() - m_last_present_in_real_time);
+ return m_last_present_in_media_time;
+}
+
+Time PlaybackManager::duration()
+{
+ return m_demuxer->duration();
+}
+
+void PlaybackManager::update_presented_frame()
+{
+ bool out_of_queued_frames = false;
+ Optional<FrameQueueItem> frame_item_to_display;
+
+ while (true) {
+ out_of_queued_frames = out_of_queued_frames || !prepare_next_frame();
+ if (out_of_queued_frames)
+ break;
+ VERIFY(m_next_frame.has_value());
+ if (m_next_frame->timestamp > current_playback_time() || m_next_frame->is_eos_marker())
+ break;
+
+ if (frame_item_to_display.has_value()) {
+ dbgln_if(PLAYBACK_MANAGER_DEBUG, "At {}ms: Dropped frame with timestamp {}ms for the next at {}ms", current_playback_time().to_milliseconds(), frame_item_to_display->timestamp.to_milliseconds(), m_next_frame->timestamp.to_milliseconds());
+ m_skipped_frames++;
+ }
+ frame_item_to_display = m_next_frame.release_value();
+ }
+
+ if (!out_of_queued_frames && frame_item_to_display.has_value()) {
+ m_main_loop.post_event(*this, make<VideoFramePresentEvent>(frame_item_to_display->bitmap));
+ m_last_present_in_media_time = current_playback_time();
+ m_last_present_in_real_time = Time::now_monotonic();
+ frame_item_to_display.clear();
+ }
+
+ if (frame_item_to_display.has_value()) {
+ VERIFY(!m_next_frame.has_value());
+ m_next_frame = frame_item_to_display;
+ dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set next frame back to dequeued item at timestamp {}ms", m_next_frame->timestamp.to_milliseconds());
+ }
+
+ if (!is_playing())
+ return;
+
+ if (!out_of_queued_frames) {
+ if (m_next_frame->is_eos_marker()) {
+ set_playback_status(PlaybackStatus::Stopped);
+ m_next_frame.clear();
+ return;
+ }
+
+ auto frame_time_ms = (m_next_frame.value().timestamp - current_playback_time()).to_milliseconds();
+ VERIFY(frame_time_ms <= NumericLimits<int>::max());
+ dbgln_if(PLAYBACK_MANAGER_DEBUG, "Time until next frame is {}ms", frame_time_ms);
+ m_present_timer->start(max(static_cast<int>(frame_time_ms), 0));
+ return;
+ }
+
+ set_playback_status(PlaybackStatus::Buffering);
+ m_decode_timer->start();
+}
+
+void PlaybackManager::restart_playback()
+{
+ m_last_present_in_media_time = Time::zero();
+ m_last_present_in_real_time = Time::zero();
+ TRY_OR_POST_ERROR(m_demuxer->seek_to_most_recent_keyframe(m_selected_video_track, 0));
+}
+
+bool PlaybackManager::decode_and_queue_one_sample()
+{
+ if (m_frame_queue->size() >= FRAME_BUFFER_COUNT)
+ return false;
+#if PLAYBACK_MANAGER_DEBUG
+ auto start_time = Time::now_monotonic();
+#endif
+
+ auto frame_sample_result = m_demuxer->get_next_video_sample_for_track(m_selected_video_track);
+ if (frame_sample_result.is_error()) {
+ if (frame_sample_result.error().category() == DecoderErrorCategory::EndOfStream) {
+ m_frame_queue->enqueue(FrameQueueItem::eos_marker());
+ return false;
+ }
+ m_main_loop.post_event(*this, make<DecoderErrorEvent>(frame_sample_result.release_error()));
+ return false;
+ }
+ auto frame_sample = frame_sample_result.release_value();
+
+ TRY_OR_POST_ERROR_AND_RETURN(m_decoder->receive_sample(frame_sample->data()), false);
+ auto decoded_frame = TRY_OR_POST_ERROR_AND_RETURN(m_decoder->get_decoded_frame(), false);
+
+ auto& cicp = decoded_frame->cicp();
+ cicp.adopt_specified_values(frame_sample->container_cicp());
+ cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::ColorRange::Studio });
+
+ auto bitmap = TRY_OR_POST_ERROR_AND_RETURN(decoded_frame->to_bitmap(), false);
+ m_frame_queue->enqueue(FrameQueueItem { bitmap, frame_sample->timestamp() });
+
+#if PLAYBACK_MANAGER_DEBUG
+ auto end_time = Time::now_monotonic();
+ dbgln("Decoding took {}ms", (end_time - start_time).to_milliseconds());
+#endif
+
+ return true;
+}
+
+void PlaybackManager::on_decode_timer()
+{
+ if (!decode_and_queue_one_sample() && is_buffering()) {
+ set_playback_status(PlaybackStatus::Playing);
+ return;
+ }
+
+ // Continually decode until buffering is complete
+ if (is_buffering())
+ m_decode_timer->start();
+}
+
+}
diff --git a/Userland/Libraries/LibVideo/PlaybackManager.h b/Userland/Libraries/LibVideo/PlaybackManager.h
new file mode 100644
index 0000000000..77a91fc17a
--- /dev/null
+++ b/Userland/Libraries/LibVideo/PlaybackManager.h
@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Atomic.h>
+#include <AK/Function.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/Queue.h>
+#include <AK/Time.h>
+#include <LibCore/EventLoop.h>
+#include <LibCore/SharedCircularQueue.h>
+#include <LibGfx/Bitmap.h>
+#include <LibThreading/ConditionVariable.h>
+#include <LibThreading/Mutex.h>
+#include <LibThreading/Thread.h>
+
+#include "Demuxer.h"
+#include "MatroskaDocument.h"
+#include "VideoDecoder.h"
+
+namespace Video {
+
+enum class PlaybackStatus {
+ Playing,
+ Paused,
+ Buffering,
+ Seeking,
+ Stopped,
+};
+
+struct FrameQueueItem {
+ static FrameQueueItem eos_marker()
+ {
+ return { nullptr, Time::max() };
+ }
+
+ RefPtr<Gfx::Bitmap> bitmap;
+ Time timestamp;
+
+ bool is_eos_marker() const { return !bitmap; }
+};
+
+static constexpr size_t FRAME_BUFFER_COUNT = 4;
+using VideoFrameQueue = Queue<FrameQueueItem, FRAME_BUFFER_COUNT>;
+
+class PlaybackManager : public Core::Object {
+ C_OBJECT(PlaybackManager)
+
+public:
+ static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> from_file(Object* event_handler, StringView file);
+ static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> from_data(Object* event_handler, Span<u8> data);
+
+ PlaybackManager(Object* event_handler, NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>& decoder);
+ ~PlaybackManager() override = default;
+
+ void resume_playback();
+ void pause_playback();
+ void restart_playback();
+ bool is_playing() const { return m_status == PlaybackStatus::Playing; }
+ bool is_buffering() const { return m_status == PlaybackStatus::Buffering; }
+ bool is_stopped() const { return m_status == PlaybackStatus::Stopped; }
+
+ u64 number_of_skipped_frames() const { return m_skipped_frames; }
+
+ void event(Core::Event& event) override;
+
+ Time current_playback_time();
+ Time duration();
+
+ Function<void(NonnullRefPtr<Gfx::Bitmap>, Time)> on_frame_present;
+
+private:
+ void set_playback_status(PlaybackStatus status);
+
+ bool prepare_next_frame();
+ void update_presented_frame();
+
+ // Runs off the main thread
+ bool decode_and_queue_one_sample();
+ void on_decode_timer();
+
+ Core::EventLoop& m_main_loop;
+
+ PlaybackStatus m_status { PlaybackStatus::Stopped };
+ Time m_last_present_in_media_time = Time::zero();
+ Time m_last_present_in_real_time = Time::zero();
+
+ NonnullOwnPtr<Demuxer> m_demuxer;
+ Track m_selected_video_track;
+ NonnullOwnPtr<VideoDecoder> m_decoder;
+
+ NonnullOwnPtr<VideoFrameQueue> m_frame_queue;
+ Optional<FrameQueueItem> m_next_frame;
+
+ NonnullRefPtr<Core::Timer> m_present_timer;
+ unsigned m_decoding_buffer_time_ms = 16;
+
+ NonnullRefPtr<Core::Timer> m_decode_timer;
+
+ u64 m_skipped_frames;
+};
+
+enum EventType : unsigned {
+ DecoderErrorOccurred = (('v' << 2) | ('i' << 1) | 'd') << 4,
+ VideoFramePresent,
+ PlaybackStatusChange,
+};
+
+class DecoderErrorEvent : public Core::Event {
+public:
+ explicit DecoderErrorEvent(DecoderError error)
+ : Core::Event(DecoderErrorOccurred)
+ , m_error(move(error))
+ {
+ }
+ virtual ~DecoderErrorEvent() = default;
+
+ DecoderError error() { return m_error; }
+
+private:
+ DecoderError m_error;
+};
+
+class VideoFramePresentEvent : public Core::Event {
+public:
+ VideoFramePresentEvent() = default;
+ explicit VideoFramePresentEvent(RefPtr<Gfx::Bitmap> frame)
+ : Core::Event(VideoFramePresent)
+ , m_frame(move(frame))
+ {
+ }
+ virtual ~VideoFramePresentEvent() = default;
+
+ RefPtr<Gfx::Bitmap> frame() { return m_frame; }
+
+private:
+ RefPtr<Gfx::Bitmap> m_frame;
+};
+
+class PlaybackStatusChangeEvent : public Core::Event {
+public:
+ PlaybackStatusChangeEvent() = default;
+ explicit PlaybackStatusChangeEvent(PlaybackStatus status, PlaybackStatus previous_status)
+ : Core::Event(PlaybackStatusChange)
+ , m_status(status)
+ , m_previous_status(previous_status)
+ {
+ }
+ virtual ~PlaybackStatusChangeEvent() = default;
+
+ PlaybackStatus status();
+ PlaybackStatus previous_status();
+
+private:
+ PlaybackStatus m_status;
+ PlaybackStatus m_previous_status;
+};
+
+inline StringView playback_status_to_string(PlaybackStatus status)
+{
+ switch (status) {
+ case PlaybackStatus::Playing:
+ return "Playing"sv;
+ case PlaybackStatus::Paused:
+ return "Paused"sv;
+ case PlaybackStatus::Buffering:
+ return "Buffering"sv;
+ case PlaybackStatus::Seeking:
+ return "Seeking"sv;
+ case PlaybackStatus::Stopped:
+ return "Stopped"sv;
+ }
+ return "Unknown"sv;
+};
+
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-17 11:06:51 +0000