Cogs.Core
IVideoDecoder.h
1#pragma once
2
3#include "Resources/Resources.h"
4#include "Rendering/Common.h"
5#include "Rendering/TextureData.h"
6
7#include "Foundation/Platform/Threads.h"
8#include "Foundation/Reflection/Name.h"
9
10#include <map>
11#include <stdint.h>
12
13namespace Cogs {
14 class IGraphicsDevice;
15
16 namespace Core {
17 class Context;
18 struct RenderTexture;
19 struct Texture;
20
21 enum class TransferFunction : uint32_t {
22 Linear,
23 BT601,
24 sRGB,
25 Gamma
26 };
27
28 enum class ColourSpace : uint32_t {
29 BT601,
30 BT601Full,
31 BT709,
32 BT709Full,
33 };
34
36 {
37 class IVideoDecoder* decoder;
38 void* userData;
39 const struct VideoDecoderPayload& payload;
40 const unsigned char* spspps;
41 size_t spspps_size;
42 bool is_idr;
43 bool is_meta;
44 };
45 using VideoDecoderCallback = void(const VideoDecoderCallbackData&);
46
48 {
49 void *data;
50 };
51 using VideoPresentCallback = void(const VideoPresentCallbackData&);
52
53 enum struct Codec
54 {
55 None = -1,
56 H264,
57 HEVC
58 };
59
60 enum struct ChromaFormat : unsigned
61 {
62 Monochrome,
63 Chroma420,
64 Chroma422,
65 Chroma444
66 };
67
68 enum struct VideoDecoderPresentMode
69 {
70 Smooth = 1, // Keep a steady framerate
71 Realtime, // Reduce latency while keeping a steady framerate
72 Latest, // Always show the latest frame
73 };
74
76 {
77 const void *data = nullptr;
78 size_t size = 0;
79 int64_t timestamp = 0; // See VideoDecoderDescription::clock_rate for units
80 bool end_of_stream = false;
81 bool discontinuity = false;
82 bool end_of_picture = false;
83 bool use_timestamp = false;
84 };
85
87 {
88 uint64_t clock_rate = 10000000; // Timestamp units in Hz (0=default=10000000Hz)
89 uint32_t error_threshold = 100; // Error threshold (0-100) for presenting
90 // (100=always present even if fully corrupt)
91 uint32_t max_display_delay = 2; // Max display queue delay (improves pipelining of decode with display)
92 // 0=no delay (recommended values: 2..4)
93
94 Codec codec = Codec::H264;
95 std::string vps;
96 std::string sps;
97 std::string pps;
98
99 VideoDecoderPresentMode present_mode = VideoDecoderPresentMode::Smooth;
100 uint32_t speedup_queue_count = 3; // Speedup queue start count for Realtime playback (Increase when using B-frames)
101 float speedup_factor = 1.0f; // Speedup factor is additional playback speed for Realtime playback
102
103 bool generateMipMaps = true; // Generate mipmaps for each decoded frame.
104 bool skip_discontinuities = false;
105 bool recordToDisk = false;
106 std::string recordPath;
107 VideoDecoderCallback* decodeCallback = nullptr;
108 void* decodeCallbackData = nullptr;
109 VideoPresentCallback* presentCallback = nullptr;
110 void* presentCallbackData = nullptr;
111 };
112
114 {
115 int64_t timestamp = -1; // See VideoDecoderDescription::clock_rate for units
116 };
117
119 {
120 Codec codec = Codec::H264;
121 ChromaFormat chroma_format = ChromaFormat::Chroma420;
122 uint32_t bit_depth = 8;
123 };
124
125 class COGSCORE_DLL_API IVideoDecoder {
126 public:
127 struct Rect {
128 int left = -1;
129 int top = -1;
130 int right = -1;
131 int bottom = -1;
132 };
133
134 IVideoDecoder(Context& ctx) : context(ctx) {}
135 virtual ~IVideoDecoder() = default;
136
137 Codec getCodec() const { return codec; }
138 VideoDecoderPresentMode getPresentMode() const { return presentMode; }
139 double getDiscontinuityDuration() const { return discontinuityDuration; }
140 uint32_t getSpeedupQueueCount() const { return speedupQueueCount; }
141 float getSpeedupFactor() const { return speedupFactor; }
142 uint32_t getSurfaceCount() const { return surfaceCount; }
143 uint32_t getChromaFormatIdc() const { return chromaFormatIdc; }
144 uint32_t getChromaBitDepthMinus8() const { return chromaBitDepthMinus8; }
145 uint32_t getLumaBitDepthMinus8() const { return lumaBitDepthMinus8; }
146 uint32_t getCodedWidth() const { return codedWidth; }
147 uint32_t getCodedHeight() const { return codedHeight; }
148 const Rect& getDisplayArea() const { return displayArea; }
149 uint32_t getOutputWidth() const { return outputWidth; }
150 uint32_t getOutputHeight() const { return outputHeight; }
151 ColourSpace getColourSpace() const { return colourSpace; }
152 TransferFunction getTransferFunction() const { return transferFunction; }
153 float getGamma() const { return gamma; }
154 int64_t getClockRate() const { return clockRate; }
155 int64_t getPreviousClock() const { return previousClock; }
156 int64_t getClockResidual() const { return clockResidual; }
157 int64_t getPreviousClockResidual() const { return previousClockResidual; }
158 bool getCheckUnderflow() const { return checkUnderflow; }
159 int64_t getPresentationTime() const { return presentationTime; }
160 int64_t getPreviousPresentationTime() const { return previousPresentationTime; }
161 int64_t getPlayTime() const { return playTime; }
162 int64_t getPreviousPlayTime() const { return previousPlayTime; }
163 uint32_t getBufferUnderflowCounter() const { return bufferUnderflowCounter; }
164 uint32_t getBufferUnderflowMajorStutterCounter() const { return bufferUnderflowMajorStutterCounter; }
165 uint32_t getBufferOverflowCounter() const { return bufferOverflowCounter; }
166 uint32_t getDecoderRestartCounter() const { return decoderRestartCounter; }
167 uint32_t getDiscontinuitiesSkippedCounter() const { return discontinuitiesSkippedCounter; }
168 const TextureDescription& getTextureDescription() const { return textureDesc; }
169 Cogs::TextureHandle getTexture() const { return texture; }
170 RenderTexture* getRenderTexture() const { return renderTexture; }
171
172 bool& refSkipDiscontinuities() { return skipDiscontinuities; }
173 uint32_t& refSpeedupQueueCount() { return speedupQueueCount; }
174 float& refSpeedupFactor() { return speedupFactor; }
175
176 bool shouldSkipDiscontinuities() const { return skipDiscontinuities; }
177
178 void setPresentMode(VideoDecoderPresentMode mode) { presentMode = mode; }
179
180 protected:
181 Context& context;
182 Codec codec = Codec::None;
183 VideoDecoderPresentMode presentMode = VideoDecoderPresentMode::Latest;
184 bool skipDiscontinuities = true;
185 bool generateMipMaps = false;
186 double discontinuityDuration = 0.5;
187 uint32_t surfaceCount = 0;
188 std::vector<uint8_t> parameterSets;
189
190 // Speedup parameters used in Realtime present mode...
191 uint32_t speedupQueueCount = 3;
192 float speedupFactor = 1.0f;
193
194 VideoDecoderCallback* decodeCallback = nullptr;
195 void* decodeCallbackData = nullptr;
196 VideoPresentCallback* presentCallback = nullptr;
197 void* presentCallbackData = nullptr;
198
199 // Video parameters, as decoded from the video parameter sets...
200 uint32_t chromaFormatIdc = static_cast<uint32_t>(-1);
201 uint32_t chromaBitDepthMinus8 = static_cast<uint32_t>(-1);
202 uint32_t lumaBitDepthMinus8 = static_cast<uint32_t>(-1);
203 uint32_t codedWidth = static_cast<uint32_t>(-1);
204 uint32_t codedHeight = static_cast<uint32_t>(-1);
205 Rect displayArea;
206 uint32_t outputWidth = 0;
207 uint32_t outputHeight = 0;
208 ColourSpace colourSpace = ColourSpace::BT709Full;
209 TransferFunction transferFunction = TransferFunction::sRGB;
210 float gamma = 1.0f / 2.2f;
211
212 int64_t clockRate = 0;
213 int64_t previousClock = 0;
214 int64_t clockResidual = 0;
215 int64_t previousClockResidual = 0;
216 bool checkUnderflow = false;
217
218 int64_t presentationTime = 0;
219 int64_t previousPresentationTime = 0;
220
221 int64_t playTime = 0;
222 int64_t previousPlayTime = 0;
223
224 // Debug counters
225 uint32_t bufferUnderflowCounter = 0;
226 uint32_t bufferUnderflowMajorStutterCounter = 0;
227 uint32_t bufferOverflowCounter = 0;
228 uint32_t decoderRestartCounter = 0;
229 uint32_t discontinuitiesSkippedCounter = 0;
230
231 // Output texture
232 TextureDescription textureDesc;
234 RenderTexture* renderTexture = nullptr;
236
237 virtual void cleanup();
238 virtual int64_t getPresentationTime(size_t queuePosition) = 0;
239
240 void parseVideoDescription(const VideoDecoderDescription& desc);
241 void createTexture();
242 int selectDisplayFrame(size_t queueLength);
243 };
244
246 {
247 virtual ~IVideoDecoderContext() = default;
248 virtual void update() = 0;
249
250 virtual bool capsSupported(VideoDecoderCaps &caps) = 0;
251 virtual void loadVideoTexture(Cogs::Core::TextureHandle textureHandle, const VideoDecoderDescription &desc) = 0;
252 virtual void reloadVideoTexture(Cogs::Core::TextureHandle textureHandle, const VideoDecoderDescription &desc) = 0;
253 virtual TextureDescription getVideoDescription(Cogs::Core::TextureHandle textureHandle) = 0;
254 virtual VideoDecoderStatus getVideoStatus(Cogs::Core::TextureHandle textureHandle) = 0;
255 virtual void streamVideoData(Cogs::Core::TextureHandle textureHandle, const VideoDecoderPayload &packet) = 0;
256 virtual void endVideoStream(Cogs::Core::TextureHandle textureHandle) = 0;
257
258 protected:
259 // Pending data is incoming video stream data that can't be processed because the decoder isn't ready.
260 // IVideoDecoderContext implementations are free to discard incoming data that can't be processed, or
261 // package it in a PendingData object and queue it for decoding at at later time.
262
263 struct PendingData {
264 std::vector<uint8_t> data;
265 VideoDecoderPayload payload;
266
267 PendingData(const VideoDecoderPayload& p) : data(static_cast<const uint8_t*>(p.data), static_cast<const uint8_t*>(p.data) + p.size), payload(p) {
268 payload.data = data.data();
269 }
270 };
271
272 using pendingDataMap = std::map<Texture*, std::vector<PendingData>>;
273
274 pendingDataMap pendingData;
275 Mutex pendingDataMutex;
276 };
277 }
278}
279
280template<> inline Cogs::StringView getName<Cogs::Core::Codec>() { return "Codec"; }
A Context instance contains all the services, systems and runtime components needed to use Cogs.
Definition: Context.h:83
std::vector< uint8_t > parameterSets
Cached VPS, SPS, and PPS data.
Provides a weakly referenced view over the contents of a string.
Definition: StringView.h:24
@ sRGB
Value is a color and is subject to gamma correction.
Contains all Cogs related functionality.
Definition: FieldSetter.h:23
static const Handle_t NoHandle
Represents a handle to nothing.
Definition: Common.h:77