IVideoDecoder.h

#pragma once
 
#include "Resources/Resources.h"
#include "Rendering/Common.h"
#include "Rendering/TextureData.h"
 
#include "Foundation/Platform/Threads.h"
#include "Foundation/Reflection/Name.h"
 
#include <map>
#include <stdint.h>
 
namespace Cogs {
  class IGraphicsDevice;
 
  namespace Core {
    class Context;
    struct RenderTexture;
    struct Texture;
 
    enum class TransferFunction : uint32_t {
      Linear,
      BT601,
      sRGB,
      Gamma
    };
 
    enum class ColourSpace : uint32_t {
      BT601,
      BT601Full,
      BT709,
      BT709Full,
    };
 
    struct VideoDecoderCallbackData
    {
      class IVideoDecoder* decoder;
      void* userData;
      const struct VideoDecoderPayload& payload;
      const unsigned char* spspps;
      size_t spspps_size;
      bool is_idr;
      bool is_meta;
    };
    using VideoDecoderCallback = void(const VideoDecoderCallbackData&);
 
    struct VideoPresentCallbackData
    {
      void *data;
      int64_t timestamp;
    };
 
    using VideoPresentCallback = void(const VideoPresentCallbackData&);
 
    enum struct Codec
    {
      None = -1,
      H264,
      HEVC
    };
 
    enum struct ChromaFormat : unsigned
    {
      Monochrome,
      Chroma420,
      Chroma422,
      Chroma444
    };
 
    enum struct VideoDecoderPresentMode
    {
      Smooth = 1, // Keep a steady framerate
      Realtime,   // Reduce latency while keeping a steady framerate
      Latest,     // Always show the latest frame
    };
 
    struct VideoDecoderPayload
    {
      const void *data = nullptr;
      size_t size = 0;
      int64_t timestamp = 0; // See VideoDecoderDescription::clock_rate for units
      bool end_of_stream = false;
      bool discontinuity = false;
      bool end_of_picture = false;
      bool use_timestamp = false;
    };
 
    struct VideoDecoderDescription
    {
      uint64_t clock_rate = 10000000; // Timestamp units in Hz (0=default=10000000Hz)
      uint32_t error_threshold = 100; // Error threshold (0-100) for presenting
                                      // (100=always present even if fully corrupt)
      uint32_t max_display_delay = 2; // Max display queue delay (improves pipelining of decode with display)
                                      // 0=no delay (recommended values: 2..4)
 
      Codec codec = Codec::H264;
      std::string vps;
      std::string sps;
      std::string pps;
 
      VideoDecoderPresentMode present_mode = VideoDecoderPresentMode::Smooth;
      uint32_t speedup_queue_count = 3; // Speedup queue start count for Realtime playback (Increase when using B-frames)
      float speedup_factor = 1.0f; // Speedup factor is additional playback speed for Realtime playback
 
      bool generateMipMaps = true;         // Generate mipmaps for each decoded frame.
      bool skip_discontinuities = false;
      bool recordToDisk = false;
      std::string recordPath;
      VideoDecoderCallback* decodeCallback = nullptr;
      void* decodeCallbackData = nullptr;
      VideoPresentCallback* presentCallback = nullptr;
      void* presentCallbackData = nullptr;
    };
 
    struct VideoDecoderStatus
    {
      int64_t timestamp = -1;  // See VideoDecoderDescription::clock_rate for units
    };
 
    struct VideoDecoderCaps
    {
      Codec codec = Codec::H264;
      ChromaFormat chroma_format = ChromaFormat::Chroma420;
      uint32_t bit_depth = 8;
    };
 
    class COGSCORE_DLL_API IVideoDecoder {
    public:
              struct Rect {
                int left = -1;
                int top = -1;
                int right = -1;
                int bottom = -1;
              };
 
                                        IVideoDecoder(Context& ctx) : context(ctx) {}
      virtual                           ~IVideoDecoder() = default;
 
              Codec                     getCodec() const                              { return codec; }
              VideoDecoderPresentMode   getPresentMode() const                        { return presentMode; }
              double                    getDiscontinuityDuration() const              { return discontinuityDuration; }
              uint32_t                  getSpeedupQueueCount() const                  { return speedupQueueCount; }
              float                     getSpeedupFactor() const                      { return speedupFactor; }
              uint32_t                  getSurfaceCount() const                       { return surfaceCount; }
              uint32_t                  getChromaFormatIdc() const                    { return chromaFormatIdc; }
              uint32_t                  getChromaBitDepthMinus8() const               { return chromaBitDepthMinus8; }
              uint32_t                  getLumaBitDepthMinus8() const                 { return lumaBitDepthMinus8; }
              uint32_t                  getCodedWidth() const                         { return codedWidth; }
              uint32_t                  getCodedHeight() const                        { return codedHeight; }
              const Rect&               getDisplayArea() const                        { return displayArea; }
              uint32_t                  getOutputWidth() const                        { return outputWidth; }
              uint32_t                  getOutputHeight() const                       { return outputHeight; }
              ColourSpace               getColourSpace() const                        { return colourSpace; }
              TransferFunction          getTransferFunction() const                   { return transferFunction; }
              float                     getGamma() const                              { return gamma; }
              int64_t                   getClockRate() const                          { return clockRate; }
              int64_t                   getPreviousClock() const                      { return previousClock; }
              int64_t                   getClockResidual() const                      { return clockResidual; }
              int64_t                   getPreviousClockResidual() const              { return previousClockResidual; }
              bool                      getCheckUnderflow() const                     { return checkUnderflow; }
              int64_t                   getPresentationTime() const                   { return presentationTime; }
              int64_t                   getPreviousPresentationTime() const           { return previousPresentationTime; }
              int64_t                   getPlayTime() const                           { return playTime; }
              int64_t                   getPreviousPlayTime() const                   { return previousPlayTime; }
              uint32_t                  getBufferUnderflowCounter() const             { return bufferUnderflowCounter; }
              uint32_t                  getBufferUnderflowMajorStutterCounter() const { return bufferUnderflowMajorStutterCounter; }
              uint32_t                  getBufferOverflowCounter() const              { return bufferOverflowCounter; }
              uint32_t                  getDecoderRestartCounter() const              { return decoderRestartCounter; }
              uint32_t                  getDiscontinuitiesSkippedCounter() const      { return discontinuitiesSkippedCounter; }
              const TextureDescription& getTextureDescription() const                 { return textureDesc; }
              Cogs::TextureHandle       getTexture() const                            { return texture; }
              RenderTexture*            getRenderTexture() const                      { return renderTexture; }
 
              bool&                     refSkipDiscontinuities()                      { return skipDiscontinuities; }
              uint32_t&                 refSpeedupQueueCount()                        { return speedupQueueCount; }
              float&                    refSpeedupFactor()                            { return speedupFactor; }
 
              bool                      shouldSkipDiscontinuities() const             { return skipDiscontinuities; }
 
              void                      setPresentMode(VideoDecoderPresentMode mode)  { presentMode = mode; }
 
    protected:
              Context&                  context;
              Codec                     codec = Codec::None;
              VideoDecoderPresentMode   presentMode = VideoDecoderPresentMode::Latest;    
              bool                      skipDiscontinuities = true;                       
              bool                      generateMipMaps = false;                          
              double                    discontinuityDuration = 0.5;                      
              uint32_t                  surfaceCount = 0;
              std::vector<uint8_t>      parameterSets;                                    
 
              // Speedup parameters used in Realtime present mode...
              uint32_t                  speedupQueueCount = 3;
              float                     speedupFactor = 1.0f;
 
              VideoDecoderCallback*     decodeCallback = nullptr;                         
              void*                     decodeCallbackData = nullptr;
              VideoPresentCallback*     presentCallback = nullptr;                        
              void*                     presentCallbackData = nullptr;
 
              // Video parameters, as decoded from the video parameter sets...
              uint32_t                  chromaFormatIdc = static_cast<uint32_t>(-1);
              uint32_t                  chromaBitDepthMinus8 = static_cast<uint32_t>(-1);
              uint32_t                  lumaBitDepthMinus8 = static_cast<uint32_t>(-1);
              uint32_t                  codedWidth = static_cast<uint32_t>(-1);
              uint32_t                  codedHeight = static_cast<uint32_t>(-1);
              Rect                      displayArea;
              uint32_t                  outputWidth = 0;
              uint32_t                  outputHeight = 0;
              ColourSpace               colourSpace = ColourSpace::BT709Full;
              TransferFunction          transferFunction = TransferFunction::sRGB;
              float                     gamma = 1.0f / 2.2f;                              
 
              int64_t                   clockRate = 0;
              int64_t                   previousClock = 0;                                
              int64_t                   clockResidual = 0;                                
              int64_t                   previousClockResidual = 0;
              bool                      checkUnderflow = false;
 
              int64_t                   presentationTime = 0;                             
              int64_t                   previousPresentationTime = 0;                     
 
              int64_t                   playTime = 0;                                     
              int64_t                   previousPlayTime = 0;                             
 
              // Debug counters
              uint32_t                  bufferUnderflowCounter = 0;
              uint32_t                  bufferUnderflowMajorStutterCounter = 0;
              uint32_t                  bufferOverflowCounter = 0;
              uint32_t                  decoderRestartCounter = 0;
              uint32_t                  discontinuitiesSkippedCounter = 0;
 
              // Output texture
              TextureDescription        textureDesc;
              Cogs::TextureHandle       texture = Cogs::TextureHandle::NoHandle;
              RenderTexture*            renderTexture = nullptr;
              RenderTargetHandle        renderTarget = Cogs::RenderTargetHandle::NoHandle;
 
      virtual void                      cleanup();
      virtual int64_t                   getPresentationTime(size_t queuePosition) = 0;
 
              void                      parseVideoDescription(const VideoDecoderDescription& desc);
              void                      createTexture();
              int                       selectDisplayFrame(size_t queueLength);
    };
 
    struct IVideoDecoderContext
    {
      virtual ~IVideoDecoderContext() = default;
      virtual void update() = 0;
 
      virtual bool capsSupported(VideoDecoderCaps &caps) = 0;
      virtual void loadVideoTexture(Cogs::Core::TextureHandle textureHandle, const VideoDecoderDescription &desc) = 0;
      virtual void reloadVideoTexture(Cogs::Core::TextureHandle textureHandle, const VideoDecoderDescription &desc) = 0;
      virtual TextureDescription getVideoDescription(Cogs::Core::TextureHandle textureHandle) = 0;
      virtual VideoDecoderStatus getVideoStatus(Cogs::Core::TextureHandle textureHandle) = 0;
      virtual void streamVideoData(Cogs::Core::TextureHandle textureHandle, const VideoDecoderPayload &packet) = 0;
      virtual void endVideoStream(Cogs::Core::TextureHandle textureHandle) = 0;
 
    protected:
      // Pending data is incoming video stream data that can't be processed because the decoder isn't ready.
      // IVideoDecoderContext implementations are free to discard incoming data that can't be processed, or
      // package it in a PendingData object and queue it for decoding at at later time.
 
      struct PendingData {
        std::vector<uint8_t>  data;
        VideoDecoderPayload   payload;
 
        PendingData(const VideoDecoderPayload& p) : data(static_cast<const uint8_t*>(p.data), static_cast<const uint8_t*>(p.data) + p.size), payload(p) {
          payload.data = data.data();
        }
      };
 
      using pendingDataMap  = std::map<Texture*, std::vector<PendingData>>;
 
      pendingDataMap        pendingData;
      Mutex                 pendingDataMutex;
    };
  }
}
 
template<> inline Cogs::StringView getName<Cogs::Core::Codec>() { return "Codec"; }