CogsBin.h

#pragma once
#include <glm/vec3.hpp>
#include <glm/mat4x4.hpp>
#include <glm/ext/quaternion_float.hpp>
#include <cstdint>
#include "Flags.h"
 
namespace Cogs::Core
{
  namespace CogsBin
  {
    // COGSBIN file format
    // ====================
    //
    // File layout:
    //   - Header. Contains number of sections in file.
    //   - N instances of SectionInfo, where N is the number of sections in file (see header).
    //   - Section 0
    //   - ...
    //   - Section N-1
    //
    // Sections:
    //   Each section is described by its SectionInfo in the beginning of the file. The info
    //   contains the offset inside the file where the section is located, along with its size.
    //   Sections can be compressed or raw on a section-by-section basis. The info tells if the
    //   section is compressed, and what the uncompressed file size is.
    //
    //   The idea is that if the cogsbin file is used as an archive, it should be simple to only
    //   pay the decompression cost for what is needed, and thus the format allows the free-form
    //   data of models to be placed in different sections.
    //
    // Subsections:
    //   Data that corresponds to given structs are stored in sub-sections. Items in the
    //   subsections contains reference into other subsections *within the same section*.
    //
    //   Currently, the writer only puts subsection into section 0, but the file format doesn't
    //   require this. The reasoning is that the data in subsections is quite small compared to
    //   the free-form data, and compression is more efficient if similar data (e.g. strings)
    //   are grouped. Also, the writer does de-duplication, so shared materials etc. are shared
    //   inside the file as well.
    //
    //   The 'models' subsection contain the entry point when extracting data, and contains the
    //   list of models contained in the file. Usually there is only one model in the file.
    //
    // Free-form data:
    //   Some data, like vertex data, indices, and animation value arrays are stored outside
    //   the subsections.
    //
    //   **Free-form data offsets:** Data is referenced by an uin64_t, where the upper 16 bits is
    //   the subsection number and the lower 48 contains the offset within the subsection.
    //
 
    const uint32_t HeaderMagic = 0xC065C065;
    const uint32_t Version = 5;
    const size_t CogsSectionAlignment = 16;
 
    struct Header
    {
      uint32_t  magic;
      uint32_t  version;
      uint64_t  fileLength;
      uint32_t  sectionCount;
      uint32_t  unused[3];
    };
    static_assert(sizeof(Header) == 4 * 8);
 
    enum struct Compression : uint32_t
    {
      None = 0,
      ZSTD = 1
    };
 
    enum struct SectionType : uint16_t
    {
      META = 1<<0,
      DATA = 1<<1
    };
    ENABLE_ENUM_FLAGS(SectionType);
 
    struct SectionInfo
    {
      uint64_t    fileOffset;       // Offset in file
      uint64_t    fileSize;         // Number of bytes in file (i.e. compressed size)
      uint64_t    uncompressedSize; // Number of bytes when uncompressed.
      Compression compression;      // Compression method.
      SectionType type;
      uint16_t    subSectionCount;
    };
    static_assert(sizeof(SectionInfo) == 4 * 8);
 
    enum struct SubSectionType: uint32_t
    {
      Buffers,
      ExternalTextures,
      EmbeddedTextures,
      SamplerStates,
      Strings,
      StringData,
      Nodes,
      Transforms,
      BoundingBoxes,
      MaterialInstances,
      Properties,
      PropertyData,
      Meshes,
      VertexStreams,
      VertexAttributes,
      Bones,
      Skeletons,
      AnimTracks,
      AnimClips,
      Models,
      EnumSize
    };
 
    struct SubSectionInfo
    {
      uint64_t        offset;
      uint32_t        size;
      SubSectionType  type;
    };
//    static_assert(sizeof(SubSection)==
 
    // enum values matches DXGI_FORMAT
    enum struct Format : uint16_t {
      Unknown,
      R8_UNORM,
      R8G8_UNORM,
      R8G8B8_UNORM,
      R8G8B8A8_UNORM,
      R16_UNORM,
      R16G16_UNORM,
      R16G16B16_UNORM,
      R16G16B16A16_UNORM,
      R8_SNORM,
      R8G8_SNORM,
      R8G8B8_SNORM,
      R8G8B8A8_SNORM,
      R16_SNORM,
      R16G16_SNORM,
      R16G16B16_SNORM,
      R16G16B16A16_SNORM,
      R8_UINT,
      R8G8_UINT,
      R8G8B8_UINT,
      R8G8B8A8_UINT,
      R16_UINT,
      R16G16_UINT,
      R16G16B16_UINT,
      R16G16B16A16_UINT,
      R32_UINT,
      R32G32_UINT,
      R32G32B32_UINT,
      R32G32B32A32_UINT,
      R8_SINT,
      R8G8_SINT,
      R8G8B8_SINT,
      R8G8B8A8_SINT,
      R16_SINT,
      R16G16_SINT,
      R16G16B16_SINT,
      R16G16B16A16_SINT,
      R32_SINT,
      R32G32_SINT,
      R32G32B32_SINT,
      R32G32B32A32_SINT,
      R16_FLOAT,
      R16G16_FLOAT,
      R16G16B16_FLOAT,
      R16G16B16A16_FLOAT,
      R32_FLOAT,
      R32G32_FLOAT,
      R32G32B32_FLOAT,
      R32G32B32A32_FLOAT,
      D16_UNORM,
      D24_UNORM,
      D24S8_UNORM,
      D32_FLOAT,
      R32_TYPELESS,
      R16_TYPELESS,
      R8T,
      R8G8T,
      R8G8B8T,
      R8G8B8A8T,
      B8G8R8,
      B8G8R8A8,
      A8_UNORM,
      BC1_TYPELESS,
      BC1_UNORM,
      BC1_UNORM_SRGB,
      BC2_TYPELESS,
      BC2_UNORM,
      BC2_UNORM_SRGB,
      BC3_TYPELESS,
      BC3_UNORM,
      BC3_UNORM_SRGB,
      BC4_TYPELESS,
      BC4_UNORM,
      BC4_SNORM,
      BC5_TYPELESS,
      BC5_UNORM,
      BC5_SNORM,
      R8G8B8A8_UNORM_SRGB,
      R64_FLOAT,
      R64G64_FLOAT,
      R64G64B64_FLOAT,
      R64G65B64A64_FLOAT,
      EnumSize
    };
 
    enum struct DataFormatV4 : uint16_t
    {
      Unknown = 0,
      R8_UNORM,
      R8G8_UNORM,
      R8G8B8_UNORM,
      R8G8B8A8_UNORM,
      R16_UNORM,
      R16G16_UNORM,
      R16G16B16_UNORM,
      R16G16B16A16_UNORM,
      R8_SNORM,
      R8G8_SNORM,
      R8G8B8_SNORM,
      R8G8B8A8_SNORM,
      R16_SNORM,
      R16G16_SNORM,
      R16G16B16_SNORM,
      R16G16B16A16_SNORM,
      R8_UINT,
      R8G8_UINT,
      R8G8B8_UINT,
      R8G8B8A8_UINT,
      R16_UINT,
      R16G16_UINT,
      R16G16B16_UINT,
      R16G16B16A16_UINT,
      R32_UINT,
      R32G32_UINT,
      R32G32B32_UINT,
      R32G32B32A32_UINT,
      R8_SINT,
      R8G8_SINT,
      R8G8B8_SINT,
      R8G8B8A8_SINT,
      R16_SINT,
      R16G16_SINT,
      R16G16B16_SINT,
      R16G16B16A16_SINT,
      R32_SINT,
      R32G32_SINT,
      R32G32B32_SINT,
      R32G32B32A32_SINT,
      R16_FLOAT,
      R16G16_FLOAT,
      R16G16B16_FLOAT,
      R16G16B16A16_FLOAT,
      R32_FLOAT,
      R32G32_FLOAT,
      R32G32B32_FLOAT,
      R32G32B32A32_FLOAT,
      D16_UNORM,
      D24_UNORM,
      D24S8_UNORM,
      D32_FLOAT,
      R32_TYPELESS,
      R16_TYPELESS,
      R8T,
      R8G8T,
      R8G8B8T,
      R8G8B8A8T,
      B8G8R8,
      B8G8R8A8,
      A8_UNORM,
      BC1_TYPELESS,
      BC1_UNORM,
      BC1_UNORM_SRGB,
      BC2_TYPELESS,
      BC2_UNORM,
      BC2_UNORM_SRGB,
      BC3_TYPELESS,
      BC3_UNORM,
      BC3_UNORM_SRGB,
      BC4_TYPELESS,
      BC4_UNORM,
      BC4_SNORM,
      BC5_TYPELESS,
      BC5_UNORM,
      BC5_SNORM,
      R8G8B8_UNORM_SRGB,
      R8G8B8A8_UNORM_SRGB,
      R10G10B10A2_TYPELESS,
      R10G10B10A2_UNORM,
      R10G10B10A2_UINT,
      R11G11B10_FLOAT,
      R5G6B5_UNORM,
      R5G5B5A1_UNORM,
      R4G4B4A4_UNORM,
      R9G9B9E5_FLOAT,
      MAT4X4_FLOAT,
      Format_Size,
    };
    static_assert(static_cast<int>(DataFormatV4::Format_Size) == 88);
 
    enum struct DataFormatV5 : uint16_t
    {
      Unknown = 0,
      R8_UNORM,
      R8G8_UNORM,
      R8G8B8_UNORM,
      R8G8B8A8_UNORM,
      R16_UNORM,
      R16G16_UNORM,
      R16G16B16_UNORM,
      R16G16B16A16_UNORM,
      R8_SNORM,
      R8G8_SNORM,
      R8G8B8_SNORM,
      R8G8B8A8_SNORM,
      R16_SNORM,
      R16G16_SNORM,
      R16G16B16_SNORM,
      R16G16B16A16_SNORM,
      R8_UINT,
      R8G8_UINT,
      R8G8B8_UINT,
      R8G8B8A8_UINT,
      R16_UINT,
      R16G16_UINT,
      R16G16B16_UINT,
      R16G16B16A16_UINT,
      R32_UINT,
      R32G32_UINT,
      R32G32B32_UINT,
      R32G32B32A32_UINT,
      R8_SINT,
      R8G8_SINT,
      R8G8B8_SINT,
      R8G8B8A8_SINT,
      R16_SINT,
      R16G16_SINT,
      R16G16B16_SINT,
      R16G16B16A16_SINT,
      R32_SINT,
      R32G32_SINT,
      R32G32B32_SINT,
      R32G32B32A32_SINT,
      R16_FLOAT,
      R16G16_FLOAT,
      R16G16B16_FLOAT,
      R16G16B16A16_FLOAT,
      R32_FLOAT,
      R32G32_FLOAT,
      R32G32B32_FLOAT,
      R32G32B32A32_FLOAT,
      D16_UNORM,
      D24_UNORM,
      D24S8_UNORM,
      D32_FLOAT,
      R32_TYPELESS,
      R16_TYPELESS,
      R8T,
      R8G8T,
      R8G8B8T,
      R8G8B8A8T,
      B8G8R8,
      B8G8R8A8,
      A8_UNORM,
      BC1_TYPELESS,
      BC1_UNORM,
      BC1_UNORM_SRGB,
      BC2_TYPELESS,
      BC2_UNORM,
      BC2_UNORM_SRGB,
      BC3_TYPELESS,
      BC3_UNORM,
      BC3_UNORM_SRGB,
      BC4_TYPELESS,
      BC4_UNORM,
      BC4_SNORM,
      BC5_TYPELESS,
      BC5_UNORM,
      BC5_SNORM,
      BC6H_SIGNED_FLOAT_RGB,
      BC6H_UNSIGNED_FLOAT_RGB,
      BC7_UNORM_RGBA,
      BC7_UNORM_SRGBA,
      PVRTC1_2BPP_UNORM_RGB,
      PVRTC1_4BPP_UNORM_RGB,
      PVRTC1_2BPP_UNORM_RGBA,
      PVRTC1_4BPP_UNORM_RGBA,
      EAC_BLOCK8_UNSIGNED_FLOAT_R,
      EAC_BLOCK8_SIGNED_FLOAT_R,
      EAC_BLOCK16_UNSIGNED_FLOAT_RG,
      EAC_BLOCK16_SIGNED_FLOAT_RG,
      ETC2_BLOCK8_UNORM_RGB,
      ETC2_BLOCK8_UNORM_SRGB,
      ETC2_BLOCK16_UNORM_RGBA,
      ETC2_BLOCK16_UNORM_SRGBA,
      ASTC_4x4_UNORM_RGBA,
      ASTC_5x4_UNORM_RGBA,
      ASTC_5x5_UNORM_RGBA,
      ASTC_6x5_UNORM_RGBA,
      ASTC_6x6_UNORM_RGBA,
      ASTC_8x5_UNORM_RGBA,
      ASTC_8x6_UNORM_RGBA,
      ASTC_8x8_UNORM_RGBA,
      ASTC_10x5_UNORM_RGBA,
      ASTC_10x6_UNORM_RGBA,
      ASTC_10x8_UNORM_RGBA,
      ASTC_10x10_UNORM_RGBA,
      ASTC_12x10_UNORM_RGBA,
      ASTC_12x12_UNORM_RGBA,
      ASTC_4x4_UNORM_SRGBA,
      ASTC_5x4_UNORM_SRGBA,
      ASTC_5x5_UNORM_SRGBA,
      ASTC_6x5_UNORM_SRGBA,
      ASTC_6x6_UNORM_SRGBA,
      ASTC_8x5_UNORM_SRGBA,
      ASTC_8x6_UNORM_SRGBA,
      ASTC_8x8_UNORM_SRGBA,
      ASTC_10x5_UNORM_SRGBA,
      ASTC_10x6_UNORM_SRGBA,
      ASTC_10x8_UNORM_SRGBA,
      ASTC_10x10_UNORM_SRGBA,
      ASTC_12x10_UNORM_SRGBA,
      ASTC_12x12_UNORM_SRGBA,
      R8G8B8_UNORM_SRGB,
      R8G8B8A8_UNORM_SRGB,
      R10G10B10A2_TYPELESS,
      R10G10B10A2_UNORM,
      R10G10B10A2_UINT,
      R11G11B10_FLOAT,
      R5G6B5_UNORM,
      R5G5B5A1_UNORM,
      R4G4B4A4_UNORM,
      R9G9B9E5_FLOAT,
      MAT4X4_FLOAT,
      Format_Size,
    };
    static_assert(static_cast<int>(DataFormatV5::Format_Size) == 132);
 
    enum struct MeshFlags : uint8_t
    {
      None      =    0,
      Clockwise = 1<<0,
      Skinned   = 1<<1
    };
    ENABLE_ENUM_FLAGS(MeshFlags);
 
    enum struct PrimitiveType : uint8_t
    {
      Unknown                 = 255,
      TriangleList            = 0,
      TriangleStrip           = 1,
      LineList                = 2,
      LineStrip               = 3,
      PointList               = 4,
      TriangleListAdjacency   = 5,
      TriangleStripAdjacency  = 6,
      LineListAdjacency       = 7,
      LineStripAdjacency      = 8,
      ControlPoint1PatchList  = 9,
      ControlPoint2PatchList  = 10,
      ControlPoint3PatchList  = 11,
      ControlPoint4PatchList  = 12,
      EnumSize
    };
 
    enum struct Semantic : uint8_t
    {
      Position        = 0,
      Normal          = 1,
      Color           = 2,
      TexCoord        = 3,
      Tangent         = 4,
      InstanceVector  = 5,
      InstanceMatrix  = 6,
      EnumSize
    };
 
    enum struct InputType : uint8_t
    {
      PerVertex,
      PerInstance,
      EnumSize
    };
 
    // Reference to string in string buffer, non-zero terminated.
    struct String
    {
      uint32_t offset;
      uint32_t length;
    };
 
    struct BoundingBox
    {
      float min[3];
      float max[3];
    };
 
    enum struct BufferContentFlags : uint32_t
    {
      None          = 0,
      VertexData    = 1<<0,             
      IndexData     = 1<<1,             
      TextureData   = 1<<2,             
      AnimationData = 1<<3,
      ProcessedData = 1<<4              
    };
    ENABLE_ENUM_FLAGS(BufferContentFlags);
 
    struct Buffer
    {
      uint64_t            sectionOffset;
      uint32_t            size;         
      BufferContentFlags  flags;        
    };
    static_assert(sizeof(Buffer) == 2 * 8);
 
    struct VertexAttribute
    {
      uint16_t    offset;           // uint16_t Offset from element start
      Format      format;           // uint16_t
      Semantic    semantic;         // uint8_t
      uint8_t     semanticIndex;    // uint8_t
      uint16_t    instanceStepRate; // 0 - vertex input, !0 -> instance stepping = instanceStepRate-1
    };
    static_assert(sizeof(VertexAttribute) == 2 * 4);
 
    struct VertexStream
    {
      uint32_t  buffer;
      uint32_t  offset;
      uint32_t  count;
      uint32_t  firstAttribute;
      uint32_t  attributeCount;
      uint16_t  stride;
      uint8_t   vertexDataType;
      uint8_t   encoding;
    };
    static_assert(sizeof(VertexStream) == 3 * 8);
 
    struct Mesh
    {
      uint32_t        name;               
      uint32_t        boundingBox;        
      uint32_t        firstStream;        
      uint8_t         streamCount;
      MeshFlags       flags;
      PrimitiveType   primitiveType;
    };
    static_assert(sizeof(Mesh) == 4 * 4);
 
    // Contains one LOD-level of texture data.
    // - data size deducable from width,height/depth/layers/format
    struct Level
    {
      uint32_t  bufferView;     
      uint32_t  offset;
      uint32_t  rowStride;
      uint32_t  depthStride;
    };
    static_assert(sizeof(Level) == 4 * 4);
 
    enum struct TextureAddressMode : uint8_t
    {
      Clamp,    
      Wrap,     
      Mirror,   
      Border,   
      EnumSize,
    };
 
    enum struct TextureFilterMode : uint8_t
    {
      MinMagMipPoint,             
      MinMagMipLinear,            
      ComparisonMinMagMipPoint,   
      ComparisonMinMagMipLinear,  
      EnumSize
    };
 
    enum struct TextureComparisonFunction : uint8_t
    {
      Never,
      Less,
      Equal,
      LessEqual,
      Greater,
      NotEqual,
      GreaterEqual,
      Always,
      EnumSize
    };
 
    struct SamplerState
    {
      TextureAddressMode addressModeS;        
      TextureAddressMode addressModeT;        
      TextureAddressMode addressModeW;        
 
      TextureFilterMode filter;                      
      TextureComparisonFunction comparisonFunction;  
 
      uint32_t maxAnisotropy;   
      float borderColor[4];     
    };
    static_assert(sizeof(SamplerState) == 7 * 4);
 
    // Later, add support for PNG prediction filters so we compress as well as PNG.
    enum struct TextureEncoding : uint8_t
    {
      None,
      EnumSize
    };
 
    enum struct TextureFlags : uint8_t
    {
      None,
      EnumSize
    };
 
    enum struct TextureDimensions : uint8_t
    {
      Unknown,
      Buffer,
      Texture1D,
      Texture1DArray,
      Texture2D,
      Texture2DArray,
      Texture2DMS,
      Texture2DMSArray,
      Texture3D,
      Texture3DArray,
      TextureCube,
      TextureCubeArray,
      EnumSize
    };
 
    struct EmbeddedTexture
    {
      uint32_t          buffer;             
      uint32_t          offset;             
      uint32_t          size;               
      uint32_t          name;
      uint32_t          width;
      uint32_t          height;
      uint32_t          depth;
      uint32_t          faces;
      uint32_t          layers;
      uint32_t          samplerState;
      Format            format;
      TextureEncoding   encoding;
      TextureFlags      flags;
      TextureDimensions dimension;
    };
    static_assert(sizeof(EmbeddedTexture) == 12 * 4);
 
    struct ExternalTexture
    {
      uint32_t  source;
      uint32_t  samplerState;
    };
    static_assert(sizeof(ExternalTexture) == 2 * 4);
 
    enum struct PropertyKeyType : uint16_t
    {
      String,
      Index,
      EnumSize
    };
 
    enum struct PropertyValueType : uint16_t
    {
      Bool,
      UInt,
      Float,
      Float2,
      Float3,
      Float4,
      Variant,
      Option,
      ExternalTexture,
      EmbeddedTexture,
      EnumSize
    };
 
    struct Property
    {
      PropertyKeyType   keyType;
      PropertyValueType valueType;
      uint32_t          key;
      uint32_t          value;
    };
    static_assert(alignof(Property) == 4);
 
    struct MaterialInstance
    {
      uint32_t  material;               
      uint32_t  permutation;            
      uint32_t  propertyFirst;          
      uint32_t  propertyCount;          
    };
 
    struct Transform
    {
      float m[12];
 
      size_t hash(size_t hashValue = Cogs::hash()) const {
        for (const float& f : m) {
          hashValue = Cogs::hash(f, hashValue);
        }
        return hashValue;
      }
    };
    static_assert(sizeof(Transform) == 6 * 8);
 
 
    enum struct BoneFlags : uint16_t
    {
      None      =    0,
      HasOffset = 1<<0,
      Used      = 1<<1,
      Animated  = 1<<2
    };
    ENABLE_ENUM_FLAGS(BoneFlags);
 
    struct Bone
    {
      glm::mat4 inverseBindPose;
      glm::mat4 relative;
      glm::quat rot;
      glm::vec3 pos;
      glm::vec3 scale;
      uint32_t name;                    
      uint16_t parentBone;              
      BoneFlags flags;
    };
    static_assert(sizeof(Bone) == 44 * 4);
 
    struct Skeleton
    {
      glm::mat4 bindPose;
      uint32_t firstBone;               
      uint32_t boneCount;               
    };
 
    struct AnimTrack
    {
      uint32_t boneIndex;               
      uint32_t buffer;                  
      uint32_t translationOffset;
      uint32_t translationCount;
      uint32_t rotationOffset;
      uint32_t rotationCount;
      uint32_t scaleOffset;
      uint32_t scaleCount;
    };
    static_assert(sizeof(AnimTrack) == 8 * 4);
 
    struct AnimClip
    {
      uint32_t  name;                    
      float     duration;
      float     resolution;
      uint32_t  trackFirst;
      uint32_t  trackCount;
    };
 
    struct Node
    {
      uint32_t      parent;             
      uint32_t      name;               
      uint32_t      mesh;               
      uint32_t      materialInstance;   
      uint32_t      boundingBox;        
      uint32_t      transform;          
      PrimitiveType primitiveType;      
      uint32_t      vertexFirst;        
      uint32_t      vertexCount;        
    };
 
    struct Model
    {
      uint32_t  skeleton;               
      uint32_t  animClipFirst;
      uint32_t  animClipCount;
      uint32_t  firstNode;              
      uint32_t  nodeCount;              
    };
 
  }
}