RayPick.h

#pragma once
 
#include "Base.h"
#include "PickingFlags.h"
#include "Components/Core/RenderComponent.h"  // RenderLayers
 
#include "Rendering/DataFormat.h"
 
 
#include <glm/glm.hpp>
#include <glm/gtx/quaternion.hpp>
 
#include <cmath>
#include <limits>
#include <vector>
#include <span>
 
namespace Cogs::Core
{
 
  class CameraComponent;
  class Context;
  class IRayPickable;
  struct Mesh;
  struct ClipShapeData;
  struct RayIntersectionHit;
 
 
  class RayPicking
  {
  public:
    static constexpr glm::vec2 NoPickTextureCoords = { std::numeric_limits<float>::quiet_NaN(), std::numeric_limits<float>::quiet_NaN() };
    static constexpr size_t NoPickEntityType = static_cast<size_t>(-1);
 
    struct COGSCORE_DLL_API Ordinal
    {
      uint32_t value = std::numeric_limits<uint32_t>::max();
    };
 
    struct COGSCORE_DLL_API RayPickFilter
    {
      RayPickFilter() = default;
      RayPickFilter(RenderLayers layerMask, size_t entityTypeHash) : layerMask(layerMask), entityTypeHash(entityTypeHash) {}
      RayPickFilter(std::span<const RenderLayers> ordering, RenderLayers layerMask, size_t entityTypeHash);
 
      Ordinal getOrdinal(const RenderComponent& comp) const
      {
        for (size_t i = 0; i < std::size(ordering); i++) {
          if ((comp.layer & ordering[i]) != RenderLayers::None) {
            return Ordinal { static_cast<uint32_t>(i) };
          }
        }
        return Ordinal { std::numeric_limits<uint32_t>::max() };
      }
 
      RenderLayers ordering[4] = { RenderLayers::Overlay,  RenderLayers::All };
 
      RenderLayers layerMask = RenderLayers::All;
      size_t entityTypeHash = RayPicking::NoPickEntityType;
      [[nodiscard]] bool isUnwantedType(const ComponentModel::Component& comp) const;
    };
 
    struct COGSCORE_DLL_API RayPickHit {
      EntityId entity = NoEntity;
 
      EntityId root = NoEntity;
 
      glm::vec3 position;
 
      glm::vec2 textureCoords = NoPickTextureCoords;
 
      float distance = std::numeric_limits<float>::max();
 
      Ordinal ordinal = { std::numeric_limits<uint32_t>::max() };
 
      RayPickHit() = default;
      RayPickHit(const RenderComponent& comp, bool returnChildEntity, const glm::vec3& pos, Ordinal ordinal, float dist, const glm::vec2& texCoords = NoPickTextureCoords);
 
      // Strictly in front, i.e. not !isBehind
      bool isInfront(Ordinal otherOrdinal, float otherDistance) const
      {
        // Lexicographical ordering of [order, distance]
        if (ordinal.value == otherOrdinal.value) return distance < otherDistance;
        return ordinal.value < otherOrdinal.value;
      }
 
      // Strictly behind, i.e. not !isInfront
      bool isBehind(Ordinal otherOrdinal, float otherDistance) const
      {
        // Lexicographical ordering of [order, distance]
        if (otherOrdinal.value == ordinal.value) return otherDistance < distance;
        return otherOrdinal.value < ordinal.value;
      }
 
      // Needed to std::sort results without custom compare
      bool operator<(const RayPickHit& rhs) const
      {
        // Lexicographical ordering of [order, distance]
        if (ordinal.value == rhs.ordinal.value) return distance < rhs.distance;
        return ordinal.value < rhs.ordinal.value;
      }
    };
 
    COGSCORE_DLL_API bool pickCamera(Context* context,
                                     const CameraComponent& camera,
                                     const glm::vec2& cameraPosition,
                                     float rayLength,
                                     float radius,
                                     PickingFlags pickingFlags,
                                     PicksReturned returnFlag,
                                     const RayPicking::RayPickFilter& filter,
                                     std::vector<RayPicking::RayPickHit>& hits);
 
    COGSCORE_DLL_API bool pickRay(Context* context,
                                  const glm::vec3& position,
                                  const glm::quat& rotation,
                                  float rayLength,
                                  float radius,
                                  PickingFlags pickingFlags,
                                  PicksReturned returnFlag,
                                  const RayPicking::RayPickFilter& filter,
                                  std::vector<RayPicking::RayPickHit>& hits);
 
    COGSCORE_DLL_API void addPickable(IRayPickable * pickable);
 
    COGSCORE_DLL_API void removePickable(IRayPickable * pickable);
 
  private:
    std::vector<IRayPickable *> pickables;
  };
 
  class IRayPickable
  {
  public:
    enum struct Order : int32_t {
      Default = 0,
      Last = std::numeric_limits<int32_t>::max()
    };
    Order order = Order::Default;
 
    IRayPickable(Order order = Order::Default) : order(order) {}
 
    virtual ~IRayPickable() = default;
 
    COGSCORE_DLL_API virtual bool pickCamera(Context* context,
                                             const CameraComponent& camera,
                                             const glm::vec2& queryClip,            
                                             float rayLength,                       
                                             float rayRadius,                       
                                             PickingFlags pickingFlags,
                                             PicksReturned returnFlag,
                                             const RayPicking::RayPickFilter& filter,
                                             std::vector<RayPicking::RayPickHit>& hits) = 0;
    COGSCORE_DLL_API virtual bool pickRay(Context* /*context*/,
                                          const glm::vec3& /*startPos*/,
                                          const glm::quat& /*rot*/,
                                          float /*rayLength*/,
                                          float /*radius*/,
                                          PickingFlags /*pickingFlags*/,
                                          PicksReturned /*returnFlag*/,
                                          const RayPicking::RayPickFilter& /*filter*/,
                                          std::vector<RayPicking::RayPickHit>& /*hits*/) { return{}; }
  };
 
  class MeshRayPickableBase : public IRayPickable
  {
  public:
    COGSCORE_DLL_API bool pickCamera(Context* context,
                                     const CameraComponent& camera,
                                     const glm::vec2& normPosition,
                                     float /*rayLength*/,
                                     float radius,
                                     PickingFlags pickingFlags,
                                     PicksReturned returnFlag,
                                     const RayPicking::RayPickFilter& filter,
                                     std::vector<RayPicking::RayPickHit>& hits) override;
 
    COGSCORE_DLL_API bool pickRay(Context* context,
                                  const glm::vec3& startPos,
                                  const glm::quat& rot,
                                  float rayLength,
                                  float radius,
                                  PickingFlags pickingFlags,
                                  PicksReturned returnFlag,
                                  const RayPicking::RayPickFilter& filter,
                                  std::vector<RayPicking::RayPickHit>& hits) override;
 
  protected:
    COGSCORE_DLL_API virtual bool pickImpl(Context* context,
                                           const glm::mat4& worldPickMatrix,
                                           const glm::mat4& rawViewProjection,
                                           const glm::mat4& viewMatrix,
                                           const RayPicking::RayPickFilter& filter,
                                           PickingFlags pickingFlags,
                                           PicksReturned returnFlag,
                                           std::vector<RayPicking::RayPickHit>& hits) = 0;
 
 
    // Helpers for picking meshes
    struct TextureCoordinateInfo {
      const uint8_t* ptr = nullptr;
      Cogs::DataFormat format = Cogs::DataFormat::Unknown;
      uint32_t idOffset = 0;
      uint32_t stride = 0;
      uint32_t count = 0;
      bool hasIdOffset = false;
    };
    [[nodiscard]] COGSCORE_DLL_API static TextureCoordinateInfo getTextureCoordinateInfo(const Cogs::Core::Mesh& mesh);
    [[nodiscard]] COGSCORE_DLL_API static bool clippedByClipComp(const ClipShapeData& clipData, const glm::vec4& position);
    [[nodiscard]] COGSCORE_DLL_API static glm::vec2 getTextureCoords(const TextureCoordinateInfo& textureInfo, const RayIntersectionHit& hit, const PickingFlags pickingFlags);
  };
}