RayPick.h

#pragma once
 
#include "Base.h"
#include "PickingFlags.h"
#include "Components/Core/RenderComponent.h"  // RenderLayers
 
#include "Rendering/DataFormat.h"
 
 
#include <glm/glm.hpp>
#include <glm/gtx/quaternion.hpp>
 
#include <cmath>
#include <limits>
#include <vector>
 
namespace Cogs::Core
{
 
  class CameraComponent;
  class Context;
  class IRayPickable;
  struct Mesh;
  struct ClipShapeData;
  struct RayIntersectionHit;
 
 
  class RayPicking
  {
  public:
    static constexpr glm::vec2 NoPickTextureCoords = { std::numeric_limits<float>::quiet_NaN(), std::numeric_limits<float>::quiet_NaN() };
    static constexpr size_t NoPickEntityType = static_cast<size_t>(-1);
 
    struct COGSCORE_DLL_API RayPickHit {
      EntityId entity = NoEntity;
 
      EntityId root = NoEntity;
 
      glm::vec3 position;
 
      glm::vec2 textureCoords = NoPickTextureCoords;
 
      float distance = std::numeric_limits<float>::max();
 
      RayPickHit() = default;
      RayPickHit(const ComponentModel::Component& comp, bool returnChildEntity, const glm::vec3& pos, float dist, const glm::vec2& texCoords = NoPickTextureCoords);
 
      // Needed to std::sort results without custom compare
      bool operator<(const RayPickHit& rhs) const
      {
        return distance < rhs.distance;
      }
    };
 
    struct COGSCORE_DLL_API RayPickFilter
    {
      RenderLayers layerMask = RenderLayers::All;
      size_t entityTypeHash = RayPicking::NoPickEntityType;
      [[nodiscard]] bool isUnwantedType(const ComponentModel::Component& comp) const;
    };
 
    COGSCORE_DLL_API bool pickCamera(Context* context,
                                     const CameraComponent& camera,
                                     const glm::vec2& cameraPosition,
                                     float rayLength,
                                     float radius,
                                     PickingFlags pickingFlags,
                                     PicksReturned returnFlag,
                                     const RayPicking::RayPickFilter& filter,
                                     std::vector<RayPicking::RayPickHit>& hits);
 
    COGSCORE_DLL_API bool pickRay(Context* context,
                                  const glm::vec3& position,
                                  const glm::quat& rotation,
                                  float rayLength,
                                  float radius,
                                  PickingFlags pickingFlags,
                                  PicksReturned returnFlag,
                                  const RayPicking::RayPickFilter& filter,
                                  std::vector<RayPicking::RayPickHit>& hits);
 
    COGSCORE_DLL_API void addPickable(IRayPickable * pickable);
 
    COGSCORE_DLL_API void removePickable(IRayPickable * pickable);
 
  private:
    std::vector<IRayPickable *> pickables;
  };
 
  class IRayPickable
  {
  public:
    enum struct Order : int32_t {
      Default = 0,
      Last = std::numeric_limits<int32_t>::max()
    };
    Order order = Order::Default;
 
    IRayPickable(Order order = Order::Default) : order(order) {}
 
    virtual ~IRayPickable() = default;
 
    COGSCORE_DLL_API virtual bool pickCamera(Context* context,
                                             const CameraComponent& camera,
                                             const glm::vec2& queryClip,            
                                             float rayLength,                       
                                             float rayRadius,                       
                                             PickingFlags pickingFlags,
                                             PicksReturned returnFlag,
                                             const RayPicking::RayPickFilter& filter,
                                             std::vector<RayPicking::RayPickHit>& hits) = 0;
    COGSCORE_DLL_API virtual bool pickRay(Context* /*context*/,
                                          const glm::vec3& /*startPos*/,
                                          const glm::quat& /*rot*/,
                                          float /*rayLength*/,
                                          float /*radius*/,
                                          PickingFlags /*pickingFlags*/,
                                          PicksReturned /*returnFlag*/,
                                          const RayPicking::RayPickFilter& /*filter*/,
                                          std::vector<RayPicking::RayPickHit>& /*hits*/) { return{}; }
  };
 
  class MeshRayPickableBase : public IRayPickable
  {
  public:
    bool pickCamera(Context* context,
                    const CameraComponent& camera,
                    const glm::vec2& normPosition,
                    float /*rayLength*/,
                    float radius,
                    PickingFlags pickingFlags,
                    PicksReturned returnFlag,
                    const RayPicking::RayPickFilter& filter,
                    std::vector<RayPicking::RayPickHit>& hits) override;
 
    bool pickRay(Context* context,
                 const glm::vec3& startPos,
                 const glm::quat& rot,
                 float rayLength,
                 float radius,
                 PickingFlags pickingFlags,
                 PicksReturned returnFlag,
                 const RayPicking::RayPickFilter& filter,
                 std::vector<RayPicking::RayPickHit>& hits) override;
 
  protected:
    virtual bool pickImpl(Context* context,
                          const glm::mat4& worldPickMatrix,
                          const glm::mat4& rawViewProjection,
                          const glm::mat4& viewMatrix,
                          const RayPicking::RayPickFilter& filter,
                          PickingFlags pickingFlags,
                          PicksReturned returnFlag,
                          std::vector<RayPicking::RayPickHit>& hits) = 0;
 
 
    // Helpers for picking meshes
    struct TextureCoordinateInfo {
      const uint8_t* ptr = nullptr;
      Cogs::DataFormat format = Cogs::DataFormat::Unknown;
      uint32_t idOffset = 0;
      uint32_t stride = 0;
      uint32_t count = 0;
      bool hasIdOffset = false;
    };
    [[nodiscard]] static TextureCoordinateInfo getTextureCoordinateInfo(const Cogs::Core::Mesh& mesh);
    [[nodiscard]] static bool clippedByClipComp(const ClipShapeData& clipData, const glm::vec4& position);
    [[nodiscard]] static glm::vec2 getTextureCoords(const TextureCoordinateInfo& textureInfo, const RayIntersectionHit& hit, const PickingFlags pickingFlags);
  };
}