CameraArraySystem.cpp

#include <glm/glm.hpp>
#include <glm/gtc/type_ptr.hpp>
#include <glm/gtx/quaternion.hpp>
 
#include "CameraArraySystem.h"
#include "Foundation/Logging/Logger.h"
#include "Utilities/Math.h"
 
#include "Resources/Texture.h"
#include "Rendering/ICapabilities.h"
 
namespace
{
  using namespace Cogs::Core;
 
  Cogs::Logging::Log logger = Cogs::Logging::getLogger("CameraArraySystem");
 
  bool updateViewBuffer(Context* context,
                        std::vector<ViewBufferEntry>& viewBuffer,
                        const glm::mat4& worldFromBase,
                        const std::vector<glm::mat4>& projections,
                        const std::vector<glm::mat4>& transforms,
                        const size_t numViews,
                        const float overrideNearValue,
                        const float overrideFarValue,
                        const bool overrideProjectionNearAndFar)
  {
    // We need a matching set of transform/projections
    if (projections.size() != numViews || transforms.size() != numViews) {
      return false;
    }
 
    viewBuffer.resize(numViews);
    for (size_t i = 0; i < numViews; i++) {
      ViewBufferEntry& entry = viewBuffer[i];
      entry.clipFromView = context->renderer->getProjectionMatrix(projections[i]);
 
      // Optionally change the near and far planes embedded in the provided projections. This
      // can be used to let on cameras use different depth ranges when using camera stacking.
      //
      // Code taken from cogs.js Main.cpp. I think it should in theory be possible
      // to just post-multiply with a matrix that adjusts where near and far is.
      //
      if (overrideProjectionNearAndFar && overrideNearValue>0.0 && overrideNearValue < overrideFarValue) {
 
        // Extract Left Bottom (lb) and Right Top (rt) from Inverse Projection Matrix
        glm::mat4 IP = glm::inverse(projections[i]);
        glm::vec4 lb = IP * glm::vec4(-1, -1, -1.0, 1);
        glm::vec4 rt = IP * glm::vec4(1, 1, -1.0, 1);
        lb = lb / lb.w;
        rt = rt / rt.w;
        // Construct new frustum with modified near and far
        float slb = -overrideNearValue / lb.z;
        float srt = -overrideNearValue / rt.z;
        float left = lb.x * slb;
        float bottom = lb.y * slb;
        float right = rt.x * srt;
        float top = rt.y * srt;
 
        glm::mat4 P = glm::frustum(left, right, bottom, top, overrideNearValue, overrideFarValue);
        entry.clipFromView = P;
        entry.clipFromView = context->renderer->getProjectionMatrix(P);
      }
 
      entry.worldFromView = worldFromBase * transforms[i];
      entry.viewFromClip = glm::inverse(entry.clipFromView);
      entry.viewFromWorld = glm::inverse(entry.worldFromView);
      entry.clipFromWorld = entry.clipFromView * entry.viewFromWorld;
      entry.viewFromViewport = context->renderer->getViewFromViewportMatrix(entry.viewFromClip);
    }
 
    return true;
  }
 
  bool makeCameraEncloseViews(Context* context,
                              CameraComponent& refCamComp,
                              TransformComponent& refCamTrComp,
                              std::span<ViewBufferEntry> viewBuffer,
                              const uint32_t height,
                              const float debugFrustumSqueeze,
                              const bool moveCameraBack,
                              const bool updateNearAndFar,
                              const bool overrideViewportResolution)
  {
    const size_t numViews = viewBuffer.size();
 
    // Create camera space from frustum. If there is one view, use the same camera
    // space. If there are both left-and right views, create a view in the middle.
    //
    // NOTE: this assumes that the camera is either a root node or there are no
    // transformations above it in the hierarchy.
    assert(0 < numViews);
 
    // The matrices have been modified so they are with relation to engine origin.
    glm::vec3 refCamPos;
    glm::quat refCamRot;
    if (numViews == 1) {
      refCamPos = glm::vec3(viewBuffer[0].worldFromView[3]);
      refCamRot = glm::quat_cast(glm::mat3(viewBuffer[0].worldFromView));
    }
    else {
      refCamPos = 0.5f * (glm::vec3(viewBuffer[0].worldFromView[3]) +
                          glm::vec3(viewBuffer[1].worldFromView[3]));
      refCamRot = glm::shortMix(glm::quat_cast(glm::mat3(viewBuffer[0].worldFromView)),
                                glm::quat_cast(glm::mat3(viewBuffer[1].worldFromView)), 0.5f);
    }
 
    // Convert to world coordinates, and update camera transform
    refCamTrComp.position = glm::vec3(0.f);
    refCamTrComp.coordinates = context->transformSystem->worldFromEngineCoords(refCamPos);
    refCamTrComp.rotation = refCamRot;
    refCamTrComp.setChanged();
    context->transformSystem->updateLocalToWorldTransform(refCamTrComp, true);
 
    // Find intersection of frustum edges and Z axis and back camera back,
    // tries to minimize field - of - view of combined frustum.
    if (moveCameraBack) {
      float back = 0.f;
      const glm::mat4 camViewFromWorld = glm::inverse(context->transformSystem->getLocalToWorld(&refCamTrComp));
      for (size_t i = 0; i < numViews; i++) {
 
        const glm::mat4 camViewFromViewClip = camViewFromWorld * viewBuffer[i].worldFromView * viewBuffer[i].viewFromClip;
 
        for (size_t c = 0; c < 4; c++) {
          const float x = ((c >> 0) & 1) ? -1.f : 1.f;
          const float y = ((c >> 1) & 1) ? -1.f : 1.f;
 
          const glm::vec3 a = euclidean(camViewFromViewClip * glm::vec4(x, y, -1.f, 1.f));
          const glm::vec3 b = euclidean(camViewFromViewClip * glm::vec4(x, y, 1.f, 1.f));
 
          // Let a and b form two similar right-angled triangles with shared vertex q at z-axis.
          // We want to find q:
 
          //     a.x/(a.z-q.z) = b.x/(b.z-q.z)
          //   (b.z-q.z) * a.x = (a.z-q.z) * b.x 
          // a.x b.z - a.x q.z = b.x a.z - b.x q.z
          // b.x q.z - a.x q.z = b.x a.z - a.x b.z
          //   (b.x - a.x) q.z = b.x a.z - a.x b.z
          //               q.z = (b.x a.z - a.x b.z) / (b.x - a.x)
          if (float qz = (b.x * a.z - a.x * b.z) / (b.x - a.x); std::isfinite(qz) && qz < back) {
            back = qz;
          }
          // Same for y
          if (float qz = (b.y * a.z - a.y * b.z) / (b.y - a.y); std::isfinite(qz) && qz < back) {
            back = qz;
          }
        }
      }
 
      glm::vec3 newPosEngine = euclidean(context->transformSystem->getLocalToWorld(&refCamTrComp) * glm::vec4(0.f, 0.f, -back, 1.f));
      glm::dvec3 newPosWorld = context->transformSystem->worldFromEngineCoords(newPosEngine);
      refCamTrComp.position = newPosWorld - refCamTrComp.coordinates;
      refCamTrComp.setChanged();
      context->transformSystem->updateLocalToWorldTransform(refCamTrComp, true);
    }
 
    // Figure out field-of-view by transforming all the corners of the frustums of
    // the view into the camera space we created and find the field-of-view that
    // contains all points.
    float xAngle = 0.f;
    float yAngle = 0.f;
    float nearDist = std::numeric_limits<float>::max();
    float farDist = 0.f;
    const glm::mat4 camViewFromWorld = glm::inverse(context->transformSystem->getLocalToWorld(&refCamTrComp));
    for (size_t i = 0; i < numViews; i++) {
 
      // Transform from view's clip-space to new camera space.
      const glm::mat4 camViewFromViewClip = camViewFromWorld * viewBuffer[i].worldFromView * viewBuffer[i].viewFromClip;
 
      for (size_t c = 0; c < 8; c++) {
        // Project frustum corner into new view space
        const glm::vec4 h = camViewFromViewClip * glm::vec4(((c >> 0) & 1) ? -1.f : 1.f,
                                                            ((c >> 1) & 1) ? -1.f : 1.f,
                                                            ((c >> 2) & 1) ? -1.f : 1.f,
                                                            1.f);
        const glm::vec3 p = (1.f / h.w) * glm::vec3(h);
 
        // Update bounds
        const glm::vec3 pp = glm::vec3(std::abs(p.x),
                                       std::abs(p.y),
                                       std::max(1e-3f, -p.z));
        nearDist = std::min(nearDist, pp.z);
        farDist = std::max(farDist, pp.z);
        xAngle = std::max(xAngle, std::atan2(pp.x, pp.z));
        yAngle = std::max(yAngle, std::atan2(pp.y, pp.z));
      }
    }
 
    // calc aspect ratio that is wide enough to contain all views.
    const float aspect = glm::tan(0.5f * xAngle) / glm::tan(0.5f * yAngle);
 
 
    // And update main camera.
    refCamComp.fieldOfView = 2.0f * glm::clamp(1.f - debugFrustumSqueeze, 0.1f, 1.f) * yAngle;
 
    // Optionally use near and far calculated here that matches the View's near and far
    // pretty good instead of letting cogs calculate this the normal way.
    if (updateNearAndFar) {
      refCamComp.nearDistance = nearDist;
      refCamComp.farDistance = farDist;
      refCamComp.enableClippingPlaneAdjustment = false;
    }
    else {
      refCamComp.enableClippingPlaneAdjustment = true;
    }
 
    refCamComp.viewportOrigin = glm::vec2(0.f);
    if (overrideViewportResolution) {
      // Use VR viewport size for pixel resolution calculations
      refCamComp.viewportSize.y = static_cast<float>(height);
    }
    refCamComp.viewportSize.x = aspect * refCamComp.viewportSize.y;
 
    refCamComp.setChanged();
    context->cameraSystem->updateProjection(context, refCamComp);
    return true;
  }
 
  void tryValidateTextureArraySetup(const CameraArrayComponent& camArrComp, CameraArrayData& camArrData, uint32_t& width, uint32_t& height)
  {
    assert(camArrData.textureMode == CameraArrayData::TextureMode::None);
    if (camArrComp.colorTextures.size() == 1) {
      Texture* colorTex = camArrComp.colorTextures[0].resolve();
      if (colorTex && (colorTex->description.target == Cogs::ResourceDimensions::Texture2DArray || colorTex->description.target == Cogs::ResourceDimensions::Texture2DMSArray)) {
 
        width = colorTex->description.width;
        height = colorTex->description.height;
 
        camArrData.numViews = colorTex->description.layers;
        camArrData.textureMode = CameraArrayData::TextureMode::TextureArray;
        camArrData.colorTextures = { camArrComp.colorTextures[0] };
 
        if (camArrComp.enableDepth) {
 
          if (camArrComp.depthTextures.empty()) {
            camArrData.depthMode = CameraArrayData::DepthMode::Create;
          }
 
          if (camArrComp.depthTextures.size() == 1) {
            Texture* depthTex = camArrComp.depthTextures[0].resolve();
            if (colorTex->description.target == Cogs::ResourceDimensions::Texture2DArray || colorTex->description.target == Cogs::ResourceDimensions::Texture2DMSArray) {
              if ((depthTex->description.width == width) &&
                  (depthTex->description.height == height) &&
                  (depthTex->description.layers == camArrData.numViews))
              {
                camArrData.depthMode = CameraArrayData::DepthMode::Provided;
                camArrData.depthTextures = { camArrComp.depthTextures[0] };
              }
            }
          }
 
        }
 
      }
    }
  }
 
  void tryValidateArrayOfTexturesSetup(const CameraArrayComponent& camArrComp, CameraArrayData& camArrData, uint32_t& width, uint32_t& height)
  {
    if (camArrData.textureMode != CameraArrayData::TextureMode::None || camArrComp.colorTextures.empty()) {
      return;
    }
 
    camArrData.numViews = static_cast<uint32_t>(camArrComp.colorTextures.size());
    
    camArrData.depthMode = CameraArrayData::DepthMode::None;
    if (camArrComp.enableDepth) {
      if (camArrComp.depthTextures.empty()) {
        camArrData.depthMode = CameraArrayData::DepthMode::Create;
      }
 
      if (camArrComp.depthTextures.size() == camArrComp.colorTextures.size()) {
        camArrData.depthMode = CameraArrayData::DepthMode::Provided;
        // We validate the each depth texture in the loop below
      }
    }
 
    for (size_t i = 0; i < camArrData.numViews; i++) {
 
      TextureHandle colorTexHandle = camArrComp.colorTextures[i];
      const Texture* colorTex = colorTexHandle.resolve();
      if (!(colorTex && (colorTex->description.target == Cogs::ResourceDimensions::Texture2D || colorTex->description.target == Cogs::ResourceDimensions::RenderBuffer))) {
        return;
      }
      if (i == 0) {
        width = colorTex->description.width;
        height = colorTex->description.height;
      }
      else if ((colorTex->description.width != width) || (colorTex->description.height != height)) {
        return;
      }
      camArrData.colorTextures.push_back(colorTexHandle);
 
      if (camArrData.depthMode == CameraArrayData::DepthMode::Provided) {
        TextureHandle depthTexHandle = camArrComp.depthTextures[i];
        const Texture* depthTex = depthTexHandle.resolve();
        if (depthTex &&
            (depthTex->description.target == Cogs::ResourceDimensions::Texture2D || depthTex->description.target == Cogs::ResourceDimensions::RenderBuffer) &&
            (depthTex->description.width == width) &&
            (depthTex->description.height == height))
        {
          camArrData.depthTextures.push_back(depthTexHandle);
        }
      }
    }
 
    camArrData.textureMode = CameraArrayData::TextureMode::ArrayOfTextures;
 
    if ((camArrData.depthMode == CameraArrayData::DepthMode::Provided) && (camArrData.colorTextures.size() != camArrData.depthTextures.size())) {
      camArrData.depthMode = CameraArrayData::DepthMode::None;
    }
  }
 
 
}
 
 
 
void Cogs::Core::CameraArraySystem::update(Context* context)
{
  for (CameraArrayComponent& camArrComp : pool) {
 
    const TransformComponent* camArrTrComp = camArrComp.getComponent<TransformComponent>();
    if (!camArrTrComp) continue;
 
    const glm::mat4 worldFromBase = context->transformSystem->getLocalToWorld(camArrTrComp);
 
    CameraArrayData& camArrData = getData(&camArrComp);
    camArrData.textureMode = CameraArrayData::TextureMode::None;
    camArrData.depthMode = CameraArrayData::DepthMode::None;
    camArrData.colorTextures.clear();
    camArrData.depthTextures.clear();
 
    uint32_t width = 0;
    uint32_t height = 0;
 
    tryValidateTextureArraySetup(camArrComp, camArrData, width, height);
    tryValidateArrayOfTexturesSetup(camArrComp, camArrData, width, height);
 
 
 
 
    if (camArrData.textureMode != CameraArrayData::TextureMode::None) {
      assert(0 < camArrData.numViews);
 
      // Only support 2 multiviews for now as that is currently the only use-case,
      // and generalizing this beyond that requires memory here and there.
      // Update: Allow 1 view to facilitate VR simulators with only one view
      if (camArrData.numViews <= 2) {
 
        // Check number of matrices etc and oopulate the view buffer
        if (updateViewBuffer(context,
                             camArrData.viewBuffer,
                             worldFromBase,
                             camArrComp.projections,
                             camArrComp.transforms,
                             camArrData.numViews,
                             camArrComp.overrideNearValue,
                             camArrComp.overrideFarValue,
                             camArrComp.overrideProjectionNearAndFar))
        {
          if (const EntityPtr referenceCamera = camArrComp.referenceCamera.lock(); referenceCamera) {
            if (CameraComponent* refCamComp = referenceCamera->getComponent<CameraComponent>(); refCamComp) {
              if (TransformComponent* refCamTrComp = refCamComp->getComponent<TransformComponent>(); refCamTrComp) {
                if (camArrComp.updateReferenceCamera == false || makeCameraEncloseViews(context,
                                                                                        *refCamComp, *refCamTrComp,
                                                                                        camArrData.viewBuffer,
                                                                                        height,
                                                                                        camArrComp.debugFrustumSqueeze,
                                                                                        camArrComp.moveCameraBack,
                                                                                        camArrComp.updateNearAndFar,
                                                                                        camArrComp.updateViewportResolution))
                {
                  const CameraData& camData = context->cameraSystem->getData(refCamComp);
 
                  camArrData.camData = camData;
                  camArrData.pipeline = refCamComp->renderPipeline;
 
                  camArrData.camData.viewportOrigin = { 0, 0 };
                  camArrData.camData.viewportSize.x = static_cast<float>(width);
                  camArrData.camData.viewportSize.y = static_cast<float>(height);
 
                  camArrData.camData.cameraArray = camArrComp.getComponentHandle<CameraArrayComponent>();
 
                  continue; // sucess
                }
              }
            }
          }
        }
      }
    }
 
    // One of the tests failed
    camArrData.textureMode = CameraArrayData::TextureMode::None;
  }
}