ResourceStore.cpp

#include "ResourceStore.h"
#include "ResourceStoreStorage.h"
 
#include "DataFetcherManager.h"
#include "Engine.h"
#include "Context.h"
#include "MemoryContext.h"
#include "Services/Variables.h"
 
#include "Rendering/IEffects.h"
 
#include "Foundation/Logging/Logger.h"
#include "Foundation/Platform/FileContents.h"
#include "Foundation/Platform/IO.h"
#include "Foundation/StringView.h"
 
#include <functional>
 
namespace
{
  using namespace Cogs::Core;
 
  Cogs::Logging::Log logger = Cogs::Logging::getLogger("ResourceStore");
 
  ResourceBuffer emptyBytes;
 
  struct ResourceLookup
  {
    constexpr operator bool() const
    {
      return found;
    }
    ResourceBuffer buffer;
    bool found = false;
  };
 
  struct PathAndProtocol
  {
    const std::string_view path;
    ResourceProtocol protocol;
  };
 
  PathAndProtocol getPathAndProtocol(std::string_view resourceName)
  {
    auto protocol = ResourceProtocol::None;
 
    if (resourceName.substr(0, 10) == "archive://") protocol = ResourceProtocol::Archive;
    else if (resourceName.substr(0, 7) == "file://") protocol = ResourceProtocol::File;
    else if (resourceName.substr(0, 7) == "http://") protocol = ResourceProtocol::Http;
    else {
      auto offset = resourceName.find("://");
      if (offset != std::string_view::npos) {
        return { resourceName.substr(offset + 3), ResourceProtocol::Other };
      }
    }
 
    if (protocol == ResourceProtocol::File) return { resourceName.substr(7), protocol };
    else if (protocol == ResourceProtocol::Archive) return { resourceName.substr(10), protocol };
    else return { resourceName, protocol };
  }
 
  ResourceLookup getCacheBytes(ResourceStoreStorage* data, const std::string& resourceName)
  {
    Cogs::LockGuard lock(data->cache->lock);
    if (auto resourceIt = data->cache->resources.find(resourceName); resourceIt != data->cache->resources.end()) {
      return ResourceLookup{ resourceIt->second.bytes , true };
    }
 
    return ResourceLookup{};
  }
 
#ifndef EMSCRIPTEN
  ResourceLookup getFileBytes(ResourceStoreStorage* data, const std::string& resourceName, bool useCache)
  {
    auto contents = DataFetcherManager::fetchSync(resourceName);
 
    if (contents) {
      if (!useCache) {
        return ResourceLookup{ {std::make_shared<Cogs::Memory::MemoryBuffer>(contents->take())}, true };
      }
      else {
        Cogs::LockGuard lock(data->cache->lock);
 
        Resource& cached = data->cache->resources[resourceName] = Resource{ resourceName, resourceName, {std::make_shared<Cogs::Memory::MemoryBuffer>(contents->take())} };
 
        return ResourceLookup{ cached.bytes, true };
      }
    }
 
    return ResourceLookup{};
  }
#endif
 
  ResourceLookup getArchiveBytes(ResourceStoreStorage* data, const std::string& resourceName)
  {
    for (auto& archive : data->archives) {
      if (archive.hasResource(resourceName)) {
        Cogs::Memory::MemoryBuffer contents = archive.getResource(resourceName);
 
        {
          Cogs::LockGuard lock(data->cache->lock);
 
          Resource& cached = data->cache->resources[resourceName] = Resource{ resourceName, resourceName, {std::make_shared<Cogs::Memory::MemoryBuffer>(std::move(contents))} };
 
          return ResourceLookup{ cached.bytes, true };
        }
      }
    }
 
    return ResourceLookup{};
  }
 
  void loadFromArchive(ResourceStoreStorage* data, const std::string& resourceName, const std::function<void(Cogs::Core::ResourceBuffer data)>& callback)
  {
    std::string name(resourceName);
 
    std::replace(name.begin(), name.end(), '\\', '/');
 
    ResourceLookup archiveContents = getArchiveBytes(data, name);
 
    if (archiveContents) {
      callback(archiveContents.buffer);
      data->context->engine->setDirty();
      return;
    }
  }
 
  void readFileAsync(ResourceStoreStorage* data, const std::string& resourceName, std::function<void(const ResourceBuffer data)> callback)
  {
    ResourceLookup contents = getCacheBytes(data, resourceName);
 
    if (contents) {
      callback(contents.buffer);
      return;
    }
 
#ifdef EMSCRIPTEN
    auto path = (resourceName.find("https://") == std::string::npos) ? data->dataDir + resourceName : resourceName;
#else
    // Note that isRelative can't replace find here.
    std::string path = !Cogs::IO::isAbsolute(resourceName) && (resourceName.find("../") == std::string::npos) && (resourceName.find("..\\") == std::string::npos) 
      ? Cogs::IO::combine(data->dataDir, resourceName) : resourceName;
 
    if (!Cogs::IO::isFile(path)) {
      loadFromArchive(data, resourceName, callback);
      return;
    }
#endif
 
    DataFetcherManager::fetchAsync(data->context, path, [=](std::unique_ptr<Cogs::FileContents> contents)
                                   {
                                     if (contents) {
                                       ResourceBuffer result;
 
                                       {
                                         Cogs::LockGuard lock(data->cache->lock);
 
                                         Resource& c = data->cache->resources[resourceName] = Resource{ resourceName, path, { std::make_shared<Cogs::Memory::MemoryBuffer>(contents->take()) } };
 
                                         LOG_TRACE(logger, "Cached file %s. Size: %zd", resourceName.c_str(), c.bytes.size());
 
                                         result = c.bytes;
                                       }
 
                                       callback(result);
                                       data->context->engine->setDirty();
                                     }
                                     else {
#ifdef EMSCRIPTEN
                                       loadFromArchive(data, resourceName, callback);
#endif
                                     }
                                   });
  }
 
}
 
 
namespace Cogs::Core {
 
  struct IOHandler : public Cogs::IIOHandler
  {
    IOHandler(Context* context, ResourceStore* store) :
      context(context),
      store(store)
    {}
 
    bool resolveFile(const Cogs::StringView& source, const Cogs::StringView& fileName, std::string& content) override
    {
      if (source.size()) {
        // We might add the path of the source here as a temporary search path to use while resolving this
        // resource.
      }
 
      auto resolvedContents = store->getResourceContents(fileName);
 
      if (resolvedContents.size()) {
        content.append(resolvedContents.toStringView());
 
        return true;
      }
      else {
        // Let resource loading continue with empty contents.
        return true;
      }
    }
 
    bool openFile(const Cogs::StringView& fileName, std::string& content) override
    {
      return resolveFile("", fileName, content);
    }
 
    void pushSearchPaths() override { }
 
    void addSearchPath(const Cogs::StringView& path) override
    {
      store->addSearchPath(std::string(path));
    }
 
    void popSearchPaths() override { }
 
    std::string getPath(FileType type, const std::string& fileName) const
    {
      if (type == IIOHandler::FileType::ShaderCache) {
        std::string path = context->variables->get("cache.shaderCache", R"(%TEMP%\Kongsberg Digital\Cogs\ShaderCache\)");
        return Cogs::IO::expandPath(Cogs::IO::combine(path, fileName));
      }
      else if (type == IIOHandler::FileType::ShaderDump) {
        std::string path = context->variables->get("effects.dumpPath", R"(%TEMP%\Kongsberg Digital\Cogs\ShaderDumps\)");
        return Cogs::IO::expandPath(Cogs::IO::combine(path, fileName));
      }
 
      return{};
    }
 
    bool exists(FileType type, const Cogs::StringView& fileName) override
    {
      auto content = store->getResourceContents(getPath(type, std::string(fileName)), ResourceStoreFlags::QueryOnly);
 
      return !content.empty();
    }
 
    bool writeBinaryFile(FileType type, const Cogs::StringView& fileName, const void* content, size_t contentSize) override
    {
#ifdef EMSCRIPTEN
      (void)type;
      (void)fileName;
      (void)content;
      (void)contentSize;
      return false;
#else
      return Cogs::IO::writeBinaryFile(getPath(type, std::string(fileName)), content, contentSize);
#endif
    }
 
    bool openBinaryFile(FileType type, const Cogs::StringView& fileName, std::vector<uint8_t>& content) override
    {
      auto buffer = store->getResourceContents(getPath(type, std::string(fileName)));
      if (!buffer.size()) return false;
 
      const auto view = buffer.toSpan<uint8_t>();
      content.assign(view.begin(), view.end());
 
      return true;
    }
 
    Context* context = nullptr;
    ResourceStore* store = nullptr;
  };
 
}
 
Cogs::Core::ResourceStoreStorage::ResourceStoreStorage(Context* context, ResourceStore* resourceStore) :
  context(context),
  cache(std::make_unique<ResourceCache>()),
  ioHandler(std::make_unique<IOHandler>(context, resourceStore))
{}
 
 
Cogs::Core::ResourceStore::ResourceStore(Context * context) : data(std::make_unique<ResourceStoreStorage>(context, this))
{
  // Load resources from zip as default since anything else is application specific.
  // Note that Cogs Demos and Example will inject --resources.dataDir=../Data/ as command line argument for you.
  // This is useful for experimenting with resources on disk without rebuilding.
  auto dataDirSetting = context->variables->get("resources.dataDir", "");
  LOG_DEBUG(logger, "Setting data directory to %s.", dataDirSetting);
  data->dataDir = dataDirSetting;
  // Emscripten needs to see the correct path first, since it can't easily check existence of files etc.
  addSearchPath(data->dataDir);
  // In case ../Data/ is passed in as dataDir, we still want to find extension resource files.
  // This does not apply to Cogs.js since it currently relies on preloading of extension resources.
  addSearchPath("");
}
  
 
// This must be explicitly defined to make forward declaring ResourceCache in header possible.
Cogs::Core::ResourceStore::~ResourceStore() = default;
 
 
void Cogs::Core::ResourceStore::preloadResources(const std::vector<std::string> & resourceNames)
{
  for (const std::string& name : resourceNames) {
    {
      LockGuard lock(data->cache->lock);
 
      // Skip resources already being loaded, or finished loading.
      if (data->cache->loading.contains(name)) continue;
      if (data->cache->resources.contains(name)) continue;
      
      data->cache->loading.insert(name);
    }
 
    readFileAsync(data.get(), name, [this, name](const ResourceBuffer& bytes)
    {
      if (bytes.size()) {
        LOG_DEBUG(logger, "Preloaded file %s.", name.c_str());
      } else {
        LOG_DEBUG(logger, "Failed preloading file %s.", name.c_str());
      }
 
      {
        LockGuard lock(data->cache->lock);
        data->cache->loading.erase(data->cache->loading.find(name));
      }
      data->context->engine->setDirty();
    });
  }
}
 
bool Cogs::Core::ResourceStore::hasResources(const std::vector<std::string> & resourceNames)
{
  bool loaded = true;
 
  for (auto & name : resourceNames) {
    loaded &= hasResource(name);
  }
 
  return loaded;
}
 
bool Cogs::Core::ResourceStore::hasResource(const std::string & resourceName)
{
  {
    LockGuard lock(data->cache->lock);
    if (auto resourceIt = data->cache->resources.find(resourceName); resourceIt != data->cache->resources.end()) {
      return true;
    }
  }
 
#ifdef EMSCRIPTEN
  for (auto & path : data->orderedSearchPaths) {
    const auto resourcePath = path + resourceName;
 
    for (auto & archive : data->archives) {
      if (archive.hasResource(resourcePath)) return true;
    }
  }
#endif
  return false;
}
 
Cogs::Core::ResourceBuffer Cogs::Core::ResourceStore::getResourceContents(std::string_view resourceName, ResourceStoreFlags flags) const
{
  for (auto & path : data->orderedSearchPaths) {
    std::string resourcePath = IO::combine(path, std::string(resourceName));
    std::replace(resourcePath.begin(), resourcePath.end(), '\\', '/');
 
#ifndef EMSCRIPTEN
    if ((flags & (ResourceStoreFlags::NoCachedContent | ResourceStoreFlags::PreferUncachedContent)) == 0) {
#else
    {
#endif
      auto cacheContents = getCacheBytes(data.get(), resourcePath);
 
      if (cacheContents) {
        return cacheContents.buffer;
      }
    }
 
#ifndef EMSCRIPTEN
    if (!IO::exists(resourcePath)) {
      if ((flags & ResourceStoreFlags::PreferUncachedContent) != 0) {
        auto cacheContents = getCacheBytes(data.get(), resourcePath);
        if (cacheContents) {
          return cacheContents.buffer;
        }
      }
      continue;
    }
 
    bool useCache = (flags & ResourceStoreFlags::NoCachedContent) == 0;
    auto contents = getFileBytes(data.get(), resourcePath, useCache);
 
    if (contents) {
      return contents.buffer;
    }
#endif
  }
 
  for (auto & path : data->orderedSearchPaths) {
    std::string resourcePath = IO::combine(path, std::string(resourceName));
 
    std::replace(resourcePath.begin(), resourcePath.end(), '\\', '/');
 
    auto archiveContents = getArchiveBytes(data.get(), resourcePath);
 
    if (archiveContents) {
      return archiveContents.buffer;
    }
  }
 
  if ((flags & ResourceStoreFlags::QueryOnly) == 0) {
    LOG_ERROR(logger, "Could not resolve resource: %.*s.", StringViewFormat(resourceName));
  }
 
  return emptyBytes;
}
 
std::string Cogs::Core::ResourceStore::getResourceContentString(std::string_view resourceName, ResourceStoreFlags flags) const
{
 
  auto bytes = getResourceContents(resourceName, flags);
 
  std::string result;
  if (bytes.size()) {
    std::string_view view = bytes.toStringView();
    result+= view;
  }
 
  return result;
}
 
const std::string& Cogs::Core::ResourceStore::getDataDir() const
{
  return data->dataDir;
}
 
 
void Cogs::Core::ResourceStore::addResource(std::string_view resourceName, std::string_view content)
{
  Memory::MemoryBuffer buffer(content.size(), data->context->memory->ioAllocator, MemBlockType::IOResource);
  std::memcpy(buffer.data(), content.data(), buffer.size());
 
  std::string name(resourceName);
  std::replace(name.begin(), name.end(), '\\', '/');
 
  LockGuard lock(data->cache->lock);
  data->cache->resources[name] = Resource{ std::string(resourceName), std::string(resourceName), {std::make_shared<Memory::MemoryBuffer>(std::move(buffer))} };
}
 
void Cogs::Core::ResourceStore::addResource(std::string_view resourceName, const void* content_data, size_t content_size)
{
  Memory::MemoryBuffer  buffer(content_size, data->context->memory->ioAllocator, MemBlockType::IOResource);
  std::memcpy(buffer.data(), content_data, std::min(content_size, buffer.size()));
 
  std::string name(resourceName);
  std::replace(name.begin(), name.end(), '\\', '/');
 
  LockGuard lock(data->cache->lock);
  data->cache->resources[name] = Resource{ std::string(resourceName), std::string(resourceName), {std::make_shared<Memory::MemoryBuffer>(std::move(buffer))} };
}
 
void Cogs::Core::ResourceStore::addResourceArchive(const std::string & archiveName, bool prepend)
{
  if (prepend) {
    data->archives.emplace(data->archives.begin(), this, archiveName);
  }
  else {
    data->archives.emplace_back(this, archiveName);
  }
}
 
std::string Cogs::Core::ResourceStore::getResourcePath(std::string_view resourceName, ResourceStoreFlags flags) const
{
  for (auto & searchPath : data->orderedSearchPaths) {
    std::string path = IO::combine(searchPath, std::string(resourceName));
 
    std::replace(path.begin(), path.end(), '\\', '/');
 
    if ((flags & ResourceStoreFlags::NoCachedContent) == 0) {
      auto cacheContents = getCacheBytes(data.get(), path);
 
      if (cacheContents) return path;
    }
 
    if (IO::exists(path) && IO::isFile(path)) {
      return path;
    }
  }
 
  return {};
}
 
Cogs::Core::ResourceQueryResult Cogs::Core::ResourceStore::getResourceLocation(std::string_view resourceName, ResourceStoreFlags flags) const
{
  if (resourceName.empty()) return {};
 
  auto [resourcePath, protocol] = getPathAndProtocol(resourceName);
 
  if (protocol == ResourceProtocol::File || protocol == ResourceProtocol::None) {
    std::string str(resourcePath);
 
    if (protocol == ResourceProtocol::None && !IO::isRelative(str)) {
      if ((flags & ResourceStoreFlags::NoCachedContent) == 0) {
        auto cacheContents = getCacheBytes(data.get(), str);
 
        if (cacheContents) return { str, ResourceProtocol::Cache };
      }
 
      return { str, ResourceProtocol::File };
    } else {
      for (auto & searchPath : data->orderedSearchPaths) {
        const auto path = IO::combine(searchPath, str);
 
        if ((flags & ResourceStoreFlags::NoCachedContent) == 0) {
          auto cacheContents = getCacheBytes(data.get(), path);
 
          if (cacheContents) return { path, ResourceProtocol::Cache };
        }
 
        if (IO::exists(path) && IO::isFile(path)) {
          if (protocol == ResourceProtocol::None) {
            return { path, ResourceProtocol::File };
          }
        }
      }
    }
  }
 
  if (protocol == ResourceProtocol::None || protocol == ResourceProtocol::Archive) {
    for (auto & searchPath : data->orderedSearchPaths) {
      const auto archivePath = IO::combine(searchPath, std::string(resourcePath));
      
      for (auto & archive : data->archives) {
        if (archive.hasResource(archivePath)) {
          return { archivePath, ResourceProtocol::Archive };
        }
      }
    }
  }
 
  return { std::string(resourceName), ResourceProtocol::None };
}
 
void Cogs::Core::ResourceStore::addSearchPath(const std::string & path, bool prepend)
{
  if (!data->searchPaths.contains(path)) {
    data->searchPaths.insert(path);
    if (prepend) {
      data->orderedSearchPaths.insert(data->orderedSearchPaths.begin(), path);
    }
    else {
      data->orderedSearchPaths.emplace_back(path);
    }
  }
}
 
void Cogs::Core::ResourceStore::purge(const std::string & resourceName)
{
  LockGuard lock(data->cache->lock);
  if (auto it = data->cache->resources.find(resourceName);  it != data->cache->resources.end()) {
    data->cache->resources.erase(it);
  }
 
  auto it = data->archives.begin();
  for (auto& archive : data->archives)
  {
    if (archive.hasResource(resourceName))
    {
      data->archives.erase(it);
      return;
    }
    ++it;
  }
}
 
void Cogs::Core::ResourceStore::clear()
{
  LockGuard lock(data->cache->lock);
  data->cache->resources.clear();
}
 
Cogs::IIOHandler * Cogs::Core::ResourceStore::getIOHandler()
{
  return data->ioHandler.get();
}