JsonParser.cpp

#include "JsonParser.h"
 
#include "Context.h"
#include "Resources/ResourceStore.h"
 
#include "Foundation/Logging/Logger.h"
#include "Foundation/Platform/FileContents.h"
 
#define _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING
#include "rapidjson/reader.h"
#include "rapidjson/error/en.h"
#undef _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING
 
#include <algorithm>
#include <filesystem>
#include "zstd.h"
 
namespace
{
  const Cogs::Logging::Log logger = Cogs::Logging::getLogger("JsonParser");
}
 
namespace Cogs
{
  size_t countLines(const StringView & source, size_t /*offset*/)
  {
    size_t count = 0;
 
    const char* data = source.data();
 
    while (data != source.end()) {
      if (*data == '\n') count++;
      ++data;
    }
 
    return count;
  }
 
  const char * findLineOffset(const StringView & source, size_t offset, int direction)
  {
    int skip = direction > 0 ? 1 : -1;
    int lines = 0;
    int target = direction > 0 ? direction - 1 : -direction;
 
    auto term = direction > 0 ? source.end() : source.begin();
    auto cur = source.data() + offset;
 
    if (direction == 1 && (*cur == '\n')) return cur + 1;
    else if (direction == 1 && (*cur == '\r')) return cur + 2;
 
    while (cur != term) {
      if (*cur == '\n') {
        if (++lines > target) {
          ++cur;
          break;
        }
      }
      cur += skip;
    }
 
    return cur;
  }
 
  StringView findLine(const StringView & source, size_t offset, int direction)
  {
    if (source.empty()) return {};
    if (offset >= source.size()) return {};
 
    const char* begin = findLineOffset(source, offset, direction);
    const char* end = findLineOffset(source, offset, direction + 1);
    size_t len = std::distance(begin, end);
 
    assert(len >= 0 && "JSON string offsets invalid.");
 
    return StringView(begin, len > 0 ? len - 2 : 0);
  }
}
 
Document Cogs::Core::parseJson(const StringView & content, JsonParseFlags flags)
{
  Document document;
  Cogs::Memory::MemoryBuffer decompressed(MemBlockType::IOResource);
  StringView jsonContent = content;
 
  bool zstdMagic = (4 < content.size()) && (uint8_t(content[0])== 0x28 && uint8_t(content[1]) == 0xb5 && uint8_t(content[2]) == 0x2f && uint8_t(content[3]) == 0xfd);
  // Handle JSON files compressed with ZSTD
  if (zstdMagic ||((flags & JsonParseFlags::Compressed) == JsonParseFlags::Compressed))
  {
    LOG_TRACE(logger, "ZSTD detected");
 
    ZSTD_DCtx* zstd_dctx = ZSTD_createDCtx();
    assert(zstd_dctx);
 
    const auto size = ZSTD_getFrameContentSize(content.data(), content.size());
    if (size == ZSTD_CONTENTSIZE_UNKNOWN) {
      LOG_ERROR(logger, "ZSTD_CONTENTSIZE_UNKNOWN");
      return document;
    }
    else if (size == ZSTD_CONTENTSIZE_ERROR) {
      LOG_ERROR(logger, "ZSTD_CONTENTSIZE_ERROR");
      return document;
    }
#if defined( EMSCRIPTEN )
    // 'size_t' and 'unsigned long long' are the same types on x64 platform.
    // The 'if' statement is only valid on x32 (Emscripten).
    else if (std::numeric_limits<size_t>::max() < size) {
      LOG_ERROR(logger, "File too large: %s", std::to_string(size).c_str());
      return document;
    }
#endif
 
    decompressed.resize(static_cast<size_t>(size), false);
    const auto dataSize = ZSTD_decompressDCtx(zstd_dctx, decompressed.data(), decompressed.size(), content.data(), content.size());
    if (ZSTD_isError(dataSize)) {
      LOG_ERROR(logger, "ZSTD decompression failed: %s", ZSTD_getErrorName(dataSize));
      return document;
    }
 
    ZSTD_freeDCtx(zstd_dctx);
    LOG_TRACE(logger, "ZSTD decoding done");
 
    const void* dataPtr = decompressed.data();
    jsonContent = StringView(static_cast<const char*>(dataPtr), dataSize);
  }
 
  // Parse JSON data
  document.Parse<ParseFlag::kParseCommentsFlag | kParseTrailingCommasFlag>(jsonContent.data(), jsonContent.size());
  if (document.HasParseError()) {
    const ParseErrorCode errorCode = document.GetParseError();
    const size_t offset = document.GetErrorOffset();
 
    auto prevLine = findLine(jsonContent, offset, -1);
    auto currLine = findLine(jsonContent, offset, 0);
    auto nextLine = findLine(jsonContent, offset, 1);
 
    if (currLine.size()) {
      LOG_ERROR(logger, "Error parsing JSON: %s", GetParseError_En(errorCode));
      auto lineNo = countLines(jsonContent, offset);
 
      if (prevLine.size()) LOG_ERROR(logger, "%3zd:  %.*s", lineNo - 1, StringViewFormat(prevLine));
      if (currLine.size()) LOG_ERROR(logger, "%3zd:  %.*s", lineNo, StringViewFormat(currLine));
      auto len = (jsonContent.data() + offset) - currLine.data();
      auto lineOffset = std::max((size_t)0, (size_t)len);
 
      std::string spaces(lineOffset, ' ');
      spaces += "^";
 
      if (jsonContent[offset] == '\n') spaces += " (\\n)";
      else if (jsonContent[offset] == '\r') spaces += " (\\r)";
 
      LOG_ERROR(logger, "      %s", spaces.c_str());
 
      if (nextLine.size()) LOG_ERROR(logger, "%3zd:  %.*s", lineNo + 1, StringViewFormat(nextLine));
    } else {
      LOG_ERROR(logger, "Error parsing JSON: %s", GetParseError_En(errorCode));
    }
  }
 
  return document;
}
 
Document Cogs::Core::parseJson(Context * context, const StringView & fileName, JsonParseFlags flags)
{
  ResourceStoreFlags resourceFlags = ResourceStoreFlags::None;
#ifndef EMSCRIPTEN
  if ((flags & JsonParseFlags::NoCachedContent) == JsonParseFlags::NoCachedContent) {
    resourceFlags |= ResourceStoreFlags::NoCachedContent;
  }
  else if ((flags & JsonParseFlags::PreferUncachedContent) == JsonParseFlags::PreferUncachedContent) {
    resourceFlags |= ResourceStoreFlags::PreferUncachedContent;
  }
#endif
 
  ResourceBuffer content = context->resourceStore->getResourceContents(fileName, resourceFlags);
  if (!content.size()) {
    LOG_ERROR(logger, "Could not open JSON file %.*s.", StringViewFormat(fileName));
    return {};
  }
 
  return parseJson(content.toStringView(), flags);
}
 
Document Cogs::Core::parseJson(class Context* /*context*/, std::unique_ptr<FileContents> contents, JsonParseFlags flags)
{
  return parseJson(StringView(static_cast<const char*>(contents->data()), contents->size), flags);
}