AssetPipeCommand.cpp

#include <queue>
#include <fstream>
#include <unordered_set>
 
#include "AssetPipeCommand.h"
 
#include "Batch.h"
 
#include "Foundation/Platform/Atomic.h"
#include "Foundation/Logging/Logger.h"
#include "Foundation/Platform/IO.h"
#include "Foundation/Platform/Timer.h"
 
#include "Rendering/VertexFormat.h"
 
#include "Services/TaskManager.h"
#include "Resources/AssetManager.h"
#include "Resources/ModelManager.h"
#include "Resources/MaterialInstance.h"
#include "Serialization/AssetWriter.h"
#include "Serialization/ModelWriter.h"
 
#include "Components/Core/AssetComponent.h" // AssetFlags
 
#include "zstd.h"
#include "rapidjson/stringbuffer.h"
#include "rapidjson/prettywriter.h"
 
#include "PackMeshCommand.h"
 
namespace {
  using namespace Cogs::Core;
 
  Cogs::Logging::Log logger = Cogs::Logging::getLogger("AssetPipe");
 
  constexpr uint32_t NoValue = static_cast<uint32_t>(-1);
  const Cogs::Core::StringRef bboxString = Cogs::Core::Strings::add("bbox");
  const Cogs::Core::StringRef errorsString = Cogs::Core::Strings::add("errors");
  const Cogs::Core::StringRef byteSizeString = Cogs::Core::Strings::add("byteSize");
 
  struct IdRange
  {
    uint32_t min;
    uint32_t max;
  };
  typedef std::vector<IdRange> IdRanges;
 
  struct AssetItem
  {
    enum struct State {
      None,
      Issued,
      Loading,
      IssueDependencies,
      WaitingForDeps,
      Processing,
      Done,
      Failed
    } state = State::None; 
 
    AssetHandle handle;
    std::string sourcePath;
    std::string targetPath;
    std::vector<uint32_t> resourceDependencies;
    TaskId task = NoTask;
    uint32_t resourceIndex = ~0u;
    bool useRelativePaths = true;
  };
 
  struct ModelItem
  {
    enum struct State {
      None,
      Issued,
      Loading,
      Packing,
      Writing,
      Done,
      Failed
    } state = State::None;
 
    ModelHandle handle;
    std::vector<glm::mat4> meshTransforms;
    std::vector<IdRanges> partIdRanges;
    std::string sourcePath;
    std::string targetPath;
    TaskId task = NoTask;
    uint32_t resourceIndex = ~0u;
  };
 
  struct Ctx
  {
    Ctx() {}
    // Just to make sure we do not get any unintended copies while capturing lambdas etc.
    Ctx(const Ctx&) = delete;
    Ctx& operator=(const Ctx&) = delete;
 
    Context* context = nullptr;
    std::string destination;
    TaskId processGroup = NoTask;
 
    Cogs::Atomic<bool> failure = false;
 
    struct {
      Cogs::Mutex lock;
      std::queue<std::unique_ptr<ModelItem>> queue;
      size_t count = 0;
    } modelQueue;
 
    struct {
      Cogs::Mutex lock;
      std::queue<std::unique_ptr<AssetItem>> queue;
      size_t count = 0;
    } assetQueue;
 
    struct {
      Cogs::Mutex lock;
      std::unordered_map<std::string, uint32_t> map;
      std::vector<bool> processed;
      std::vector<std::vector<uint32_t>> partSizes;
      std::vector<std::vector<IdRanges>> resourcePartIdRanges; 
    } resources;
 
 
    TaskId group = NoTask;
 
    PackMeshCommand::Options packMeshOptions{};
 
    bool prettyPrint = false;
    bool compressAsFile = true; // Requires at least cogsengine 1576 from 2023-06-21
    uint32_t trackIdRanges = 0;   // Number of id ranges to track, zero to not track id ranges
 
    uint32_t assetsProcessed = 0;
    uint32_t modelsProcessed = 0;
 
  };
 
  void validateIdRanges(const Ctx* /*ctx*/, const IdRanges& ranges)
  {
    size_t n = ranges.size();
 
    // Ranges should be well-formed
    for (size_t i = 0; i < n; i++) {
      assert(ranges[i].min <= ranges[i].max);
    }
 
    // Ranges should be separated by at least one item
    for (size_t i = 0; i + 1 < n; i++) {
      assert(ranges[i].max + 1 < ranges[i + 1].min);
    }
  }
 
  struct Gap
  {
    uint32_t size;
    uint32_t offset;
  };
 
  void restrictIdRangeCount(const Ctx* ctx, IdRanges& ranges)
  {
    size_t n = ranges.size();
    if (n <= ctx->trackIdRanges) return;
    assert(1 <= ctx->trackIdRanges);
 
    validateIdRanges(ctx, ranges);
 
    // The entire range covering both inputs, used to check result at function end.
    IdRange overall{
      .min = ranges.front().min,
      .max = ranges.back().max
    };
 
    std::vector<Gap> gaps;
    for (size_t i = 0; i + 1 < ranges.size(); i++) {
      gaps.push_back({
        .size = ranges[i + 1].min - ranges[i].max,
        .offset = static_cast<uint32_t>(i)
      });
    }
    std::sort(gaps.begin(), gaps.end(), [](const Gap& a, const Gap& b) { return a.size < b.size; });
    assert(gaps.front().size <= gaps.back().size);
 
    auto moo0 = ranges;
 
    // Make ranges adjacent to a gap to be removed identical covering the union
    size_t toRemove = n - ctx->trackIdRanges;
    for (size_t i = 0; i < toRemove; i++) {
      const Gap& gap = gaps[i];
      ranges[gap.offset + 1].min = ranges[gap.offset].min;
      ranges[gap.offset].max = ranges[gap.offset + 1].max;
    }
    assert(ranges.front().min == overall.min);
    assert(ranges.back().max == overall.max);
    auto moo1 = ranges;
 
    // Remove overlaps
    {
      size_t d = 0; // The "current" range we write to. d will never be larger than i.
      for (size_t i = 1; i < n; i++) {
        if (ranges[d].max < ranges[i].min) {  // No overlap
          d = d + 1;              // The current "current" range won't grow anymore, leave it
          ranges[d] = ranges[i];  // Initialize new "current" range with i.
        }
        else {
          ranges[d].max = ranges[i].max;      // Overlap, just extend range at d
        }
      }
      ranges.resize(d + 1); // Resize to include just the "current" range.
    }
 
    assert(ranges.size() == ctx->trackIdRanges);
 
    validateIdRanges(ctx, ranges);
 
    // Verify that the result cover at least the entire range of the inputs.
    assert(ranges.front().min == overall.min);
    assert(ranges.back().max == overall.max);
  }
 
  IdRanges mergeIdRanges(const Ctx* ctx, const IdRanges& a, const IdRanges& b)
  {
    // Validate sanity of inputs
    validateIdRanges(ctx, a);
    validateIdRanges(ctx, b);
 
    if (a.empty()) {
      return b;
    }
    else if (b.empty()) {
      return a;
    }
    assert(!a.empty() && !b.empty());
 
    // The entire range covering both inputs, used to check result at function end.
    IdRange overall{
      .min = std::min(a.front().min, b.front().min),
      .max = std::max(a.back().max, b.back().max)
   };
 
    size_t na = a.size();
    size_t nb = b.size();
    size_t ia = 0;
    size_t ib = 0;
 
    IdRanges d;
 
    // Merge the smallest angle 
    while (true) {
 
      bool any_a = ia < na;
      bool any_b = ib < nb;
 
      // Choose the range with smallest min
      IdRange t;
      if (any_a && ((any_b && (a[ia].min <= b[ib].min)) || !any_b)) {
        t = a[ia]; ia++;
      }
      else if(any_b) {
        t = b[ib]; ib++;
      }
      else {
        assert(!any_a && !any_b);
        break;
      }
 
      // add or extend range
      if (d.empty() || d.back().max + 1 < t.min) {
        d.push_back(t); // new range
      }
      else {
        d.back().max = std::max(d.back().max, t.max); // just extent current range
      }
    }
 
    // Verify that the output is well-formed.
    validateIdRanges(ctx, d);
 
    // Verify that the result cover at least the entire range of the inputs.
    assert(d.front().min == overall.min);
    assert(d.back().max == overall.max);
 
    return d;
  }
 
  void findIdRanges(Ctx* /*ctx*/, IdRanges& idRanges, std::vector<uint32_t>& ids)
  {
    // Find id ranges in model
    std::sort(ids.begin(), ids.end());
 
    if (!ids.empty()) {
 
      idRanges.push_back(IdRange{ .min = ids.front(), .max = ids.front() });
      for (uint32_t id : ids) {
 
        // Grow current range by 0 or 1
        if (id <= idRanges.back().max + 1) {
          idRanges.back().max = id;
        }
        else {
          // Create new range
          assert(idRanges.back().max + 1 < id);
          idRanges.push_back(IdRange{ .min = id, .max = id });
        }
      }
    }
  }
 
  void markResourceAsDone(Ctx* ctx, std::vector<uint32_t>&& partSizes, std::vector<IdRanges>&& partIdRanges, uint32_t resourceIndex)
  {
    if (resourceIndex == ::NoValue) return;
 
    Cogs::LockGuard guard(ctx->resources.lock);
    assert(ctx->resources.processed[resourceIndex] == false);
    assert(ctx->resources.map.size() == ctx->resources.processed.size());
    assert(ctx->resources.map.size() == ctx->resources.partSizes.size());
    assert(ctx->resources.map.size() == ctx->resources.resourcePartIdRanges.size());
    ctx->resources.processed[resourceIndex] = true;
    ctx->resources.partSizes[resourceIndex] = std::move(partSizes);
    ctx->resources .resourcePartIdRanges[resourceIndex] = std::move(partIdRanges);
  }
 
  [[nodiscard]] bool enqueueModel(Ctx* ctx, const std::string& sourcePath, const std::string& targetPath, uint32_t resourceIndex)
  {
    if (ctx->failure) return false;
 
    std::unique_ptr<ModelItem> modelItem = std::make_unique<ModelItem>(ModelItem{
      .state = ModelItem::State::Issued,
      .sourcePath = sourcePath,
      .targetPath = targetPath,
      .resourceIndex = resourceIndex
    });
 
    Cogs::LockGuard guard(ctx->modelQueue.lock);
    ctx->modelQueue.queue.push(std::move(modelItem));
    ctx->modelQueue.count++;
    return true;
  }
 
  [[nodiscard]] bool enqueueAsset(Ctx* ctx, const std::string& sourcePath, const std::string& targetPath, uint32_t resourceIndex, bool useRelativePaths)
  {
    if (ctx->failure) return false;
 
    std::unique_ptr<AssetItem> assetItem = std::make_unique<AssetItem>(AssetItem{
      .state = AssetItem::State::Issued,
      .sourcePath = sourcePath,
      .targetPath = targetPath,
      .resourceIndex = resourceIndex,
      .useRelativePaths = useRelativePaths
    });
 
    //LOG_DEBUG(logger, "Enqued asset: %s -> %s", sourcePath.c_str(), targetPath.c_str());
 
    Cogs::LockGuard guard(ctx->assetQueue.lock);
    ctx->assetQueue.queue.push(std::move(assetItem));
    ctx->assetQueue.count++;
    return true;
  }
 
  [[nodiscard]] bool packModel(Ctx* ctx, ModelItem* item)
  {
    // Meshes might disappear due to quantization of vertex positions, as
    // degenerate triangles are pruned. That is not an error, but an empty
    // mesh handle is returned. In this case, the list of meshes is
    // compacted by removing empty meshes, and the mesh indices are updated.
 
    Model* model = item->handle.resolve();
    const size_t initialMeshCount = model->meshes.size();
 
    std::vector<MeshHandle> newMeshes;
    newMeshes.reserve(initialMeshCount);
 
    std::vector<uint32_t> newMeshIndices(initialMeshCount, NoIndex);
 
    std::vector<IdRanges> meshIdRanges(initialMeshCount);
 
    for (size_t i = 0; i < initialMeshCount; i++) {
 
      MeshHandle mesh = std::move(model->meshes[i]);
 
      if (ctx->trackIdRanges) {
        std::vector<uint32_t> ids;
        if (mesh->hasStream(VertexDataType::TexCoords0)) {
          const DataStream& dataStream = mesh->getStream(VertexDataType::TexCoords0);
          ids.reserve(dataStream.numElements);
          MappedStreamReadOnly<glm::vec2> tex = mesh->mapReadOnly<glm::vec2>(VertexDataType::TexCoords0, VertexFormats::Tex2f, 0, dataStream.numElements);
          for (size_t k = 0; k < dataStream.numElements; k++) {
            uint32_t id = uint32_t(tex[k].x);
            if (ids.empty() || ids.back() != id) {
              ids.push_back(id);
            }
          }
          IdRanges idRanges;
          findIdRanges(ctx, idRanges, ids);
          restrictIdRangeCount(ctx, idRanges);
          meshIdRanges[i] = std::move(idRanges);
        }
      }
 
      glm::mat4 transform(1.f);
      std::string message;
      if (!PackMeshCommand::pack(ctx->context, message, transform, mesh, ctx->packMeshOptions)) {
        LOG_ERROR(logger, "PackMeshCommand failed: %s", message.c_str());
        return false;
      }
      else if (!message.empty()) {
        //LOG_DEBUG(logger, "%s", message.c_str());
      }
 
      // If mesh was empty, we skip it (and update part mesh index below)
      if (HandleIsValid(mesh)) {
        newMeshIndices[i] = static_cast<uint32_t>(newMeshes.size());
        newMeshes.emplace_back(std::move(mesh));
        item->meshTransforms.emplace_back(transform);
      }
    }
    model->meshes = std::move(newMeshes);
 
    // Update part mesh index with possibly compacted mesh indices
    size_t partCount = model->parts.size();
    item->partIdRanges.resize(partCount);
    for (size_t i = 0; i < partCount; i++) {
      ModelPart& part = model->parts[i];
      if (part.meshIndex != NoIndex) {
        assert(part.meshIndex < newMeshIndices.size());
        item->partIdRanges[i] = std::move(meshIdRanges[part.meshIndex]);
        part.meshIndex = newMeshIndices[part.meshIndex];
      }
    }
 
    assert(item->meshTransforms.size() == model->meshes.size());
    return true;
  }
 
 
  [[nodiscard]] bool processModel(Ctx* ctx, ModelItem* item)
  {
    Model* model = item->handle.resolve();
    const size_t partCount = model->parts.size();
 
 
    const std::vector<glm::mat4>& meshTransforms = item->meshTransforms;
 
    std::vector<uint32_t> partSizes(partCount);
    for (size_t i = 0; i < partCount; i++) {
      ModelPart& part = model->parts[i];
 
      // Mesh bytesize
      if (part.meshIndex != NoIndex) {
        assert(part.meshIndex < model->meshes.size());
 
        const glm::mat4& tt = meshTransforms[part.meshIndex];
        if (part.transformIndex != NoIndex) {
          std::span<float> values = model->properties.getFloatArray(part.transformIndex);
          assert(values.size() == 16);
 
          glm::mat4 transform = glm::make_mat4(values.data()) * tt;
          std::memcpy(values.data(), glm::value_ptr(transform), 16 * sizeof(float));
        }
        else {
          part.transformIndex = model->properties.addProperty("t", std::span(glm::value_ptr(tt), 16));
        }
 
        const Mesh* mesh = model->meshes[part.meshIndex].resolve();
 
        size_t vertexByteSize = 0;
        const MeshStreamsLayout& layout = mesh->getStreamsLayout();
        for (size_t k = 0; k < layout.numStreams; k++) {
          vertexByteSize += Cogs::getSize(layout.vertexFormats[k]);
        }
 
        if (mesh->isIndexed()) {
          uint32_t minIndex = ~0u;
          uint32_t maxIndex = 0u;
          const DataStream& indexStream = mesh->getStream(VertexDataType::Indexes);
          if (indexStream.stride == sizeof(uint16_t)) {
            const uint16_t* indices = reinterpret_cast<const uint16_t*>(indexStream.data());
            for (size_t k = 0; k < indexStream.numElements; k++) {
              minIndex = std::min(minIndex, uint32_t(indices[k]));
              maxIndex = std::max(maxIndex, uint32_t(indices[k]));
            }
          }
          else if (indexStream.stride == sizeof(uint32_t)) {
            const uint32_t* indices = reinterpret_cast<const uint32_t*>(indexStream.data());
            for (size_t k = 0; k < indexStream.numElements; k++) {
              minIndex = std::min(minIndex, indices[k]);
              maxIndex = std::max(maxIndex, indices[k]);
            }
          }
          else return false;
          if (minIndex < maxIndex) {
            partSizes[i] = uint32_t(vertexByteSize * (maxIndex - minIndex) + indexStream.stride * indexStream.numElements);
          }
        }
        else {
          uint32_t startIndex = std::min(part.startIndex, mesh->getCount());
          uint32_t vertexCount = std::min(part.vertexCount, mesh->getCount() - startIndex);
          partSizes[i] = uint32_t(vertexByteSize * vertexCount);
        }
      }
    }
 
   
 
#if 0
    if (!Cogs::IO::copyFile(item->sourcePath, item->targetPath)) return false;
#else
    uint32_t numVerticess = 0;
    uint32_t numIndices = 0;
    WriteModelSettings settings{
      .flags = (ctx->compressAsFile ? WriteModelFlags::COMPRESS_ZSTD_AS_FILE : WriteModelFlags::COMPRESS_ZSTD) | WriteModelFlags::COMPRESS_MAX
    };
    if (!writeModel(ctx->context, numVerticess, numIndices, item->targetPath, model, &settings)) return false;
    //LOG_DEBUG(logger, "Wrote %s -> %s", item->sourcePath.c_str(), item->targetPath.c_str());
#endif
 
    markResourceAsDone(ctx, std::move(partSizes), std::move(item->partIdRanges), item->resourceIndex);
 
    return true;
  }
 
  [[nodiscard]] bool getSourcePath(std::string& sourcePath, const std::string& sourceDirectoryPath, const AssetDefinition& definition, uint32_t sourcePropertyIndex, AssetResourceType resourceKind, bool relativePath)
  {
    if (sourcePropertyIndex == ::NoValue) {
      LOG_ERROR(logger, "Source property is neither uint nor string");
      return false;
    }
 
    const PropertyInfo& pathItem = definition.scene.properties.getPropertyByIndex(sourcePropertyIndex);
    if (pathItem.type == PropertyType::UnsignedInteger) {
      sourcePath = createAssetResourcePathFromIndex(pathItem.uintValue, resourceKind);
    }
    else if (pathItem.type == PropertyType::String) {
      sourcePath = definition.scene.properties.getString(pathItem).to_string();
    }
    else {
      LOG_ERROR(logger, "Source property is neither uint nor string");
      return false;
    }
 
    if (relativePath && Cogs::IO::isRelative(sourcePath)) {
      sourcePath = Cogs::IO::combine(sourceDirectoryPath, sourcePath);
    }
 
    return true;
  }
 
  auto checkResourceIndex(Ctx* ctx, const std::string& sourcePath)
  {
    uint32_t resourceIndex = ~0u;
    bool isProcessed = false;
    bool doEnqueue = false;
 
    Cogs::LockGuard guard(ctx->resources.lock);
    if (auto it = ctx->resources.map.find(sourcePath); it != ctx->resources.map.end()) {
      resourceIndex = it->second;
      isProcessed = ctx->resources.processed[resourceIndex];
    }
    else {
      resourceIndex = uint32_t(ctx->resources.map.size());
      ctx->resources.map[sourcePath] = resourceIndex;
      ctx->resources.processed.emplace_back(false);
      ctx->resources.partSizes.emplace_back();
      ctx->resources.resourcePartIdRanges.emplace_back();
      assert(ctx->resources.map.size() == ctx->resources.processed.size());
      assert(ctx->resources.map.size() == ctx->resources.partSizes.size());
      assert(ctx->resources.map.size() == ctx->resources.resourcePartIdRanges.size());
      doEnqueue = true;
    }
    return std::make_tuple(resourceIndex, isProcessed, doEnqueue);
  }
 
 
  [[nodiscard]] bool issueAssetDependencies(Ctx* ctx, AssetItem* item)
  {
    assert(HandleIsValid(item->handle) && item->handle->isLoaded());
    Asset* asset = item->handle.resolve();
    AssetDefinition& definition = asset->definition;
 
    const Cogs::Reflection::Type& assetCompType = Cogs::Reflection::TypeDatabase::getType<AssetComponent>();
    const Cogs::Reflection::TypeId assetCompTypeId = assetCompType.getTypeId();
 
    const Cogs::Reflection::Field* assetCompAssetFieldType = assetCompType.getField("asset");
    const Cogs::Reflection::FieldId assetCompAssetFieldId = assetCompAssetFieldType ? assetCompType.getFieldId(assetCompAssetFieldType) : Cogs::Reflection::NoField;
 
    // Keep track of which deps we already have issued
    std::unordered_set<uint32_t> depSet;
 
    std::string sourceDirectoryPath = Cogs::IO::parentPath(item->sourcePath);
    for (size_t i = 0; i < definition.scene.entities.size(); ++i) {
      SceneEntityDefinition& entityDef = definition.scene.entities[i];
 
      if (entityDef.isModel()) {
 
        std::string sourcePath;
        if (!getSourcePath(sourcePath, sourceDirectoryPath, definition, entityDef.model.index, AssetResourceType::Model, item->useRelativePaths)) return false;
 
        auto [resourceIndex, isProcessed, doEnqueue] = checkResourceIndex(ctx, sourcePath);
 
        PropertyInfo& pathItem = definition.scene.properties.getPropertyByIndex(entityDef.model.index);
        pathItem.type = PropertyType::UnsignedInteger;
        pathItem.uintValue = resourceIndex;
 
        // Avoid creating duplicate dependencies when a model is referenced multiple times
        if(!depSet.contains(resourceIndex)) {
          depSet.insert(resourceIndex);
 
          if (!isProcessed) {
            item->resourceDependencies.push_back(resourceIndex);
          }
 
          std::string targetPath = Cogs::IO::combine(ctx->destination, createAssetResourcePathFromIndex(resourceIndex, AssetResourceType::Model));
 
          if (doEnqueue && !enqueueModel(ctx, sourcePath, targetPath, resourceIndex)) return false;
        }
 
      }
      else if (entityDef.isAsset()) {
 
        std::string sourcePath;
        if (!getSourcePath(sourcePath, sourceDirectoryPath, definition, entityDef.model.index, AssetResourceType::Asset, item->useRelativePaths)) return false;
 
        auto [resourceIndex, isProcessed, doEnqueue] = checkResourceIndex(ctx, sourcePath);
 
        PropertyInfo& pathItem = definition.scene.properties.getPropertyByIndex(entityDef.model.index);
        pathItem.type = PropertyType::UnsignedInteger;
        pathItem.uintValue = resourceIndex;
 
        if (!depSet.contains(resourceIndex)) {
          depSet.insert(resourceIndex);
 
          if (!isProcessed) {
            item->resourceDependencies.push_back(resourceIndex);
          }
 
          std::string targetPath = Cogs::IO::combine(ctx->destination, createAssetResourcePathFromIndex(resourceIndex, AssetResourceType::Asset));
 
          bool useRelative = entityDef.asset.flags & uint32_t(AssetFlags::RelativePaths);
          entityDef.asset.flags |= uint32_t(AssetFlags::RelativePaths);
 
          if (doEnqueue && !enqueueAsset(ctx, sourcePath, targetPath, resourceIndex, useRelative)) return false;
        }
      }
      else {
 
        if (entityDef.numFields) {
          for (FieldValue& entry : std::span(definition.scene.fieldValues).subspan(entityDef.firstField, entityDef.numFields)) {
 
            if (entry.componentId == assetCompTypeId && entry.fieldId == assetCompAssetFieldId && entry.type == DefaultValueType::Asset) {
 
              std::string sourcePath = entry.value;
              if(item->useRelativePaths && Cogs::IO::isRelative(sourcePath)) {
                sourcePath = Cogs::IO::combine(sourceDirectoryPath, sourcePath);
              }
              if (!Cogs::IO::exists(sourcePath)) {
                LOG_ERROR(logger, "Path %s does not exist", sourcePath.c_str());
                return false;
              }
 
              auto [resourceIndex, isProcessed, doEnqueue] = checkResourceIndex(ctx, sourcePath);
              entry.value = Cogs::IO::combine(ctx->destination, createAssetResourcePathFromIndex(resourceIndex, AssetResourceType::Asset));;
 
              if (!depSet.contains(resourceIndex)) {
                depSet.insert(resourceIndex);
 
                if (!isProcessed) {
                  item->resourceDependencies.push_back(resourceIndex);
                }
 
                if (doEnqueue && !enqueueAsset(ctx, sourcePath, entry.value, resourceIndex, item->useRelativePaths)) return false;
              }
 
            }
          }
        }
 
      }
    }
 
    return true;
  }
 
  [[nodiscard]] bool processAsset(Ctx* ctx, AssetItem* item)
  {
 
    Asset* asset = item->handle.resolve();
    AssetDefinition& definition = asset->definition;
 
    PropertyStore newProperties;
 
    // Propagate id ranges from individual model parts upwards in the hierarchy.
    std::vector<IdRanges> assetIdRanges(1);                                 // Id ranges for this asset
    std::vector<IdRanges> entityIdRanges(definition.scene.entities.size()); // Id ranges for each entity in this asset
    if(ctx->trackIdRanges) {
      for (size_t i = 0; i < definition.scene.entities.size(); i++) {
 
        IdRanges itemIdRanges;
 
        SceneEntityDefinition& entityDef = definition.scene.entities[i];
        if (entityDef.isModel()) {
          const PropertyInfo& pathItem = definition.scene.properties.getPropertyByIndex(entityDef.model.index);
          assert(pathItem.type == PropertyType::UnsignedInteger);
 
          size_t part = 0;
          if (entityDef.model.part != NoValue) {
            part = entityDef.model.part;
          }
 
          // Get ranges for model part
          if (ctx->trackIdRanges) {
            Cogs::LockGuard guard(ctx->resources.lock);
            assert(pathItem.uintValue < ctx->resources.resourcePartIdRanges.size());
            assert(part < ctx->resources.resourcePartIdRanges[pathItem.uintValue].size());
            itemIdRanges = ctx->resources.resourcePartIdRanges[pathItem.uintValue][part];
            assert(!itemIdRanges.empty());
          }
        }
 
        // Add ranges to this asset
        assetIdRanges[0] = mergeIdRanges(ctx, assetIdRanges[0], itemIdRanges);
        restrictIdRangeCount(ctx, assetIdRanges[0]);
 
 
        // Propagate ranges to parents
        if (!itemIdRanges.empty()) {
          uint32_t parentIndex = static_cast<uint32_t>(i);
          do {
            entityIdRanges[parentIndex] = mergeIdRanges(ctx, entityIdRanges[parentIndex], itemIdRanges);
            restrictIdRangeCount(ctx, entityIdRanges[parentIndex]);
            parentIndex = definition.scene.entities[parentIndex].parentIndex;
          } while (parentIndex != SceneEntityDefinition::NoIndex);
        }
      }
    }
 
    for (size_t i = 0; i < definition.scene.entities.size(); ++i) {
 
      uint32_t firstProperty = newProperties.size();
 
      SceneEntityDefinition& entityDef = definition.scene.entities[i];
      entityDef.nameIndex = ::NoValue;
 
      const PropertyRange propertyRange = definition.scene.getProperties(entityDef);
 
      if (entityDef.isModel()) {
        assert(entityDef.model.index != ::NoValue); // Should be caught in issueAssetDepencies
 
 
        const PropertyInfo& pathItem = definition.scene.properties.getPropertyByIndex(entityDef.model.index);
        assert(pathItem.type == PropertyType::UnsignedInteger);
 
        entityDef.model.index = newProperties.addProperty(definition.scene.properties, entityDef.model.index);
 
        IdRanges itemIdRanges;
        std::vector<uint32_t> partSizes;
        {
          Cogs::LockGuard guard(ctx->resources.lock);
          assert(pathItem.uintValue < ctx->resources.partSizes.size());
          assert(ctx->resources.processed[pathItem.uintValue]);
          partSizes = ctx->resources.partSizes[pathItem.uintValue];
        }
 
        if (entityDef.model.part != NoValue) {
          if (partSizes.size() <= entityDef.model.part) {
            LOG_ERROR(logger, "Illegal part index %u, model has %zu parts", entityDef.model.part, partSizes.size());
            return false;
          }
        }
 
        // Forward bounding box if the model has this property.
        if (uint32_t ix = propertyRange.getPropertyIndex(bboxString); ix != PropertyStore::NoProperty) {
          newProperties.addProperty(definition.scene.properties, ix);
        }
      }
 
      else if (entityDef.isAsset()) {
 
        entityDef.asset.index = newProperties.addProperty(definition.scene.properties, entityDef.asset.index);
 
      }
 
      else if (entityDef.isLodGroup()) {
 
        if (ctx->trackIdRanges && !entityIdRanges[i].empty()) {
          size_t n = entityIdRanges[i].size();
          std::vector<uint32_t> ids(2 * n);
          for (size_t k = 0; k < n; k++) {
            ids[2 * k + 0] = entityIdRanges[i][k].min;
            ids[2 * k + 1] = entityIdRanges[i][k].max;
          }
          newProperties.addProperty("ids", ids);
        }
        newProperties.addProperty(definition.scene.properties, propertyRange.getPropertyIndex(bboxString));
        newProperties.addProperty(definition.scene.properties, propertyRange.getPropertyIndex(errorsString));
      }
 
      // Update range
      entityDef.firstProperty = firstProperty;
      entityDef.numProperties = newProperties.size()-firstProperty;
    }
 
    // Replace properties
    definition.scene.properties = std::move(newProperties);
 
    if (!writeAsset(ctx->context,
                    item->targetPath.c_str(),
                    item->handle,
                    ctx->prettyPrint ? AssetWriteFlags::PrettyPrint : (AssetWriteFlags::Compress | AssetWriteFlags::Strip)))
    {
      LOG_ERROR(logger, "Failed to write %s -> %s", item->sourcePath.c_str(), item->targetPath.c_str());
      return false;
    }
 
    std::vector<uint32_t> partSizes(1); // FIXME: Fill 
    markResourceAsDone(ctx, std::move(partSizes), std::move(assetIdRanges), item->resourceIndex);
 
    return true;
  }
 
  void manageActiveModels(Ctx* ctx,
                          std::vector<std::unique_ptr<ModelItem>>& loadingModels,
                          std::vector<std::unique_ptr<ModelItem>>& loadingModelsNext)
  {
    for (std::unique_ptr<ModelItem>& item : loadingModels) {
 
      bool keep = true;
      switch (item->state) {
 
      case ModelItem::State::Loading:
        if (item->handle->isLoaded()) { // Wait for model is loaded
 
          // Wait for subresources to be done
          bool done = true;
          const Model* model = item->handle.resolve();
 
          for (auto& mesh : model->meshes) {
            if (!mesh->isLoaded()) {
              done = false;
              if (mesh->hasFailedLoad()) {
                ctx->failure = true;
                keep = false;
              }
            }
          }
 
          for (auto& material : model->materials) {
            if (!material->isLoaded()) {
              done = false;
              if (material->hasFailedLoad()) {
                ctx->failure = true;
                keep = false;
              }
            }
          }
 
          if(done) {
            item->task = ctx->context->taskManager->enqueueChild(ctx->group,
                                                                [ctx, item_=item.get()]()
                                                                {
                                                                  if (!packModel(ctx, item_)) ctx->failure = true;
                                                                });
            if (item->task.isValid()) {
              item->state = ModelItem::State::Packing;
            }
            else {
              ctx->failure = true;
              keep = false;
            }
          }
        }
        else if (item->handle->hasFailedLoad()) {
          ctx->failure = true;
          keep = false;
        }
        break;
 
      case ModelItem::State::Packing:
        if (!ctx->context->taskManager->isActive(item->task)) {
 
          bool done = true;
          const Model* model = item->handle.resolve();
          for (auto& mesh : model->meshes) {
            if (!mesh->isLoaded()) {
              done = false;
              if (mesh->hasFailedLoad()) {
                ctx->failure = true;
                keep = false;
              }
            }
          }
 
          if (done) {
            ctx->context->taskManager->wait(item->task);
 
            item->task = ctx->context->taskManager->enqueueChild(ctx->group,
                                                                 [ctx, item_ = item.get()]()
            {
              if (!processModel(ctx, item_)) ctx->failure = true;
            });
            if (item->task.isValid()) {
              item->state = ModelItem::State::Writing;
            }
            else {
              ctx->failure = true;
              keep = false;
            }
          }
        }
        break;
 
 
      case ModelItem::State::Writing:
        if (!ctx->context->taskManager->isActive(item->task)) {
          ctx->context->taskManager->wait(item->task);
          item->task = NoTask;
          item->state = ModelItem::State::Done;
          keep = false;
        }
        break;
 
      default:
        assert(false && "Unexpected state");
        keep = false;
        break;
      }
 
      if (keep) loadingModelsNext.emplace_back(std::move(item));
      else ctx->modelsProcessed++;
    }
  }
 
  void addNewActiveModels(Ctx* ctx,
                          std::vector<std::unique_ptr<ModelItem>>& loadingModelsNext,
                          size_t maxCount)
  {
    while (loadingModelsNext.size() < maxCount) {
      std::unique_ptr<ModelItem> item;
      {
        Cogs::LockGuard guard(ctx->modelQueue.lock);
        if (ctx->modelQueue.queue.empty()) break;
        item = std::move(ctx->modelQueue.queue.front());
        ctx->modelQueue.queue.pop();
        ctx->modelQueue.count--;
      }
 
      item->state = ModelItem::State::Loading;
      item->handle = ctx->context->modelManager->loadModel(item->sourcePath, NoResourceId, ModelLoadFlags::None);
 
      if (!Cogs::IO::exists(item->targetPath)) {
        Cogs::IO::createDirectories(item->targetPath);
      }
      loadingModelsNext.emplace_back(std::move(item));
    }
  }
 
  void manageActiveAssets(Ctx* ctx,
                          std::vector<std::unique_ptr<AssetItem>>& loadingAssets,
                          std::vector<std::unique_ptr<AssetItem>>& loadingAssetsNext)
  {
    for (std::unique_ptr<AssetItem>& item : loadingAssets) {
 
      bool keep = true;
      switch (item->state) {
 
      case AssetItem::State::Loading:
        if (item->handle->isLoaded()) {
          item->task = ctx->context->taskManager->enqueueChild(ctx->group,
                                                               [ctx, item_=item.get()]()
                                                               {
                                                                 if (!issueAssetDependencies(ctx, item_)) ctx->failure = true;
                                                               });
          if (item->task.isValid()) {
            item->state = AssetItem::State::IssueDependencies;
          }
          else {
            ctx->failure = true;
            keep = false;
          }
        }
        else if (item->handle->hasFailedLoad()) {
          ctx->failure = true;
          keep = false;
        }
        break;
 
      case AssetItem::State::IssueDependencies:
        if (!ctx->context->taskManager->isActive(item->task)) {
          ctx->context->taskManager->wait(item->task);
          item->task = NoTask;
          item->state = AssetItem::State::WaitingForDeps;
        }
        break;
 
      case AssetItem::State::WaitingForDeps:
 
        if (!item->resourceDependencies.empty()) {
 
          bool depchange = false;
          Cogs::LockGuard guard(ctx->resources.lock);
          for (size_t i = 0; i < item->resourceDependencies.size();) {
            if (ctx->resources.processed[item->resourceDependencies[i]]) {
              std::swap(item->resourceDependencies[i], item->resourceDependencies.back());
              item->resourceDependencies.pop_back();
              depchange = true;
            }
            else {
              i++;
            }
          }
 
          if (depchange) {
            //LOG_DEBUG(logger, "%s: waiting for %zu deps", item->sourcePath.c_str(), item->resourceDependencies.size());
          }
        }
 
        if (item->resourceDependencies.empty()) {
          item->task = ctx->context->taskManager->enqueueChild(ctx->group,
                                                               [ctx, item_=item.get()]()
                                                               {
                                                                 if (!processAsset(ctx, item_)) ctx->failure = true;
                                                               });
          if (item->task.isValid()) {
            item->state = AssetItem::State::Processing;
          }
          else {
            ctx->failure = true;
            keep = false;
          }
        }
        break;
 
      case AssetItem::State::Processing:
        if (!ctx->context->taskManager->isActive(item->task)) {
          item->state = AssetItem::State::Done;
          keep = false;
        }
        break;
 
      default:
        assert(false && "Unexpected state");
        keep = false;
        break;
      }
 
      if (keep) loadingAssetsNext.emplace_back(std::move(item));
      else ctx->assetsProcessed++;
    }
  }
 
  void addNewActiveAssets(Ctx* ctx, std::vector<std::unique_ptr<AssetItem>>& loadingAssetsNext)
  {
    while (true) {
 
      std::unique_ptr<AssetItem> item;
      {
        Cogs::LockGuard guard(ctx->assetQueue.lock);
        if (ctx->assetQueue.queue.empty()) break;
        item = std::move(ctx->assetQueue.queue.front());
        ctx->assetQueue.queue.pop();
        ctx->assetQueue.count--;
      }
 
      item->state = AssetItem::State::Loading;
      item->handle = ctx->context->assetManager->loadAsset(item->sourcePath, NoResourceId, AssetLoadFlags::ForceUnique);
      if (!Cogs::IO::exists(item->targetPath)) {
        Cogs::IO::createDirectories(item->targetPath);
      }
      loadingAssetsNext.emplace_back(std::move(item));
    }
  }
 
}
 
 
void Cogs::Core::AssetPipeCommand::apply()
{ 
  LOG_ERROR(logger, "Use in post");
}
 
void Cogs::Core::AssetPipeCommand::undo()
{
  LOG_ERROR(logger, "Use in post");
}
 
void Cogs::Core::AssetPipeCommand::applyPost()
{
  Ctx ctx;
  ctx.context = context;
  ctx.group = context->taskManager->createGroup(TaskManager::ResourceQueue);
  ctx.destination = IO::absolute(properties.getProperty("destination", StringView()).to_string());
  ctx.packMeshOptions.allowIdOffset = properties.getProperty("allowIdOffset", ctx.packMeshOptions.allowIdOffset);
  ctx.packMeshOptions.allowSeparateIdStream = properties.getProperty("allowSeparateIdStream", ctx.packMeshOptions.allowSeparateIdStream);
  ctx.packMeshOptions.optimizeTriangleOrder = properties.getProperty("optimizeTriangleOrder", ctx.packMeshOptions.optimizeTriangleOrder);
  ctx.packMeshOptions.optimizeVertexOrder = properties.getProperty("optimizeVertexOrder", ctx.packMeshOptions.optimizeVertexOrder);
  ctx.prettyPrint = properties.getProperty("prettyPrint", ctx.prettyPrint);
  ctx.compressAsFile = properties.getProperty("compressAsFile", ctx.compressAsFile);
  ctx.trackIdRanges = properties.getProperty("trackIdRanges", ctx.trackIdRanges);
 
  LOG_DEBUG(logger, "destination=%s", ctx.destination.c_str());
  LOG_DEBUG(logger, "packMeshOptions: allowIdOffset=%d allowSeparateIdStream=%d optimizeTriangleOrder=%d optimizeVertexOrder=%d",
            ctx.packMeshOptions.allowIdOffset ? 1 : 0,
            ctx.packMeshOptions.allowSeparateIdStream ? 1 : 0,
            ctx.packMeshOptions.optimizeTriangleOrder ? 1 : 0,
            ctx.packMeshOptions.optimizeVertexOrder ? 1 : 0);
  LOG_DEBUG(logger, "prettyPrint=%d compressAsFile=%d trackIdRanges=%u", ctx.prettyPrint ? 1 : 0, ctx.compressAsFile ? 1 : 0, ctx.trackIdRanges);
 
  if (const PropertyInfo propInfo = properties.getProperty("target"); propInfo.type != PropertyType::Unknown) {
 
    const StringView target = properties.getString(propInfo);
    switch (target.hashLowercase()) {
    case Cogs::hash("webgl1"):
    case Cogs::hash("webgl1-lo"):
    case Cogs::hash("webgl1-low"):
      ctx.packMeshOptions.target = PackMeshCommand::Target::WebGL1_Low;
      LOG_DEBUG(logger, "PackMesh target WebGL1_Low");
      break;
 
    case Cogs::hash("webgl2-lo"):
    case Cogs::hash("webgl2-low"):
      ctx.packMeshOptions.target = PackMeshCommand::Target::WebGL2_Low;
      LOG_DEBUG(logger, "PackMesh target WebGL2_Low");
      break;
 
    case Cogs::hash("webgl2"):
    case Cogs::hash("webgl2-med"):
    case Cogs::hash("webgl2-medium"):
      ctx.packMeshOptions.target = PackMeshCommand::Target::WebGL2_Med;
      LOG_DEBUG(logger, "PackMesh target WebGL2_Med");
      break;
 
    default:
      LOG_ERROR(logger, "Unrecognized target value '%.*s'", StringViewFormat(target));
      break;
    }
  }
 
 
  const std::string source = IO::absolute(properties.getProperty("source", StringView()).to_string());
  if (!IO::isFile(source)) {
    LOG_ERROR(logger, "'source' (=\"%s\") must point to the root asset file.", source.c_str());
    return;
  }
 
  // Build output name of root asset
  std::string outfile = IO::fileName(source);
  const char* stripEndings[] = { ".asset", ".zst" };
  for (const char* ending : stripEndings) {
    if (size_t o = outfile.find(ending); o != std::string::npos) {
      outfile = outfile.substr(0, o);
    }
  }
 
  // Issue root
  (void)enqueueAsset(&ctx, source, IO::combine(ctx.destination, outfile + ".asset"), ~0u, true);
 
  size_t maxConcurrentModelLoads = (2 * Cogs::Thread::hardware_concurrency() + 1);
 
  bool done = false;
  double lastReport = -1000.0;
  Timer timer = Timer::startNew();
  std::vector<std::unique_ptr<ModelItem>> loadingModels, loadingModelsNext;
  std::vector<std::unique_ptr<AssetItem>> loadingAssets, loadingAssetsNext;
  do {
    runFrames(context, 1, true, false);
 
    loadingModelsNext.clear();
    manageActiveModels(&ctx, loadingModels, loadingModelsNext);
    addNewActiveModels(&ctx, loadingModelsNext, maxConcurrentModelLoads);
    loadingModels.swap(loadingModelsNext);
 
    loadingAssetsNext.clear();
    manageActiveAssets(&ctx, loadingAssets, loadingAssetsNext);
    addNewActiveAssets(&ctx, loadingAssetsNext);
    loadingAssets.swap(loadingAssetsNext);
 
    done = loadingModels.empty() && loadingAssets.empty();
    double elapsed = timer.elapsedSeconds();
    if (done || std::floor(lastReport) != std::floor(elapsed)) {
      lastReport = elapsed;
 
      size_t assetQueueCount = 0;
      {
        Cogs::LockGuard guard(ctx.assetQueue.lock);
        assetQueueCount = ctx.assetQueue.count;
      }
      size_t modelQueueCount = 0;
      {
        Cogs::LockGuard guard(ctx.modelQueue.lock);
        modelQueueCount = ctx.modelQueue.count;
      }
 
      LOG_DEBUG(logger, "%.0fs: assets: done=%u act=%zu queue=%zu,  models: done=%u act=%zu queue=%zu",
                std::floor(elapsed),
                ctx.assetsProcessed, loadingAssets.size(), assetQueueCount,
                ctx.modelsProcessed, loadingModels.size(), modelQueueCount);
    }
  }
  while (!done);
 
  context->taskManager->wait(ctx.group);
  LOG_DEBUG(logger, "Done, failure=%s", ctx.failure ? "true" : "false");
}