1#define _USE_MATH_DEFINES
4#include "rapidjson/document.h"
5#include "rapidjson/filewritestream.h"
6#include "rapidjson/prettywriter.h"
7#include "rapidjson/error/en.h"
9#include "AssetStatsCommand.h"
11#include "Foundation/Logging/Logger.h"
12#include "Foundation/Platform/IO.h"
13#include "Foundation/Platform/Timer.h"
15#include "Services/TaskManager.h"
16#include "Resources/AssetManager.h"
17#include "Resources/ModelManager.h"
18#include "Resources/MeshHelper.h"
19#include "Utilities/Math.h"
20#include "Components/Core/AssetComponent.h"
30 constexpr uint32_t NoValue =
static_cast<uint32_t
>(-1);
35 struct RunningStatistics
37 void add(
double value)
39 assert(std::isfinite(value));
42 double delta = value - mean;
43 mean += delta / count;
45 minValue = std::min(minValue, value);
46 maxValue = std::max(maxValue, value);
49 void add(RunningStatistics& other)
51 size_t newCount = count + other.count;
53 double w0 = double(count) / double(newCount);
54 double w1 = double(other.count) / double(newCount);
56 mean = w0 * mean + w1 * other.mean;
59 minValue = std::min(minValue, other.minValue);
60 maxValue = std::max(maxValue, other.maxValue);
65 double minValue = std::numeric_limits<double>::max();
66 double maxValue = -std::numeric_limits<double>::max();
69 struct ProjectedStatistics
72 float minValue = std::numeric_limits<float>::max();
73 float maxValue = -std::numeric_limits<float>::max();
80 std::vector<std::unique_ptr<ErrorRange>> items;
83 const char* issueId[] = {
85 "TriangleMeanMaxEdgeTooSmall",
86 "NearScreenCoverOutOfRange",
87 "FarScreenCoverOutOfRange",
88 "NearScreenCoverAtCoarsestLodTooBig",
89 "TooLittleTriangleIncrease",
92 enum struct IssueId : uint32_t
95 TriangleMeanMaxEdgeTooSmall,
96 NearScreenCoverOutOfRange,
97 FarScreenCoverOutOfRange,
98 NearScreenCoverAtCoarsestLodTooBig,
99 TooLittleTriangleIncrease,
102 static_assert(
sizeof(issueId) ==
sizeof(issueId[0]) * size_t(IssueId::Count));
107 IssueId
id = IssueId::None;
108 uint32_t severity = 0;
113 ErrorRange(uint32_t level,
float minError,
float maxError) : level(level), minError(minError), maxError(maxError) {}
115 const uint32_t level;
116 const float minError;
117 const float maxError;
119 ErrorRanges childRanges;
120 std::vector<Issue> issues;
124 RunningStatistics lodNodeBounds;
125 RunningStatistics modelBounds;
126 RunningStatistics triangleArea;
127 RunningStatistics triangleMinEdge;
128 RunningStatistics triangleMaxEdge;
129 uint32_t modelCount = 0;
133 ProjectedStatistics triangleArea;
134 ProjectedStatistics triangleMinEdge;
135 ProjectedStatistics triangleMaxEdge;
136 float distance = 0.f;
139 float nearSplitCover = -1.f;
140 float farSplitCover = -1.f;
143 ErrorRange* getErrorRange(ErrorRanges& ranges,
const uint32_t level,
const float minError,
const float maxError)
145 Cogs::LockGuard guard(ranges.lock);
146 std::vector<std::unique_ptr<ErrorRange>>& items = ranges.items;
148 const size_t n = items.size();
149 for (
size_t i = 0; i < n; i++) {
150 if (items[i]->minError == minError && items[i]->maxError == maxError && items[i]->level == level) {
151 return items[i].get();
155 items.emplace_back();
156 items[n] = std::make_unique<ErrorRange>(level, minError, maxError);
157 return items[n].get();
166 } kind = ResourceItem::Kind::None;
175 } state = State::None;
180 ErrorRange* errorRange =
nullptr;
182 bool useRelativePaths =
true;
186 uint32_t part = ::NoValue;
191 struct ProcessModelScratch
203 std::unordered_set<std::string> assetPaths;
204 std::unordered_set<std::string> modelPaths;
211 Ctx(
const Ctx&) =
delete;
212 Ctx& operator=(
const Ctx&) =
delete;
216 Cogs::Atomic<bool> failure =
false;
217 MemoryPoolAllocator<CrtAllocator> alloc = MemoryPoolAllocator<CrtAllocator>();
221 std::list<std::unique_ptr<ProcessModelScratch>> processModelScratch;
226 std::queue<std::unique_ptr<ResourceItem>> queue;
232 std::vector<LevelPaths> levels;
236 float referenceFieldOfView = float(M_PI / 4.0);
237 float referenceScreenSize = 1000.f;
239 float splitPointScreenCoverMin = 1.f / 2.f;
240 float splitPointScreenCoverMax = 2.f;
241 float splitPointScreenCoverFurthesSplitMax = 0.2f;
242 float splitPointMinTriangleEdgeLength = 10.f;
243 float perLevelMinTriangleIncrease = 1.5f;
246 ErrorRanges rootRanges;
252 float infinityNormDistance(
const glm::vec3& lo,
const glm::vec3& hi)
254 glm::vec3 d = glm::abs(hi - lo);
255 return std::max(std::max(d.x, d.y), d.z);
259 void enqueueAsset(Ctx* ctx, ErrorRange* errorRange,
const std::string& path,
bool useRelativePaths)
266 std::unique_ptr<ResourceItem> assetItem = std::make_unique<ResourceItem>(ResourceItem{
267 .kind = ResourceItem::Kind::AssetItem,
268 .state = ResourceItem::State::Issued,
270 .errorRange = errorRange,
272 .useRelativePaths = useRelativePaths
276 Cogs::LockGuard guard(ctx->resourceQueue.lock);
277 ctx->resourceQueue.queue.push(std::move(assetItem));
278 ctx->resourceQueue.count++;
282 Cogs::LockGuard guard(ctx->uniquePaths.lock);
283 if (ctx->uniquePaths.levels.size() <= errorRange->level) {
284 ctx->uniquePaths.levels.resize(errorRange->level + 1);
286 ctx->uniquePaths.levels[errorRange->level].assetPaths.insert(path);
290 void enqueueModel(Ctx* ctx, ErrorRange* errorRange,
const std::string& path, uint32_t part)
297 std::unique_ptr<ResourceItem> modelItem = std::make_unique<ResourceItem>(ResourceItem{
298 .kind = ResourceItem::Kind::ModelItem,
299 .state = ResourceItem::State::Issued,
301 .errorRange = errorRange,
307 Cogs::LockGuard guard(ctx->resourceQueue.lock);
308 ctx->resourceQueue.queue.push(std::move(modelItem));
309 ctx->resourceQueue.count++;
313 Cogs::LockGuard guard(ctx->uniquePaths.lock);
314 if (ctx->uniquePaths.levels.size() <= errorRange->level) {
315 ctx->uniquePaths.levels.resize(errorRange->level + 1);
317 ctx->uniquePaths.levels[errorRange->level].modelPaths.insert(path);
321 void processAssetEntityItem(Ctx* ctx,
const std::string& parentPath,
const Asset* asset, ErrorRange* errorRange,
const size_t entityIx,
bool useRelativePaths);
323 void processAssetEntityLodGroup(Ctx* ctx,
const std::string& parentPath,
const Asset* asset, ErrorRange* errorRange,
const SceneEntityDefinition& entityDef,
bool useRelativePaths)
326 const PropertyRange propertyRange = asset->definition.scene.getProperties(entityDef);
327 std::span<const float> bboxValues = propertyRange.getProperty(bboxString, std::span<const float>());
328 if (bboxValues.size() != 6) {
329 LOG_ERROR(logger,
"Lod level item bounding box has wrong number of elements (%zu != 6)", bboxValues.size());
334 float bboxSize = infinityNormDistance(glm::vec3(bboxValues[0], bboxValues[1], bboxValues[2]),
335 glm::vec3(bboxValues[3], bboxValues[4], bboxValues[5]));
337 Cogs::LockGuard guard(errorRange->lock);
338 errorRange->lodNodeBounds.add(bboxSize);
341 if (entityDef.
lod.numLods) {
342 std::span<const float> errorValues = propertyRange.getProperty(errorsString, std::span<const float>());
343 if (errorValues.size() != entityDef.
lod.numLods) {
344 LOG_ERROR(logger,
"Lod level item count (=%u) does not match number of error values (=%zu)", entityDef.
lod.numLods, errorValues.size());
349 uint32_t childIx = entityDef.firstChild;
350 for (uint32_t k = 0; k < entityDef.
lod.numLods; k++) {
351 if (childIx == NoValue) {
352 LOG_ERROR(logger,
"Empty Lod-level encountered.");
357 uint32_t level = errorRange->level + entityDef.
lod.numLods - k - 1;
358 float minErrorLevel = k == 0 ? errorRange->minError : glm::clamp(errorValues[k], errorRange->minError, errorRange->maxError);
359 float maxErrorLevel = glm::clamp(k + 1 < entityDef.
lod.numLods ? errorValues[k + 1] : errorRange->maxError, errorRange->minError, errorRange->maxError);
361 ErrorRange* childErrorRange = getErrorRange(errorRange->childRanges, level, minErrorLevel, maxErrorLevel);
362 assert(childErrorRange->minError == minErrorLevel);
363 assert(childErrorRange->maxError == maxErrorLevel);
367 processAssetEntityItem(ctx, parentPath, asset, childErrorRange, childIx, useRelativePaths);
376 bool getSourcePath(Ctx* ctx, std::string& sourcePath,
const std::string& parentPath,
const Asset* asset,
380 if (entityDef.
asset.index == ::NoValue) {
381 LOG_ERROR(logger,
"Asset without source");
385 const PropertyInfo& sourceInfo = definition.scene.properties.getPropertyByIndex(entityDef.
asset.index);
386 switch (sourceInfo.type) {
387 case PropertyType::UnsignedInteger:
388 sourcePath = createAssetResourcePathFromIndex(sourceInfo.uintValue, assetResourceType);
390 case PropertyType::String: {
391 sourcePath = definition.scene.properties.getString(sourceInfo).
to_string();
395 LOG_ERROR(logger,
"Source property is neither uint nor string");
399 if (useRelativePaths && Cogs::IO::isRelative(sourcePath)) {
400 sourcePath = Cogs::IO::combine(parentPath, sourcePath);
405 void processAssetEntityItem(Ctx* ctx,
406 const std::string& parentPath,
408 ErrorRange* errorRange,
409 const size_t entityIx,
410 bool useRelativePaths)
415 if (asset->definition.scene.entities.size() <= entityIx) {
416 LOG_ERROR(logger,
"Illegal entity ix %zu", entityIx);
420 if (entityDef.isLodGroup()) {
421 processAssetEntityLodGroup(ctx, parentPath, asset, errorRange, entityDef, useRelativePaths);
423 else if (entityDef.isAsset()) {
424 std::string sourcePath;
425 if (getSourcePath(ctx, sourcePath, parentPath, asset, entityDef, AssetResourceType::Asset, useRelativePaths))
427 enqueueAsset(ctx, errorRange, sourcePath, entityDef.
asset.
flags & uint32_t(AssetFlags::RelativePaths));
430 else if (entityDef.isModel()) {
431 std::string sourcePath;
432 if (getSourcePath(ctx, sourcePath, parentPath, asset, entityDef, AssetResourceType::Model, useRelativePaths))
434 enqueueModel(ctx, errorRange, sourcePath, entityDef.
model.part);
437 else if (!entityDef.isEmpty()) {
439 if (entityDef.numFields) {
440 for (
const FieldValue& entry : std::span(definition.scene.fieldValues).subspan(entityDef.firstField, entityDef.numFields)) {
442 if (entry.componentId == ctx->assetCompTypeId && entry.fieldId == ctx->assetCompAssetFieldId && entry.type == DefaultValueType::Asset) {
444 std::string sourcePath = entry.value;
445 if (useRelativePaths && Cogs::IO::isRelative(sourcePath)) {
446 sourcePath = Cogs::IO::combine(parentPath, sourcePath);
449 if (!Cogs::IO::exists(sourcePath)) {
450 LOG_ERROR(logger,
"Path %s does not exist", sourcePath.c_str());
455 enqueueAsset(ctx, errorRange, sourcePath, useRelativePaths);
463 void processModel(Ctx* , ProcessModelScratch* scratch, ErrorRange* errorRange,
Model* model, uint32_t part)
466 size_t begin = part != ::NoValue ? part : 0;
467 size_t end = part != ::NoValue ? part + 1 : model->parts.size();
468 assert(end > begin &&
"Invalid part range");
470 size_t count = end - begin;
471 assert(count &&
"Invalid part range.");
475 worldFromModel.resize(count);
477 RunningStatistics modelBounds;
478 RunningStatistics triangleArea;
479 RunningStatistics triangleMinEdge;
480 RunningStatistics triangleMaxEdge;
481 for (
size_t i = 0; i < count; i++) {
482 const ModelPart& modelPart = model->parts[begin + i];
484 const glm::mat4& L = model->getPartTransform(modelPart);
485 if (modelPart.parentIndex != ::NoValue) {
486 assert(begin <= modelPart.parentIndex);
487 assert(modelPart.parentIndex < begin + i);
488 worldFromModel[i] = worldFromModel[modelPart.parentIndex - begin] * L;
491 worldFromModel[i] = L;
494 if (modelPart.meshIndex != ::NoValue) {
495 const MeshHandle mesh = model->meshes[modelPart.meshIndex];
497 glm::vec3 minPos = glm::vec3(std::numeric_limits<float>::max());
498 glm::vec3 maxPos = glm::vec3(-std::numeric_limits<float>::max());
504 worldPos.resize(modelPos.size());
506 const glm::mat4& M = worldFromModel[i];
507 for (
size_t k = 0, n = modelPos.size(); k < n; k++) {
508 const glm::vec4 h = M * glm::vec4(modelPos[k], 1.f);
509 worldPos[k] = (1.f / h.w) * glm::vec3(h);
514 std::span<const uint32_t> idx;
516 idx = extractIndexStreamUint32(scratch->idxBacking, mesh);
519 size_t np = worldPos.size();
520 size_t ni = scratch->dummyIndices.size();
522 scratch->dummyIndices.resize(np,
true);
523 for (
size_t k = ni; k < np; k++) {
524 scratch->dummyIndices[k] = uint32_t(k);
527 idx = std::span<const uint32_t>(scratch->dummyIndices.data(), np);
531 size_t startIndex = std::min(
size_t(modelPart.startIndex), idx.size());
532 size_t vertexCount = std::min(
size_t(modelPart.vertexCount), idx.size() - startIndex);
534 switch (modelPart.primitiveType) {
536 for (
size_t k = startIndex; k + 3 <= startIndex + vertexCount; k += 3) {
537 const glm::vec3& a = worldPos[idx[k + 0]];
538 const glm::vec3& b = worldPos[idx[k + 1]];
539 const glm::vec3& c = worldPos[idx[k + 2]];
541 const float e_ab = glm::distance(a, b);
542 const float e_bc = glm::distance(b, c);
543 const float e_ca = glm::distance(c, a);
544 triangleMinEdge.add(std::min(std::min(e_ab, e_bc), e_ca));
545 triangleMaxEdge.add(std::max(std::max(e_ab, e_bc), e_ca));
546 triangleArea.add(0.5f * glm::length(glm::cross(b - a, c - a)));
547 minPos = glm::min(minPos, glm::min(a, glm::min(b, c)));
548 maxPos = glm::max(maxPos, glm::max(a, glm::max(b, c)));
552 LOG_WARNING(logger,
"Unsupported primitive type %u", uint32_t(modelPart.primitiveType));
557 if (modelPart.boundsIndex != ::NoValue) {
558 const Cogs::Geometry::BoundingBox& bbox = model->bounds[modelPart.boundsIndex];
559 const glm::vec3 c0 = euclidean(worldFromModel[i] * glm::vec4(bbox.min.x, bbox.min.y, bbox.min.z, 1.f));
560 const glm::vec3 c1 = euclidean(worldFromModel[i] * glm::vec4(bbox.min.x, bbox.min.y, bbox.max.z, 1.f));
561 const glm::vec3 c2 = euclidean(worldFromModel[i] * glm::vec4(bbox.min.x, bbox.max.y, bbox.min.z, 1.f));
562 const glm::vec3 c3 = euclidean(worldFromModel[i] * glm::vec4(bbox.min.x, bbox.max.y, bbox.max.z, 1.f));
563 const glm::vec3 c4 = euclidean(worldFromModel[i] * glm::vec4(bbox.max.x, bbox.min.y, bbox.min.z, 1.f));
564 const glm::vec3 c5 = euclidean(worldFromModel[i] * glm::vec4(bbox.max.x, bbox.min.y, bbox.max.z, 1.f));
565 const glm::vec3 c6 = euclidean(worldFromModel[i] * glm::vec4(bbox.max.x, bbox.max.y, bbox.min.z, 1.f));
566 const glm::vec3 c7 = euclidean(worldFromModel[i] * glm::vec4(bbox.max.x, bbox.max.y, bbox.max.z, 1.f));
567 const glm::vec3 worldMin = glm::min(glm::min(glm::min(c0, c1),
569 glm::min(glm::min(c4, c5),
571 const glm::vec3 worldMax = glm::max(glm::max(glm::max(c0, c1),
573 glm::max(glm::max(c4, c5),
575 modelBounds.add(infinityNormDistance(worldMin, worldMax));
577 else if (minPos.x <= maxPos.x) {
578 modelBounds.add(infinityNormDistance(minPos, maxPos));
583 Cogs::LockGuard guard(errorRange->lock);
585 errorRange->triangleArea.add(triangleArea);
586 errorRange->triangleMinEdge.add(triangleMinEdge);
587 errorRange->triangleMaxEdge.add(triangleMaxEdge);
588 errorRange->modelBounds.add(modelBounds);
589 errorRange->modelCount++;
592 void processResource(Ctx* ctx, ResourceItem* item)
594 if (item->kind == ResourceItem::Kind::AssetItem) {
595 assert(item->handle->getType() == ResourceTypes::Asset);
596 Asset* asset =
static_cast<Asset*
>(item->handle.get());
598 std::string sourceDirectoryPath = Cogs::IO::parentPath(item->path);
599 for (
size_t i = 0; i < definition.scene.entities.size(); ++i) {
601 if (entityDef.parentIndex == ::NoValue) {
602 processAssetEntityItem(ctx, sourceDirectoryPath, asset, item->errorRange, i, item->asset.useRelativePaths);
607 else if (item->kind == ResourceItem::Kind::ModelItem) {
608 std::unique_ptr<ProcessModelScratch> scratch;
610 Cogs::LockGuard guard(ctx->threadTemporaries.lock);
611 if (!ctx->threadTemporaries.processModelScratch.empty()) {
612 scratch = std::move(ctx->threadTemporaries.processModelScratch.front());
613 ctx->threadTemporaries.processModelScratch.pop_front();
618 scratch = std::make_unique<ProcessModelScratch>();
621 assert(item->handle->getType() == ResourceTypes::Model);
622 Model* model =
static_cast<Model*
>(item->handle.get());
623 processModel(ctx, scratch.get(), item->errorRange, model, item->model.part);
627 Cogs::LockGuard guard(ctx->threadTemporaries.lock);
628 ctx->threadTemporaries.processModelScratch.push_front(std::move(scratch));
633 assert(
false &&
"Illegal workitem kind");
638 void processAssetQueueUntilDone(Ctx* ctx,
size_t maxActive)
640 LOG_DEBUG(logger,
"Processing asset files...");
643 size_t processedCount = 0;
645 double lastReport = -1000.0;
647 std::vector<std::unique_ptr<ResourceItem>> activeResourceItems, activeResourceItemsNext;
649 runFrames(ctx->context, 1,
true,
false);
651 activeResourceItemsNext.clear();
652 for (std::unique_ptr<ResourceItem>& assetItem : activeResourceItems) {
654 switch (assetItem->state) {
655 case ResourceItem::State::None: [[fallthrough]];
656 case ResourceItem::State::Issued:
657 assert(
false &&
"Invalid state");
660 case ResourceItem::State::Loading:
663 if (assetItem->handle->isLoaded()) {
665 if (assetItem->kind == ResourceItem::Kind::ModelItem) {
666 Model* model =
static_cast<Model*
>(assetItem->handle.get());
667 for (
const MeshHandle& mesh : model->meshes) {
668 if (!mesh->isLoaded()) {
669 if (mesh->hasFailedLoad()) {
670 assetItem->state = ResourceItem::State::Failed;
672 goto not_done_loading;
678 assetItem->state = ResourceItem::State::Processing;
679 assetItem->task = ctx->context->taskManager->enqueueChild(ctx->group,
680 [ctx, item_ = assetItem.get()]()
682 processResource(ctx, item_);
684 if (!assetItem->task.isValid()) {
685 assetItem->state = ResourceItem::State::Failed;
688 else if (assetItem->handle->hasFailedLoad()) {
689 assetItem->state = ResourceItem::State::Failed;
694 case ResourceItem::State::Processing:
696 if (!ctx->context->taskManager->isActive(assetItem->task)) {
697 assetItem->state = ResourceItem::State::Done;
701 case ResourceItem::State::Done:
break;
702 case ResourceItem::State::Failed:
break;
705 assert(
false &&
"Invalid enum");
710 switch (assetItem->state) {
711 case ResourceItem::State::Loading: [[fallthrough]];
712 case ResourceItem::State::Processing:
714 activeResourceItemsNext.emplace_back(std::move(assetItem));
716 case ResourceItem::State::Done:
break;
717 case ResourceItem::State::Failed:
721 assert(
false &&
"Invalid enum");
727 while (activeResourceItemsNext.size() < maxActive) {
728 std::unique_ptr<ResourceItem> item;
730 Cogs::LockGuard guard(ctx->resourceQueue.lock);
731 if (ctx->resourceQueue.queue.empty()) {
732 assert(ctx->resourceQueue.count == 0);
735 item = std::move(ctx->resourceQueue.queue.front());
736 ctx->resourceQueue.queue.pop();
737 ctx->resourceQueue.count--;
740 assert(item->state == ResourceItem::State::Issued);
741 item->state = ResourceItem::State::Loading;
742 if (item->kind == ResourceItem::Kind::AssetItem) {
743 item->handle = ctx->context->assetManager->loadAsset(item->path, NoResourceId, AssetLoadFlags::ForceUnique);
745 else if (item->kind == ResourceItem::Kind::ModelItem) {
746 item->handle = ctx->context->modelManager->loadModel(item->path, NoResourceId, ModelLoadFlags::None);
749 activeResourceItemsNext.emplace_back(std::move(item));
753 activeResourceItems.swap(activeResourceItemsNext);
755 double elapsed = timer.elapsedSeconds();
756 if (activeResourceItems.empty() || std::floor(lastReport) != std::floor(elapsed)) {
757 lastReport = elapsed;
759 size_t queueCount = 0;
761 Cogs::LockGuard guard(ctx->resourceQueue.lock);
762 queueCount = ctx->resourceQueue.count;
765 LOG_DEBUG(logger,
"%.0fs: assets: done=%zu act=%zu queue=%zu",
766 std::floor(elapsed), processedCount, activeResourceItems.size(), queueCount);
768 }
while (!activeResourceItems.empty());
770 Cogs::LockGuard guard(ctx->resourceQueue.lock);
771 assert(ctx->resourceQueue.count == 0);
773 ctx->context->taskManager->wait(ctx->group);
776 LOG_DEBUG(logger,
"Finished processing %zu asset/model items in %.1f seconds", processedCount, timer.elapsedSeconds());
781 Value getStatisticsAsJson(Ctx& ctx,
const RunningStatistics& stats,
bool includeCount)
783 Value statsJson(kObjectType);
785 statsJson.AddMember(
"count",
static_cast<uint64_t
>(stats.count), ctx.alloc);
787 statsJson.AddMember(
"mean", stats.mean, ctx.alloc);
788 statsJson.AddMember(
"minValue", stats.minValue, ctx.alloc);
789 statsJson.AddMember(
"maxValue", stats.maxValue, ctx.alloc);
793 Value getStatisticsAsJson(Ctx& ctx,
const ProjectedStatistics& stats)
795 Value statsJson(kObjectType);
796 statsJson.AddMember(
"mean", stats.mean, ctx.alloc);
797 statsJson.AddMember(
"minValue", stats.minValue, ctx.alloc);
798 statsJson.AddMember(
"maxValue", stats.maxValue, ctx.alloc);
802 Value getRangesAsJson(Ctx& ctx, ErrorRanges& ranges)
804 Value rangesJson(kArrayType);
805 for (
const std::unique_ptr<ErrorRange>& range : ranges.items) {
806 Value rangeJson(kObjectType);
808 rangeJson.AddMember(
"minDistance", range->minError, ctx.alloc);
809 if (std::isfinite(range->maxError)) { rangeJson.AddMember(
"maxDistance", range->maxError, ctx.alloc); }
810 else { rangeJson.AddMember(
"maxDistance", Value(
"inf", ctx.alloc), ctx.alloc); }
812 if (range->lodNodeBounds.count) {
813 rangeJson.AddMember(
"lodNodeBounds", getStatisticsAsJson(ctx, range->lodNodeBounds,
true), ctx.alloc);
815 if (range->modelBounds.count) {
816 rangeJson.AddMember(
"modelBounds", getStatisticsAsJson(ctx, range->modelBounds,
true), ctx.alloc);
818 if (range->triangleArea.count) {
819 rangeJson.AddMember(
"triangleCount",
static_cast<uint64_t
>(range->triangleArea.count), ctx.alloc);
820 rangeJson.AddMember(
"triangleArea", getStatisticsAsJson(ctx, range->triangleArea,
false), ctx.alloc);
821 rangeJson.AddMember(
"triangleMinEdge", getStatisticsAsJson(ctx, range->triangleMinEdge,
false), ctx.alloc);
822 rangeJson.AddMember(
"triangleMaxEdge", getStatisticsAsJson(ctx, range->triangleMaxEdge,
false), ctx.alloc);
823 if (0.f < range->pixels.distance) {
824 Value projJson(kObjectType);
825 projJson.AddMember(
"screenDistance", range->pixels.distance, ctx.alloc);
826 projJson.AddMember(
"triangleArea", getStatisticsAsJson(ctx, range->pixels.triangleArea), ctx.alloc);
827 projJson.AddMember(
"triangleMinEdge", getStatisticsAsJson(ctx, range->pixels.triangleMinEdge), ctx.alloc);
828 projJson.AddMember(
"triangleMaxEdge", getStatisticsAsJson(ctx, range->pixels.triangleMaxEdge), ctx.alloc);
829 rangeJson.AddMember(
"pixels", projJson, ctx.alloc);
832 if (!range->issues.empty()) {
833 Value issuesJson(kArrayType);
834 for (
const Issue& issue : range->issues) {
835 Value issueJson(kObjectType);
837 issueJson.AddMember(
"id", Value(issueId[
size_t(issue.id)], ctx.alloc), ctx.alloc);
838 issueJson.AddMember(
"severity", issue.severity, ctx.alloc);
839 issueJson.AddMember(
"what", Value(issue.message.c_str(), ctx.alloc), ctx.alloc);
840 issuesJson.PushBack(issueJson, ctx.alloc);
842 rangeJson.AddMember(
"issues", issuesJson, ctx.alloc);
844 if (!range->childRanges.items.empty()) {
845 rangeJson.AddMember(
"children", getRangesAsJson(ctx, range->childRanges), ctx.alloc);
847 rangesJson.PushBack(rangeJson, ctx.alloc);
852 void writeStatsAsJson(Ctx& ctx,
const std::string& path)
854 if (ctx.failure)
return;
856 Document doc(&ctx.alloc);
859 Value rules(kObjectType);
860 rules.AddMember(
"splitPointScreenCoverMin", ctx.rules.splitPointScreenCoverMin, ctx.alloc);
861 rules.AddMember(
"splitPointScreenCoverMax", ctx.rules.splitPointScreenCoverMax, ctx.alloc);
862 rules.AddMember(
"splitPointScreenCoverFurthesSplitMax", ctx.rules.splitPointScreenCoverFurthesSplitMax, ctx.alloc);
863 rules.AddMember(
"splitPointMinTriangleEdgeLength", ctx.rules.splitPointMinTriangleEdgeLength, ctx.alloc);
864 rules.AddMember(
"perLevelMinTriangleIncrease", ctx.rules.perLevelMinTriangleIncrease, ctx.alloc);
866 Value input(kObjectType);
867 input.AddMember(
"source", Value(ctx.source.c_str(), ctx.alloc), ctx.alloc);
868 input.AddMember(
"referenceFieldOfViewDegrees", glm::degrees(ctx.referenceFieldOfView), ctx.alloc);
869 input.AddMember(
"refenceScreenPixelSize", ctx.referenceScreenSize, ctx.alloc);
870 input.AddMember(
"rules", rules, ctx.alloc);
872 doc.AddMember(
"input", input, ctx.alloc);
873 doc.AddMember(
"ranges", getRangesAsJson(ctx, ctx.rootRanges), ctx.alloc);
875 FILE* file = fopen(path.c_str(),
"w");
877 LOG_ERROR(logger,
"Error opening %s for writing", path.c_str());
883 FileWriteStream stream(file, buffer,
sizeof(buffer));
884 PrettyWriter<FileWriteStream> writer(stream);
885 writer.SetMaxDecimalPlaces(4);
887 bool ok = doc.Accept(writer);
891 LOG_ERROR(logger,
"Error writing json into %s", path.c_str());
895 LOG_DEBUG(logger,
"Wrote %s", path.c_str());
899 void addIssue(Ctx& , ErrorRange* range, IssueId
id, uint32_t severity,
const char* format, ...)
901 thread_local static char buffer[256] = {};
903 va_start(args, format);
904 int n = vsnprintf(buffer,
sizeof(buffer), format, args);
907 buffer[
sizeof(buffer) - 1] =
'\0';
908 range->issues.push_back(Issue{ .message = buffer, .id = id, .severity = severity });
911 void analyzeRanges(Ctx& ctx, ErrorRanges& ranges,
float screenSizeWorld,
float referenceScreenSize,
size_t trianglesAtParentLevel)
913 size_t childBoundsCount = 0;
914 size_t boundsCount = 0;
915 size_t trianglesAtLevel = 0;
916 for (std::unique_ptr<ErrorRange>& range : ranges.items) {
917 trianglesAtLevel += range->triangleArea.count;
918 boundsCount += range->lodNodeBounds.count + range->modelBounds.count;
919 for (std::unique_ptr<ErrorRange>& childRange : range->childRanges.items) {
920 childBoundsCount += childRange->lodNodeBounds.count + childRange->modelBounds.count;
925 bool refinedLevel = 2 < boundsCount;
928 bool refinedChildLevel = 2 < childBoundsCount;
930 for (std::unique_ptr<ErrorRange>& range : ranges.items) {
931 analyzeRanges(ctx, range->childRanges, screenSizeWorld, referenceScreenSize, trianglesAtLevel);
934 if (range->lodNodeBounds.count) {
935 if (std::isfinite(range->maxError)) {
936 range->farSplitCover = float(range->lodNodeBounds.mean) / (screenSizeWorld * range->maxError);
938 if (refinedChildLevel) {
939 if ((2.f * range->farSplitCover < ctx.rules.splitPointScreenCoverMin) || (ctx.rules.splitPointScreenCoverMax < 2.f * range->farSplitCover))
941 addIssue(ctx, range.get(), IssueId::FarScreenCoverOutOfRange, 0,
942 "At %.2f: 2 x farScreenCover=%.2f is outside ideal range [%.2f, %.2f]",
943 range->maxError, 2.f * range->farSplitCover, ctx.rules.splitPointScreenCoverMin, ctx.rules.splitPointScreenCoverMax);
949 if (range->triangleArea.count) {
951 if (
double(range->triangleArea.count) < ctx.rules.perLevelMinTriangleIncrease *
double(trianglesAtParentLevel)) {
952 addIssue(ctx, range.get(), IssueId::TooLittleTriangleIncrease, 0,
953 "Current level has %zu triangles that is only %.1f%% of %zu triangles on parent level",
954 range->triangleArea.count, 100.0 *
double(range->triangleArea.count) /
double(trianglesAtParentLevel), trianglesAtParentLevel);
959 if (range->modelBounds.count) {
960 if (0.f < range->minError) {
962 range->nearSplitCover = float(range->modelBounds.mean) / (screenSizeWorld * range->minError);
965 if ((range->nearSplitCover < ctx.rules.splitPointScreenCoverMin) || (ctx.rules.splitPointScreenCoverMax < range->nearSplitCover))
967 addIssue(ctx, range.get(), IssueId::NearScreenCoverOutOfRange, 0,
968 "At %.2f: nearScreenCover=%.2f is outside ideal range[%.2f, %.2f]",
969 range->minError, range->nearSplitCover, ctx.rules.splitPointScreenCoverMin, ctx.rules.splitPointScreenCoverMax);
973 if (!std::isfinite(range->maxError) && ctx.rules.splitPointScreenCoverFurthesSplitMax < range->nearSplitCover) {
974 addIssue(ctx, range.get(), IssueId::NearScreenCoverAtCoarsestLodTooBig, 0,
975 "At %.2f: splitPointScreenCover=%.2f of coarsest level is larger than %.2f",
976 range->minError, range->nearSplitCover, ctx.rules.splitPointScreenCoverFurthesSplitMax);
981 if (0.f < range->minError) {
982 range->pixels.distance = range->minError;
983 double pixelProj = referenceScreenSize / (screenSizeWorld * range->pixels.distance);
984 if (range->triangleArea.count) {
985 range->pixels.triangleArea.mean = float(pixelProj * pixelProj * range->triangleArea.mean);
986 range->pixels.triangleArea.minValue = float(pixelProj * pixelProj * range->triangleArea.minValue);
987 range->pixels.triangleArea.maxValue = float(pixelProj * pixelProj * range->triangleArea.maxValue);
989 if (range->triangleMinEdge.count) {
990 range->pixels.triangleMinEdge.mean = float(pixelProj * range->triangleMinEdge.mean);
991 range->pixels.triangleMinEdge.minValue = float(pixelProj * range->triangleMinEdge.minValue);
992 range->pixels.triangleMinEdge.maxValue = float(pixelProj * range->triangleMinEdge.maxValue);
994 if (range->triangleMaxEdge.count) {
995 range->pixels.triangleMaxEdge.mean = float(pixelProj * range->triangleMaxEdge.mean);
996 range->pixels.triangleMaxEdge.minValue = float(pixelProj * range->triangleMaxEdge.minValue);
997 range->pixels.triangleMaxEdge.maxValue = float(pixelProj * range->triangleMaxEdge.maxValue);
999 if (range->pixels.triangleMaxEdge.mean < ctx.rules.splitPointMinTriangleEdgeLength) {
1000 addIssue(ctx, range.get(), IssueId::TriangleMeanMaxEdgeTooSmall, 0,
"Triangle mean max edge length=%.2f is less than %.2f pixels",
1001 range->pixels.triangleMaxEdge.mean, ctx.rules.splitPointMinTriangleEdgeLength);
1009 void dumpRanges(ErrorRanges& ranges, uint32_t indent)
1011 const char* indents =
" ";
1013 for (
size_t i = ranges.items.size(); 0 < i; --i) {
1014 const std::unique_ptr<ErrorRange>& range = ranges.items[i - 1];
1015 float rangeSize = range->maxError - range->minError;
1016 LOG_DEBUG(logger,
"");
1017 LOG_DEBUG(logger,
"%.*s+- errorRange=[%f, %f), errorRangeSize=%f", 4 * indent, indents, range->minError, range->maxError, rangeSize);
1018 if (range->lodNodeBounds.count) {
1019 LOG_DEBUG(logger,
"%.*s lodNodeBounds: count=%zu world={ mean=%.1f min=%.1f max=%.1f }",
1020 4 * indent, indents, range->lodNodeBounds.count,
1021 range->lodNodeBounds.mean, range->lodNodeBounds.minValue, range->lodNodeBounds.maxValue);
1023 if (range->modelBounds.count) {
1024 LOG_DEBUG(logger,
"%.*s modelBounds: count=%zu world={ mean=%.1f min=%.1f max=%.1f }",
1025 4 * indent, indents, range->modelBounds.count,
1026 range->modelBounds.mean, range->modelBounds.minValue, range->modelBounds.maxValue);
1029 if (0.f < range->nearSplitCover) {
1030 LOG_DEBUG(logger,
"%.*s nearSplitCover=%f", 4 * indent, indents, range->nearSplitCover);
1032 if (0.f < range->farSplitCover) {
1033 LOG_DEBUG(logger,
"%.*s farSplitCover=%f", 4 * indent, indents, range->farSplitCover);
1035 if (range->triangleArea.count) {
1036 LOG_DEBUG(logger,
"%.*s triangleArea: count=%zu mean=%.1f min=%.1f max=%.1f",
1037 4 * indent, indents, range->triangleArea.count,
1038 range->triangleArea.mean, range->triangleArea.minValue, range->triangleArea.maxValue);
1039 if (0.f < range->pixels.distance) {
1040 LOG_DEBUG(logger,
"%.*s pixelTriangleArea: distance=%.1f mean=%.1f min=%.1f max=%.1f",
1041 4 * indent, indents, range->pixels.distance,
1042 range->pixels.triangleArea.mean, range->pixels.triangleArea.minValue, range->pixels.triangleArea.maxValue);
1046 if (range->triangleMinEdge.count) {
1047 LOG_DEBUG(logger,
"%.*s triangleMinEdge: count=%zu mean=%.1f min=%.1f max=%.1f",
1048 4 * indent, indents, range->triangleMinEdge.count,
1049 range->triangleMinEdge.mean, range->triangleMinEdge.minValue, range->triangleMinEdge.maxValue);
1050 if (0.f < range->pixels.distance) {
1051 LOG_DEBUG(logger,
"%.*s pixelTriangleMinEdge: distance=%.1f mean=%.1f min=%.1f max=%.1f",
1052 4 * indent, indents, range->pixels.distance,
1053 range->pixels.triangleMinEdge.mean, range->pixels.triangleMinEdge.minValue, range->pixels.triangleMinEdge.maxValue);
1056 if (range->triangleMaxEdge.count) {
1057 LOG_DEBUG(logger,
"%.*s triangleMaxEdge: count=%zu mean=%.1f min=%.1f max=%.1f",
1058 4 * indent, indents, range->triangleMaxEdge.count,
1059 range->triangleMaxEdge.mean, range->triangleMaxEdge.minValue, range->triangleMaxEdge.maxValue);
1060 if (0.f < range->pixels.distance) {
1061 LOG_DEBUG(logger,
"%.*s pixelsTriangleMaxEdge: distance=%.1f mean=%.1f min=%.1f max=%.1f",
1062 4 * indent, indents, range->pixels.distance,
1063 range->pixels.triangleMaxEdge.mean, range->pixels.triangleMaxEdge.minValue, range->pixels.triangleMaxEdge.maxValue);
1066 for (
const Issue& issue : range->issues) {
1067 LOG_WARNING(logger,
"%.*s id=%s severity=%u %s",
1068 4 * indent, indents,
1069 issueId[
size_t(issue.id)], issue.severity, issue.message.c_str());
1071 dumpRanges(range->childRanges, indent + 1);
1075 void writeStatsAsLogger(Ctx& ctx,
const char* source)
1077 LOG_DEBUG(logger,
"source: %s", source);
1078 for (
size_t i = 0; i < ctx.uniquePaths.levels.size(); i++) {
1079 LOG_DEBUG(logger,
"Level %zu: %zu unique asset files, %zu unique models files", i, ctx.uniquePaths.levels[i].assetPaths.size(), ctx.uniquePaths.levels[i].modelPaths.size());
1081 LOG_DEBUG(logger,
"");
1082 LOG_DEBUG(logger,
"Reference FOV: %f radians, screen size: %f pixels", ctx.referenceFieldOfView, ctx.referenceScreenSize);
1083 dumpRanges(ctx.rootRanges, 0);
1091 LOG_ERROR(logger,
"Use in post");
1094void Cogs::Core::AssetStatsCommand::undo()
1096 LOG_ERROR(logger,
"Use in post");
1099bool Cogs::Core::AssetStatsCommand::calculateStats(
Context* context,
1105 ctx.context = context;
1107 ctx.source = IO::absolute(source_.
to_string());
1108 const std::string destination = IO::absolute(destination_.
to_string());
1110 ctx.referenceFieldOfView = properties.getProperty(
"referenceFieldOfView", ctx.referenceFieldOfView);
1111 ctx.referenceScreenSize = properties.getProperty(
"referenceScreenSize", ctx.referenceScreenSize);
1112 ctx.rules.splitPointScreenCoverMin = properties.getProperty(
"splitPointScreenCoverMin", ctx.rules.splitPointScreenCoverMin);
1113 ctx.rules.splitPointScreenCoverMax = properties.getProperty(
"splitPointScreenCoverMax", ctx.rules.splitPointScreenCoverMax);
1114 ctx.rules.splitPointScreenCoverFurthesSplitMax = properties.getProperty(
"splitPointScreenCoverFurthesSplitMax", ctx.rules.splitPointScreenCoverFurthesSplitMax);
1115 ctx.rules.splitPointMinTriangleEdgeLength = properties.getProperty(
"splitPointMinTriangleEdgeLength", ctx.rules.splitPointMinTriangleEdgeLength);
1116 ctx.rules.perLevelMinTriangleIncrease = properties.getProperty(
"perLevelMinTriangleIncrease", ctx.rules.perLevelMinTriangleIncrease);
1119 ctx.assetCompTypeId = assetCompType.
getTypeId();
1124 if (!IO::isFile(ctx.source)) {
1125 LOG_ERROR(logger,
"'source' (=\"%s\") must point to the root asset file.", ctx.source.c_str());
1129 size_t maxActive = 2 * Threads::hardwareConcurrency();
1131 enqueueAsset(&ctx, getErrorRange(ctx.rootRanges, 0, 0.f, std::numeric_limits<float>::infinity()), ctx.source,
true);
1132 processAssetQueueUntilDone(&ctx, maxActive);
1134 float screenSizeWorld = 2.f * std::tan(0.5f * ctx.referenceFieldOfView);
1135 analyzeRanges(ctx, ctx.rootRanges, screenSizeWorld, ctx.referenceScreenSize, 0);
1138 writeStatsAsLogger(ctx, ctx.source.c_str());
1140 if (Cogs::IO::isDirectory(destination)) {
1141 std::string outfile = IO::fileName(ctx.source);
1142 const char* stripEndings[] = {
".asset",
".zst" };
1143 for (
const char* ending : stripEndings) {
1144 if (
size_t o = outfile.find(ending); o != std::string::npos) {
1145 outfile = outfile.substr(0, o);
1148 writeStatsAsJson(ctx, Cogs::IO::combine(destination, outfile +
"_stats.json"));
1151 writeStatsAsJson(ctx, destination);
1154 LOG_DEBUG(logger,
"Done, failure=%s", ctx.failure ?
"true" :
"false");
1155 return !ctx.failure;
1159void Cogs::Core::AssetStatsCommand::applyPost()
1161 calculateStats(context,
1163 properties.getProperty(
"source",
StringView()),
1164 properties.getProperty(
"destination",
StringView()));
A Context instance contains all the services, systems and runtime components needed to use Cogs.
static constexpr TaskQueueId ResourceQueue
Resource task queue.
Log implementation class.
Field definition describing a single data member of a data structure.
Represents a discrete type definition, describing a native type class.
FieldId getFieldId(const Field *field) const
Get the Reflection::FieldId of the given field.
const Field * getField(const Name &name) const
Get a pointer to the field info of the field with the given name.
constexpr TypeId getTypeId() const
Get the unique Reflection::TypeId of this instance.
Provides a weakly referenced view over the contents of a string.
std::string to_string() const
String conversion method.
Contains the Engine, Renderer, resource managers and other systems needed to run Cogs....
AssetResourceType
Utility function to format a resource index as a filename.
constexpr Log getLogger(const char(&name)[LEN]) noexcept
uint16_t TypeId
Built in type used to uniquely identify a single type instance.
uint16_t FieldId
Type used to index fields.
constexpr FieldId NoField
No field id.
constexpr TypeId NoType
Definition of no type.
@ Position
Position semantic.
void apply() override
Run the command.
Defines a value to apply to a field.
bool isIndexed() const
If the mesh uses indexed geometry.
Model resources define a template for a set of connected entities, with resources such as meshes,...
Resource handle base class handling reference counting of resources derived from ResourceBase.
struct Cogs::Core::SceneEntityDefinition::@28::@31 model
SceneEntityFlags::Model is set.
uint32_t flags
Really enum of SceneEntityFlags.
uint32_t nextSibling
Next sibling in this or next lod-level.
struct Cogs::Core::SceneEntityDefinition::@28::@32 lod
SceneEntityFlags::LodGroup is set.
uint32_t nextLodSibling
Next sibling within this lod-level.
struct Cogs::Core::SceneEntityDefinition::@28::@30 asset
SceneEntityFlags::Asset is set.
Task id struct used to identify unique Task instances.
@ TriangleList
List of triangles.