1#include "OGC3DTilesSubtree.h"
3#include "Systems/OGC3DTilesSystem.h"
4#include "ExtensionRegistry.h"
9#include <Foundation/Logging/Logger.h>
10#include <Foundation/Platform/IO.h>
11#include <Foundation/BitTwiddling/MortonCode.h>
13#include <Serialization/JsonParser.h>
20using namespace Cogs::Core::OGC3DTilesSubtree;
21using namespace std::chrono_literals;
35 uint64_t bitstreamLength;
39 inline uint32_t interleaveBits(uint16_t a, uint16_t b)
45 inline uint64_t interleaveBits(uint16_t a, uint16_t b, uint16_t c)
56 inline uint32_t concatBits(uint16_t a, uint16_t b,
int numBitsFromB)
58 b = b << (16 - numBitsFromB);
59 uint32_t ax = uint32_t(a) << 16;
61 return ax >> (16 - numBitsFromB);
69printAvailableSubtrees(
const Subtree* subtree)
71 int bitstreamIdx = subtree->childSubtreeAvailability.bitstreamIdx;
72 int level = subtree->globalCoord.level + subtree->levelsPerSubtree;
73 int range = (int)std::pow(2, level);
74 bool isOctTree = subtree->globalCoord.z >= 0;
76 printf(
"subtree->globalCoord.level=%d, levelsPerSubtree=%d\n", subtree->globalCoord.level, subtree->levelsPerSubtree);
77 LOG_DEBUG(logger,
"printAvailableSubtrees(): level=%d, range=%d, section=%d (x=%d, y=%d, z=%d):",
78 level, range, bitstreamIdx, subtree->globalCoord.x, subtree->globalCoord.y, subtree->globalCoord.z);
80 if (subtree->childSubtreeAvailability.availableCount == 0) {
81 LOG_DEBUG(logger,
" => No subtrees available\n");
85 for (
int x = 0; x < range; ++x) {
86 for (
int y = 0; y < range; ++y) {
87 for (
int z = 0; z < range; ++z) {
88 uint32_t mortonIndex = 0;
90 mortonIndex = interleaveBits(x, y, z);
93 mortonIndex = interleaveBits(x, y);
96 uint32_t levelOffset = 0;
97 uint32_t offset = levelOffset + mortonIndex;
99 assert(bitstreamIdx < subtree->bitstreamSections.size() &&
"Index out of bounds");
102 auto bsBegin = subtree->bitstreams[section.bitstreamIdx].cbegin();
103 std::vector<unsigned char>::const_iterator start = bsBegin + section.startIdx;
104 std::vector<unsigned char>::const_iterator end = bsBegin + section.startIdx + section.length;
105 std::vector<unsigned char> buf(start, end);
107 int courseIdx = offset >> 3;
108 int fineIdx = offset & 0b111;
110 assert(level >= 0 &&
"Internal error");
111 assert(courseIdx < buf.size() &&
"Index out of bounds");
112 unsigned char val = buf[courseIdx];
113 std::bitset<8> valBits(val);
115 bool isSet = valBits.test(fineIdx);
117 if (isOctTree) LOG_DEBUG(logger,
" %d.%d.%d.%d.subtree\n", level, x, y, z);
118 else LOG_DEBUG(logger,
" %d.%d.%d.subtree\n", level, x, y);
120 LOG_DEBUG(logger,
" parent?: %d:%d|%d\n", level - 1, x >> 1, y >> 1);
133parseAvailabilityJSON(
const Value& value,
Availability* target)
135 target->allIsAvailable =
false;
136 target->availableCount = 0;
138 if (value.HasMember(
"constant")) {
139 target->allIsAvailable = value[
"constant"].GetInt() >= 1;
142 if (!target->allIsAvailable && value.MemberCount() > 1) {
143 if (value.HasMember(
"availableCount")) {
144 target->availableCount = value[
"availableCount"].GetInt();
147 LOG_ERROR(logger,
"Availability struct must contain a 'availableCount' member if 'constant=0' or undefined.");
151 if (value.HasMember(
"bitstream")) {
152 target->bitstreamIdx = value[
"bitstream"].GetInt();
155 if (target->availableCount != 0) {
156 LOG_ERROR(logger,
"Availability struct must contain a 'bitstream' member if 'constant==0' or undefined and 'availibilityCount != 0'.");
171fetchSubtreeBufferFile(
Context* context,
const std::string& URI,
const std::string& originURL,
Subtree* target,
int bufferIdx, uint32_t uniqueComponentId)
173 assert(bufferIdx <
int(target->bitstreamSections.size()));
175 size_t pos = originURL.rfind(
"/");
176 assert(pos != std::string::npos &&
"Invalid originURL");
177 std::string baseURL = originURL.substr(0, pos);
178 std::string newURL = baseURL +
"/" + URI;
182 size_t accessTokenPos = originURL.find(
"access_token");
183 if (accessTokenPos != std::string::npos) {
184 size_t endPos = originURL.find(
"&", accessTokenPos);
185 if (endPos == std::string::npos) {
186 endPos = originURL.size();
188 std::string accessToken = originURL.substr(accessTokenPos, endPos - accessTokenPos);
189 newURL +=
"?" + accessToken;
192 target->bitstreams.resize(target->bitstreamSections.size());
196 target->pendingRequests += 1;
198 (void)DataFetcherManager::fetchAsync(context, newURL,
199 [context, target, bufferIdx, uniqueComponentId](std::unique_ptr<Cogs::FileContents> inData) {
204 auto data = inData.release();
206 target->bitstreams[section.bitstreamIdx].reserve(section.length);
207 for (
size_t j = 0; j < section.length; ++j) {
208 target->bitstreams[section.bitstreamIdx].emplace_back(data->ptr[section.startIdx + j]);
210 target->pendingRequests -= 1;
214 OGC3DTilesSystem* system = ExtensionRegistry::getExtensionSystem<OGC3DTilesSystem>(context);
215 for (
auto& comp : system->
pool) {
218 if (componentState->uniqueComponentId == uniqueComponentId) {
219 context->
engine->setDirty();
229parseSubtreeBufferJSON(
Context* context, std::string_view json,
Subtree* target,
const std::string& originURL, uint32_t uniqueComponentId)
231 Document doc = parseJson(json, JsonParseFlags::None);
233 std::vector<std::vector<std::pair<uint32_t, uint32_t>>> bufferSections;
234 if (doc.HasMember(
"bufferViews")) {
235 GenericArray array = doc[
"bufferViews"].GetArray();
236 for (Value& bv : array) {
237 if (!(bv.HasMember(
"buffer") && bv.HasMember(
"byteOffset") && bv.HasMember(
"byteLength"))) {
238 LOG_ERROR(logger,
"No 'buffer', 'byteOffset' and 'byteLength' in 'bufferViews' section");
241 BitstreamSection bss{ (uint8_t)bv[
"buffer"].GetUint(), bv[
"byteOffset"].GetUint(), bv[
"byteLength"].GetUint() };
242 target->bitstreamSections.emplace_back(bss);
247 if (doc.HasMember(
"tileAvailability")) {
248 ok &= parseAvailabilityJSON(doc[
"tileAvailability"], &target->tileAvailability);
250 LOG_ERROR(logger,
"Could not parse 'tileAvailability'");
253 if (!target->tileAvailability.allIsAvailable && target->tileAvailability.availableCount != 0) {
254 assert(target->tileAvailability.bitstreamIdx < target->bitstreams.size() &&
"Bitstream index out of bounds");
258 LOG_ERROR(logger,
"Subtree JSON did not contain a 'tileAvailability' section.");
262 if (doc.HasMember(
"contentAvailability")) {
263 GenericArray array = doc[
"contentAvailability"].GetArray();
264 target->contentAvailability.resize(array.Size());
266 for (Value& c : array) {
267 ok &= parseAvailabilityJSON(c, &target->contentAvailability[i]);
269 LOG_ERROR(logger,
"Could not parse 'contentAvailability' nr %d", (i + 1));
275 LOG_ERROR(logger,
"Subtree JSON did not contain a 'contentAvailability' section.");
279 if (doc.HasMember(
"childSubtreeAvailability")) {
280 ok &= parseAvailabilityJSON(doc[
"childSubtreeAvailability"], &target->childSubtreeAvailability);
282 LOG_ERROR(logger,
"Could not parse 'childSubtreeAvailability'");
286 LOG_ERROR(logger,
"Subtree JSON did not contain a 'childSubtreeAvailability' section.");
290 if (doc.HasMember(
"buffers") && target->tileAvailability.availableCount > 0) {
291 GenericArray array = doc[
"buffers"].GetArray();
293 for (Value& buf : array) {
294 assert(buf.HasMember(
"byteLength"));
296 if (buf.HasMember(
"uri")) {
298 fetchSubtreeBufferFile(context, buf[
"uri"].GetString(), originURL, target, bufferIdx, uniqueComponentId);
302 assert(bufferIdx == 0 &&
"Internal buffer must always be index 0");
308 if (!target->tileAvailability.allIsAvailable && target->tileAvailability.availableCount > 0) {
309 LOG_ERROR(logger,
"Subtree JSON has sparse tile-availibility but did not contain a 'buffers' section.");
320 const uint8_t* data =
static_cast<const uint8_t*
>(buf->data());
321 const Header* h = (
const Header*)data;
323 if (h->magic != 0x74627573) {
324 LOG_ERROR(logger,
"Subtree header magic value was not 0x74627573 (was 0x%x)", h->magic);
328 if (h->version != 1) {
329 LOG_ERROR(logger,
"Subtree version was not '1' (was %d)", h->version);
334 if (h->bitstreamLength > 0) {
336 uint32_t paddedJSONLength =
static_cast<uint32_t
>(h->jsonLength);
337 uint32_t offset =
sizeof(Header) + paddedJSONLength;
338 const uint8_t* bitstreamStartPtr = &(
static_cast<const uint8_t*
>(buf->data())[offset]);
339 target->bitstreams.emplace_back(std::vector<uint8_t>(bitstreamStartPtr, bitstreamStartPtr + h->bitstreamLength));
342 LOG_ERROR(logger,
"parseSubtreeBufferBinary(): BitstreamLength value in header <= 0");
346 const char* jsonStartPtr = &(
static_cast<const char*
>(buf->data())[
sizeof(Header)]);
347 const std::string json = std::string(jsonStartPtr, h->jsonLength);
348 bool ok = parseSubtreeBufferJSON(context, json, target, originURL, uniqueComponentId);
354 if (target->bitstreamSections.size() > 1) {
355 LOG_DEBUG(logger,
"parseSubtreeBufferBinary(): Bitstream sections:");
356 for (
int x = 1; x < target->bitstreamSections.size(); ++x) {
358 int bufIdx = section.bitstreamIdx;
359 int start = section.startIdx;
360 int len = section.length;
362 LOG_DEBUG(logger,
" #%d: BufferIdx=%d, start=%d, len=%d => [ ", x, bufIdx, start, len);
363 for (
int i = (len - 1); i >= 0; --i) {
364 std::bitset<8> x(target->bitstreams[bufIdx][i]);
365 std::cout << x <<
' ';
367 LOG_DEBUG(logger,
"] (NB: read backwards)\n");
374std::vector<OGC3DTiles::Coord>
377 std::vector<OGC3DTiles::Coord> coords;
379 if (subtree->childSubtreeAvailability.allIsAvailable) {
380 return getAllChildren(subtree, leafNodeCoord);
383 if (subtree->childSubtreeAvailability.availableCount == 0 || subtree->pendingRequests > 0) {
387 const int bitstreamIdx = subtree->childSubtreeAvailability.bitstreamIdx;
388 const int level = subtree->globalCoord.level + subtree->levelsPerSubtree;
389 const bool isOctTree = subtree->globalCoord.z >= 0;
395 assert((localCoord.x * 2 + 1) < 0xFFFF &&
"LocalCoord.x*2+1 offset out of the 16bit bounds");
396 assert((localCoord.y * 2 + 1) < 0xFFFF &&
"LocalCoord.y*2+1 offset out of the 16bit bounds");
397 assert((isOctTree ? (localCoord.z * 2 + 1) : 0) < 0xFFFF &&
"LocalCoord.z*2+1 offset out of the 16bit bounds");
398 assert(level >= 0 &&
"Internal error");
400 const uint16_t xOffset =
static_cast<uint16_t
>(localCoord.x * 2);
401 const uint16_t yOffset =
static_cast<uint16_t
>(localCoord.y * 2);
402 const uint16_t zOffset =
static_cast<uint16_t
>(isOctTree ? localCoord.z * 2 : 0);
404 coords.reserve(xOffset * yOffset * (isOctTree ? zOffset : 1));
406 const size_t bufSize = section.length;
407 const std::vector<uint8_t>& bitarray = subtree->bitstreams[section.bitstreamIdx];
409 for (uint16_t x = xOffset; x <= xOffset + 1; ++x) {
410 for (uint16_t y = yOffset; y <= yOffset + 1; ++y) {
411 for (uint16_t z = zOffset; z <= zOffset + 1; ++z) {
412 uint64_t mortonIndex = 0;
414 mortonIndex = interleaveBits(x, y, z);
417 mortonIndex = interleaveBits(x, y);
419 assert(mortonIndex < 0xFFFFFFFF &&
"Internal error. Morton index is way too large...");
421 const uint32_t courseIdx =
static_cast<uint32_t
>(mortonIndex >> 3);
422 const uint32_t fineIdx = mortonIndex & 0b111;
424 assert((section.startIdx + courseIdx) <
int(bufSize) &&
"Index out of bounds");
425 const uint8_t val = bitarray[section.startIdx + courseIdx];
427 std::bitset<8> valBits(val);
428 if (valBits.test(fineIdx)) {
445 assert(coord.level <= subtree->globalCoord.level + subtree->levelsPerSubtree &&
"Coord level is deeper into subtree than allowed max depth.");
446 const bool isLeaf = coord.level == subtree->globalCoord.level + (subtree->levelsPerSubtree - 1);
454std::vector<OGC3DTiles::Coord>
457 const bool isOctTree = subtree->globalCoord.z >= 0;
459 const int localLevel = localCoord.level + 1;
461 std::vector<OGC3DTiles::Coord> children;
462 children.reserve(isOctTree ? 8 : 4);
465 assert((localCoord.x * 2 + 1) < 0xFFFF &&
"LocalCoord.x*2+1 offset out of the 16bit bounds");
466 assert((localCoord.y * 2 + 1) < 0xFFFF &&
"LocalCoord.y*2+1 offset out of the 16bit bounds");
467 assert((isOctTree ? (localCoord.z * 2 + 1) : 0) < 0xFFFF &&
"LocalCoord.z*2+1 offset out of the 16bit bounds");
469 const uint16_t xOffset =
static_cast<uint16_t
>(localCoord.x * 2);
470 const uint16_t yOffset =
static_cast<uint16_t
>(localCoord.y * 2);
471 const uint16_t zOffset =
static_cast<uint16_t
>(isOctTree ? localCoord.z * 2 : 0);
473 for (uint16_t x = xOffset; x <= xOffset + 1; ++x) {
474 for (uint16_t y = yOffset; y <= yOffset + 1; ++y) {
475 for (uint16_t z = zOffset; z <= zOffset + 1; ++z) {
476 int numbits = localLevel;
477 int32_t gx = concatBits(
static_cast<uint16_t
>(subtree->globalCoord.x), x, numbits);
478 int32_t gy = concatBits(
static_cast<uint16_t
>(subtree->globalCoord.y), y, numbits);
479 int32_t gz = isOctTree ? concatBits(
static_cast<uint16_t
>(subtree->globalCoord.z), z, numbits) : -1;
481 children.emplace_back(childCoord);
490 assert((isOctTree && children.size() == 8) || (!isOctTree && children.size() == 4) &&
"Internal error");
499std::vector<OGC3DTiles::Coord>
502 std::vector<OGC3DTiles::Coord> children;
503 if (subtree->pendingRequests > 0) {
507 int bitstreamIdx = subtree->tileAvailability.bitstreamIdx;
508 assert(bitstreamIdx <
int(subtree->bitstreamSections.size()) &&
"Index out of bounds");
511 const bool isOctTree = subtree->globalCoord.z >= 0;
512 const float N = isOctTree ? 8.0f : 4.0f;
514 const int localLevel = localCoord.level + 1;
517 assert((localCoord.x * 2 + 1) < 0xFFFF &&
"LocalCoord.x*2+1 offset out of the 16bit bounds");
518 assert((localCoord.y * 2 + 1) < 0xFFFF &&
"LocalCoord.y*2+1 offset out of the 16bit bounds");
519 assert((isOctTree ? (localCoord.z * 2 + 1) : 0) < 0xFFFF &&
"LocalCoord.z*2+1 offset out of the 16bit bounds");
521 const uint16_t xOffset =
static_cast<uint16_t
>(localCoord.x * 2);
522 const uint16_t yOffset =
static_cast<uint16_t
>(localCoord.y * 2);
523 const uint16_t zOffset =
static_cast<uint16_t
>(isOctTree ? localCoord.z * 2 : 0);
524 children.reserve(xOffset * yOffset * (isOctTree ? zOffset : 1));
526 const size_t bufSize = section.length;
527 const std::vector<uint8_t>& bitarray = subtree->bitstreams[section.bitstreamIdx];
529 for (uint16_t x = xOffset; x <= xOffset + 1; ++x) {
530 for (uint16_t y = yOffset; y <= yOffset + 1; ++y) {
531 for (uint16_t z = zOffset; z <= zOffset + 1; ++z) {
532 const uint32_t mortonIndex =
static_cast<uint32_t
>(isOctTree ? interleaveBits(x, y, z) : interleaveBits(x, y));
533 const uint32_t levelOffset =
static_cast<uint32_t
>((std::pow(N, localLevel) - 1) / (N - 1));
534 const uint32_t offset = levelOffset + mortonIndex;
535 const int coarseIdx = offset >> 3;
536 const int fineIdx = offset & 0b111;
538 assert((section.startIdx + coarseIdx) <
int(bufSize) &&
"Index out of bounds");
539 const uint8_t val = bitarray[section.startIdx + coarseIdx];
541 std::bitset<8> valBits(val);
542 if (valBits.test(fineIdx)) {
543 int numbits = localLevel;
544 const int32_t gx = concatBits(
static_cast<uint16_t
>(subtree->globalCoord.x), x, numbits);
545 const int32_t gy = concatBits(
static_cast<uint16_t
>(subtree->globalCoord.y), y, numbits);
546 const int32_t gz = isOctTree ? concatBits(
static_cast<uint16_t
>(subtree->globalCoord.z), z, numbits) : -1;
549 assert(childCoord.level > parent.level &&
"Child coord must have a higher level than the parent");
550 children.emplace_back(childCoord);
563 for (
auto c : children) {
565 for (
auto x : children) {
566 if (x.toHash() == c.toHash()) ++count;
569 LOG_DEBUG(logger,
"ERROR: Node %s is duplicated in child list\n", c.toString().c_str());
586Cogs::Core::DataFetcherManager::FetchId
587OGC3DTilesSubtree::fetchSubtree(
Cogs::Core::Context* context,
const std::string& URL, uint32_t uniqueComponentId, FetchSubtreeCallback callback)
589 Cogs::Core::DataFetcherManager::FetchId fetchId = DataFetcherManager::fetchAsync(context, URL,
590 [context, URL, callback, uniqueComponentId](std::unique_ptr<Cogs::FileContents> inData) {
597 bool success =
false;
599 if (data && data->data()) {
600 subtree = new Subtree;
601 subtree->pendingRequests = 0;
602 subtree->tileAvailability.allIsAvailable = false;
603 subtree->childSubtreeAvailability.allIsAvailable = false;
605 size_t paramPos = URL.find_last_of(
'?');
606 const std::string fileURL = URL.substr(0, paramPos == std::string::npos ? URL.size() : paramPos);
608 Memory::MemoryBuffer contentsBuff = inData->take();
609 if (IO::extension(fileURL) ==
".subtree" || IO::extension(fileURL) ==
".SUBTREE") {
610 success = parseSubtreeBufferBinary(context, &contentsBuff, subtree, URL, uniqueComponentId);
612 else if (IO::extension(fileURL) ==
".json" || IO::extension(fileURL) ==
".JSON") {
613 std::string_view jsonStr = std::string_view(
static_cast<const char*
>(contentsBuff.data()), contentsBuff.size());
614 success = parseSubtreeBufferJSON(context, jsonStr, subtree, URL, uniqueComponentId);
617 LOG_ERROR(logger,
"fetchSubtree(): Unknown subtree extension for '%s'", fileURL.c_str());
626 LOG_ERROR(logger,
"fetchSubtree(): Could not fetch subtree '%s'", URL.c_str());
629 auto finishTask = [callback, subtree]() {
633#if defined(EMSCRIPTEN)
638 context->
engine->runTaskInMainThread(std::move(finishTask));
646checkAvailability(
const Subtree* subtree,
int bitstreamSectionIdx,
int localLevel,
int localX,
int localY,
int localZ)
648 if (subtree->pendingRequests > 0) {
652 uint64_t mortonIndex = 0;
656 mortonIndex =
static_cast<uint32_t
>(interleaveBits(
static_cast<uint16_t
>(localX),
static_cast<uint16_t
>(localY),
static_cast<uint16_t
>(localZ)));
659 mortonIndex = interleaveBits(
static_cast<uint16_t
>(localX),
static_cast<uint16_t
>(localY));
661 assert(mortonIndex < 0xFFFFFFFF &&
"Internal error. Morton index is way too large.");
663 uint32_t levelOffset =
static_cast<uint32_t
>((std::pow(N, localLevel) - 1) / (N - 1));
664 uint32_t offset = levelOffset + ((uint32_t)mortonIndex);
666 assert(bitstreamSectionIdx <
int(subtree->bitstreamSections.size()) &&
"Index out of bounds");
667 const BitstreamSection& section = subtree->bitstreamSections[bitstreamSectionIdx];
669 if (section.bitstreamIdx >= subtree->bitstreams.size()) {
670 LOG_ERROR(logger,
"checkAvailibility(): Bitstream index > number of available bitstreams (%d vs %d)", section.bitstreamIdx, (
int)subtree->bitstreams.size());
673 assert(section.bitstreamIdx < subtree->bitstreams.size());
674 const size_t bufSize = section.length;
676 int coarseIdx = offset >> 3;
677 int fineIdx = offset & 0b111;
679 if (coarseIdx >
int(bufSize)) {
680 LOG_ERROR(logger,
"checkAvailability idx out of bounds for '%d:%d|%d|%d': %d vs %d", localLevel, localX, localY, localZ, (
int)coarseIdx, (
int)bufSize);
684 unsigned char val = subtree->bitstreams[section.bitstreamIdx][section.startIdx + coarseIdx];
685 std::bitset<8> valBits(val);
686 return valBits.test(fineIdx);
694 bool hasContent =
false;
695 for (
const Availability& c : subtree->contentAvailability) {
696 if (c.allIsAvailable) {
700 else if (c.availableCount > 0) {
701 hasContent = checkAvailability(subtree, c.bitstreamIdx, localCoord.level, localCoord.x, localCoord.y, localCoord.z);
718OGC3DTilesSubtree::getSubtreesRootCoord(
int levelsPerSubtree,
const OGC3DTiles::Coord& coord)
720 int subtreeLevel = coord.level / levelsPerSubtree;
721 int levelsLeft = coord.level % levelsPerSubtree;
724 coord.x >> levelsLeft,
725 coord.y >> levelsLeft,
726 coord.z >= 0 ? coord.z >> levelsLeft : -1 };
732 const int relativeLevel = globalCoord.level - subtree->globalCoord.level;
735 globalCoord.x - (subtree->globalCoord.x << relativeLevel),
736 globalCoord.y - (subtree->globalCoord.y << relativeLevel) ,
737 globalCoord.z >= 0 ? globalCoord.z - (subtree->globalCoord.z << relativeLevel) : -1
ComponentPool< ComponentType > pool
Pool of components managed by the system.
A Context instance contains all the services, systems and runtime components needed to use Cogs.
std::unique_ptr< class Engine > engine
Engine instance.
static bool componentIsStale(Context *context, uint32_t id)
Log implementation class.
Contains the Engine, Renderer, resource managers and other systems needed to run Cogs....
constexpr Log getLogger(const char(&name)[LEN]) noexcept
constexpr uint64_t mortonCode(uint16_t i, uint16_t j, uint16_t k)
Interleave bits of 3 values to form the 3-way Morton code.
Abstract base class storing data read from a file.