Cogs.Core
Batch.cpp
1#include "Batch.h"
2
3#include "Context.h"
4#include "Engine.h"
5#include "Editor.h"
6
7#include "Serialization/JsonParser.h"
8
9#include "Utilities/Parsing.h"
10
11#include "Commands/ResourceCommands.h"
12#include "Commands/MergeCommand.h"
13#include "Commands/DumpStatsCommand.h"
14#include "Commands/ExportCommand.h"
15#include "Commands/MeshOpCommands.h"
16
17#include "Systems/Core/ModelSystem.h"
18#include "Systems/Core/RenderSystem.h"
19
20#include "Resources/AssetManager.h"
21#include "Resources/ModelManager.h"
22#include "Resources/MeshManager.h"
23#include "Resources/MaterialManager.h"
24#include "Resources/TextureManager.h"
25
26#include "Services/Variables.h"
27
28#include "ExtensionRegistry.h"
29
30#include "Foundation/Logging/Logger.h"
31#include "Foundation/Platform/IO.h"
32#include "Foundation/Platform/Timer.h"
33
34#define _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING
35#include "rapidjson/stringbuffer.h"
36#include "rapidjson/writer.h"
37#undef _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING
38
39#include <regex>
40
41namespace
42{
43 Cogs::Logging::Log logger = Cogs::Logging::getLogger("Batch");
44
46 bool matchFilename(const std::string name, const std::string pattern)
47 {
48 const auto extension = Cogs::IO::extension(pattern);
49 const auto stem = Cogs::IO::stem(pattern);
50
51 bool extensionMatch = false;
52 auto entryExtension = Cogs::IO::extension(name);
53 auto entryStem = Cogs::IO::stem(name);
54
55 if (extension == "*") {
56 extensionMatch = true;
57 }
58 else if (extension == entryExtension) {
59 extensionMatch = true;
60 }
61
62 bool patternMatch = false;
63
64 if (stem == "*") {
65 patternMatch = true;
66 }
67 else if (stem.find('*') != std::string::npos) {
68 std::string stemEx = stem;
69 stemEx.replace(stem.find('*'), 1, ".*");
70
71 std::regex stemRegex(stemEx);
72
73 std::smatch match;
74 std::regex_search(entryStem, match, stemRegex);
75 if (match.size()) patternMatch = true;
76 }
77
78 return extensionMatch && patternMatch;
79 }
80}
81
82#ifndef EMSCRIPTEN
83
84namespace Cogs::Core
85{
87 struct BatchSourceDestination {
89 std::string source;
91 std::string destination;
92
93 public:
94 BatchSourceDestination() = default;
95 BatchSourceDestination(std::string source, std::string destination)
96 {
97 this->source = source;
98 this->destination = destination;
99 }
100 };
101
102 struct Batch
103 {
105 std::string source;
107 std::string destination;
108
110 std::vector<BatchSourceDestination> sources;
111
113 std::vector<BatchCommand> commands;
114
115 std::vector<BatchCommand> post;
116
117 PropertyStore properties;
118 };
119
120 Batch readBatch(Context * context, const StringView & path)
121 {
122 ExtensionRegistry::loadExtensionModule("Cogs.Core.Extensions.Editor");
123
124 auto document = parseJson(context, path, JsonParseFlags::NoCachedContent);
125
126 if (!document.IsObject()) {
127 return {};
128 }
129
130 Batch batch;
131
132 if (document.GetObject().HasMember("extensions")) {
133 auto ext = document.GetObject()["extensions"].GetArray();
134
135 for (auto & e : ext) {
136 ExtensionRegistry::loadExtensionModule(e.GetString());
137 }
138 }
139
140 if (document.GetObject().HasMember("destination")) {
141 batch.destination = document.GetObject()["destination"].GetString();
142 }
143
144 if (document.GetObject().HasMember("sourcedir")) {
145 const std::string sourcedir = document.GetObject()["sourcedir"].GetString();
146
147 std::string pattern;
148 if (document.GetObject().HasMember("match")) {
149 pattern = document.GetObject()["match"].GetString();
150 }
151 if (pattern.empty()) pattern = "*.*";
152
153 for (auto & entry : fs::recursive_directory_iterator(sourcedir)) {
154 auto name = entry.path().string();
155 if (IO::isFile(name)) {
156 auto subdirname = IO::parentPath(name);
157 auto filename = IO::fileName(name);
158 if (matchFilename(filename, pattern)) {
159 // "subdirname" contains "sourcedir" in the beginning of its path.
160 // Here we replace that part with a destination (base) directory.
161 size_t subDirOffset = sourcedir.length();
162 if (subdirname.length() > sourcedir.length()) subDirOffset++; // deal with separators
163 std::string dest = IO::combine(batch.destination, subdirname.substr(subDirOffset));
164 batch.sources.emplace_back(name, dest);
165 }
166 }
167 }
168 }
169 else if (!document.GetObject().HasMember("source")) {
170 LOG_ERROR(logger, "%.*s: Missing required 'source' member.", StringViewFormat(path));
171 }
172 else if (!document.GetObject()["source"].IsString()) {
173 LOG_ERROR(logger, "%.*s: Member 'source' is not a string.", StringViewFormat(path));
174 }
175 else {
176 batch.source = document.GetObject()["source"].GetString();
177
178 if (batch.source.find('*') != std::string::npos) {
179 auto directory = IO::parentPath(batch.source);
180 auto pattern = IO::fileName(batch.source);
181 auto extension = IO::extension(pattern);
182 auto stem = IO::stem(batch.source);
183
184 for (auto & entry : fs::directory_iterator(directory)) {
185 auto name = entry.path().string();
186 if (matchFilename(name, pattern))
187 batch.sources.emplace_back(name, batch.destination);
188 }
189 }
190 else {
191 batch.sources = { {batch.source, batch.destination} };
192 }
193 }
194
195 if (!document.HasMember("batch")) {
196 LOG_ERROR(logger, "%.*s: Missing required 'batch' member.", StringViewFormat(path));
197 }
198 else if (!document.GetObject()["batch"].IsArray()) {
199 LOG_ERROR(logger, "%.*s: Member 'batch' is not an array.", StringViewFormat(path));
200 }
201 else {
202 auto jsonCommands = document.GetObject()["batch"].GetArray();
203
204 for (auto & jsonCommand : jsonCommands) {
205 auto & command = batch.commands.emplace_back();
206
207 if (jsonCommand.IsString()) {
208 command.type = jsonCommand.GetString();
209 }
210 else {
211 for (auto & o : jsonCommand.GetObject()) {
212 auto key = toKey(o.name);
213
214 if (key == "type") {
215 command.type = toString(o.value);
216 }
217 else {
218 auto & val = command.values.emplace_back();
219 val.key = toString(o.name);
220
221 if (o.value.IsString()) {
222 parseStringValue(toKey(o.value), val);
223 }
224 else if (o.value.IsNumber()) {
225 val.type = ParsedDataType::Float;
226 val.floatValue = o.value.GetFloat();
227 }
228 else if (o.value.IsBool()) {
229 val.type = ParsedDataType::Bool;
230 val.boolValue = o.value.GetBool();
231 }
232 }
233 }
234 }
235 }
236
237 if (document.GetObject().HasMember("properties")) {
238 auto & propertiesValue = document.GetObject()["properties"];
239 auto & properties = batch.properties;
240
241 for (auto & c : propertiesValue.GetObject()) {
242 auto propName = toKey(c.name);
243
244 if (c.value.IsBool()) {
245 properties.addProperty(propName, c.value.GetBool());
246 } else if (c.value.IsString()) {
247 properties.addProperty(propName, toKey(c.value));
248 } else if (c.value.IsInt()) {
249 properties.addProperty(propName, c.value.GetInt());
250 } else if (c.value.IsFloat()) {
251 properties.addProperty(propName, c.value.GetFloat());
252 } else if (c.value.IsArray()) {
253 auto valueArray = c.value.GetArray();
254
255 if (valueArray.Empty()) {
256 LOG_WARNING(logger, "Empty array in property \"%.*s\"", StringViewFormat(propName));
257 }
258
259 std::vector<float> floats(valueArray.Size());
260 for (uint32_t i = 0; i < valueArray.Size(); ++i) {
261 floats[i] = valueArray[i].GetFloat();
262 }
263
264 properties.addProperty(propName, std::span(floats));
265 }
266 else if (c.value.IsObject()) {
267 LOG_WARNING(logger, "Converting JSON Object to JSON string \"%.*s\"", StringViewFormat(propName));
268 StringBuffer sb;
269 Writer<StringBuffer> writer(sb);
270 c.value.Accept(writer);
271 properties.addProperty(propName, sb.GetString());
272 }
273 else {
274 auto name = toKey(c.name);
275 LOG_WARNING(logger, "properties: Unsupported property type for key \"%.*s\".", StringViewFormat(name));
276 }
277 }
278 }
279
280 auto postCommands = document.GetObject()["post"].GetArray();
281
282 for (auto & jsonCommand : postCommands) {
283 auto & command = batch.post.emplace_back();
284
285 if (jsonCommand.IsString()) {
286 command.type = jsonCommand.GetString();
287 } else {
288 for (auto & o : jsonCommand.GetObject()) {
289 auto key = toKey(o.name);
290
291 if (key == "type") {
292 command.type = toString(o.value);
293 } else if (key == "properties") {
294 auto & properties = command.properties;
295
296 for (auto & c : o.value.GetObject()) {
297 auto propName = toKey(c.name);
298
299 if (c.value.IsBool()) {
300 properties.addProperty(propName, c.value.GetBool());
301 } else if (c.value.IsString()) {
302 properties.addProperty(propName, toKey(c.value));
303 } else if (c.value.IsInt()) {
304 properties.addProperty(propName, c.value.GetInt());
305 } else if (c.value.IsFloat()) {
306 properties.addProperty(propName, c.value.GetFloat());
307 } else if (c.value.IsArray()) {
308 auto valueArray = c.value.GetArray();
309
310 if (valueArray.Empty()) {
311 LOG_WARNING(logger, "Empty array in property \"%.*s\"", StringViewFormat(propName));
312 }
313
314 std::vector<float> floats(valueArray.Size());
315 for (uint32_t i = 0; i < valueArray.Size(); ++i) {
316 floats[i] = valueArray[i].GetFloat();
317 }
318
319 properties.addProperty(propName, std::span(floats));
320 }
321 else if (c.value.IsObject()) {
322 LOG_WARNING(logger, "Converting JSON Object to JSON string \"%.*s\"", StringViewFormat(propName));
323 StringBuffer sb;
324 Writer<StringBuffer> writer(sb);
325 c.value.Accept(writer);
326 properties.addProperty(propName, sb.GetString());
327 }
328 else {
329 auto name = toKey(c.name);
330 LOG_WARNING(logger, "post.properties: Unsupported property type for key \"%.*s\".", StringViewFormat(name));
331 }
332 }
333 }
334 }
335 }
336 }
337 }
338
339 return batch;
340 }
341
342 void runFrames(Context * context, size_t numFrames, bool clear, bool wait)
343 {
344 for (size_t i = 0; i < numFrames; ++i) {
345 context->assetManager->processLoading();
346 context->modelManager->processLoading();
347 context->textureManager->processLoading();
348
349 if (wait) {
350 context->taskManager->waitAll();
351 }
352
353 context->assetManager->processSwapping();
354 context->modelManager->processSwapping();
355 context->meshManager->processSwapping();
356 context->materialInstanceManager->processSwapping();
357 context->textureManager->processSwapping();
358
359 context->assetManager->processDeletion();
360 context->modelManager->processDeletion();
361 context->meshManager->processDeletion();
362 context->materialInstanceManager->processDeletion();
363 context->textureManager->processDeletion();
364
365 context->modelSystem->update(context);
366 context->modelSystem->postUpdate(context);
367
368 context->renderSystem->update(context);
369
370 if (clear) {
371 context->assetManager->clearUpdated();
372 context->modelManager->clearUpdated();
373 context->meshManager->clearUpdated();
374 context->textureManager->clearUpdated();
375 context->materialInstanceManager->clearUpdated();
376 }
377 }
378 }
379
380 template<typename T>
381 void runCommand(Batch & /*batch*/, BatchCommand & command, Editor * editor, EntityId id)
382 {
383 T exp(editor->getState(), { id });
384 exp.options = command.values;
385
386 exp.apply();
387 }
388}
389
390void Cogs::Core::runBatch(Context * context, Editor * editor, const StringView & path)
391{
392 LOG_DEBUG(logger, "Processing batch file %.*s", StringViewFormat(path));
393 Cogs::Core::Batch batch = readBatch(context, path);
394
395 EditorState* state = editor->getState();
396
397 std::string filePath(path);
398
399 state->clearSelection();
400 state->fileName = IO::fileName(filePath);
401 state->directory = IO::parentPath(filePath);
402
403 if (batch.sources.empty()) {
404 LOG_ERROR(logger, "Batch sources empty.");
405 } else {
406 LOG_TRACE(logger, "Processing %s sources:", std::to_string(batch.sources.size()).c_str());
407 //for (auto & value : batch.sources) {
408 // auto source = value.source;
409 // auto destination = value.destination;
410 // LOG_TRACE(logger, " %s", source.c_str());
411 //}
412 }
413
414 if (batch.commands.size()) {
415 for (auto & value : batch.sources) {
416 std::string source = value.source;
417 std::string destination = value.destination;
418 std::string fileName = IO::fileName(source);
419 std::string extension = IO::extension(source);
420 std::string stem = IO::stem(fileName);
421 LOG_TRACE(logger, "Process commands for: %s, Dest:%s", source.c_str(), destination.c_str());
422
423 auto root = context->store->createEntity("Root", "ModelEntity");
424 auto modelComponent = root->getComponent<ModelComponent>();
425 modelComponent->model = context->modelManager->loadModel(source, NoResourceId, ModelLoadFlags::None);
426 // let's make sure all resources are loaded before we proceed
427 float progress = 0.f;
428 for (size_t i = 0; i < 10; ++i) { // three passes seem to be enough in practice, just adding some margin here
429 runFrames(context, 1, false);
430 // note that this won't get up to 1 if a texture (and thus material) is missing!
431 progress = context->modelSystem->getLoadProgress(modelComponent);
432 if (progress == 1.f) break;
433 context->engine->preRender();
434 }
435 if (progress < 1.f) {
436 LOG_WARNING(logger, "Skipped %s due to missing resources.", fileName.c_str());
437 continue;
438 }
439 editor->apply<SelectCommand>(root->getId());
440
441 for (auto & command : batch.commands) {
442 auto timer = Cogs::Timer::startNew();
443
444 auto oldValues = command.values;
445
446 // Batch file destination:
447 if (!command.containsKey("destination")) {
448 auto & destValue = command.values.emplace_back();
449 destValue.key = "destination";
450 destValue.value = destination;
451 }
452
453 // Path to the input file
454 if (!command.containsKey("source")) {
455 auto & destValue = command.values.emplace_back();
456 destValue.key = "source";
457 destValue.value = source;
458 }
459
460 // Path to the destination in the batch script.
461 if (!command.containsKey("batchDestination")) {
462 auto & destValue = command.values.emplace_back();
463 destValue.key = "batchDestination";
464 destValue.value = batch.destination;
465 }
466
467 if (command.type == "Remap") {
468 runCommand<RemapMaterialCommand>(batch, command, editor, root->getId());
469 } else if (command.type == "Merge") {
470 runCommand<MergeCommand>(batch, command, editor, root->getId());
471 runFrames(context, 2);
472 } else if (command.type == "MergeMesh") {
473 runCommand<MergeMeshCommand>(batch, command, editor, root->getId());
474 runFrames(context, 2);
475 } else if (command.type == "Export") {
476
477 // Make sure bounding boxes etc. are updated
478 runFrames(context, 2, false);
479
480 if (!command.containsKey("stem")) {
481 auto & stemValue = command.values.emplace_back();
482 stemValue.key = "stem";
483 stemValue.value = stem;
484 }
485
486 if (!command.containsKey("extension")) {
487 auto & extensionValue = command.values.emplace_back();
488 extensionValue.key = "extension";
489 extensionValue.value = extension;
490 }
491
492 runCommand<ExportCommand>(batch, command, editor, root->getId());
493
494 } else if (command.type == "DumpStats") {
495 runCommand<DumpStatsCommand>(batch, command, editor, root->getId());
496 } else if (command.type == "UniqueVertices") {
497 UniqueVerticesCommand exp(editor->getState());
498 exp.options = command.values;
499 exp.apply();
500 } else if (command.type == "GenerateNormals") {
501 GenerateNormalsCommand exp(editor->getState());
502 exp.options = command.values;
503 exp.apply();
504 } else if (command.type == "Select") {
505 // Apply select command. Parameter: "entityId" = EntityId to select.
506 for (auto & option : command.values) {
507 if (option.key == "entityId") {
508 EntityId id = std::stoull(option.value.c_str());
509 SelectCommand select(editor->getState(), id);
510 select.apply();
511 }
512 }
513 } else if (command.type == "Exit") {
514 (*context->variables)["editor.enabled"] = "false";
515 }
516
517 else {
518 bool found = false;
519 for (auto &cmd : Editor::getExtensionCommands()) {
520 if (cmd.key == command.type) {
521
522 // Ensure state updated. TBD: Allow command to configure.
523 runFrames(context, 2);
524 std::unique_ptr<EditorCommand> exp(cmd.creator(editor->getState()));
525 exp->options = command.values;
526 exp->apply();
527 found = true;
528 }
529 }
530
531 if (!found)
532 LOG_ERROR(logger, "Unknown command: %s", command.type.c_str());
533 }
534
535 command.values = oldValues;
536
537 LOG_TRACE(logger, "%s: %f seconds.", command.type.c_str(), timer.elapsedSeconds());
538 }
539
540 editor->getState()->clearSelection();
541
542 context->store->destroyEntity(root->getId());
543 root = {};
544
545 runFrames(context, 2);
546 }
547 }
548
549 auto getExtensionCommand = [&](const StringView & key) -> EditorCommand*
550 {
551 auto & commands = Editor::getExtensionCommands();
552
553 for (auto & command : commands) {
554 if (key == command.key) {
555 return command.creator(editor->getState());
556 }
557 }
558
559 LOG_ERROR(logger, "Missing extension command %.*s.", StringViewFormat(key));
560
561 return nullptr;
562 };
563
564 if (batch.post.size()) {
565 for (auto & command : batch.post) {
566
567 if (command.type == "Exit") {
568 (*context->variables)["editor.enabled"] = "false";
569 }
570 else {
571
572 EditorCommand* editorCommand = getExtensionCommand(command.type);
573 if (editorCommand == nullptr) {
574 LOG_ERROR(logger, "Unrecognized command '%s'", command.type.c_str());
575 continue;
576 }
577
578 PostCommand* postCommand = dynamic_cast<PostCommand*>(editorCommand);
579 if (postCommand == nullptr) {
580 LOG_ERROR(logger, "Command '%s' is not a post-capable command", command.type.c_str());
581 continue;
582 }
583
584 postCommand->properties.copyProperties(batch.properties, 0, batch.properties.size());
585 postCommand->properties.copyProperties(command.properties, 0, command.properties.size());
586 postCommand->properties.addProperty("source", batch.source);
587 postCommand->properties.addProperty("destination", batch.destination);
588
589 postCommand->applyPost();
590 }
591 }
592 }
593
594 editor->getState()->commandStates.clear();
595}
596
597#else
598void Cogs::Core::runBatch(Context * context, Editor * editor, const StringView & path)
599{
600
601}
602
603void Cogs::Core::runFrames(Context *, size_t, bool, bool)
604{
605
606}
607#endif
Cogs::Core::Context
A Context instance contains all the services, systems and runtime components needed to use Cogs.
Definition: Context.h:83
Cogs::Core::ExtensionRegistry::loadExtensionModule
static const void * loadExtensionModule(const std::string &path, void **modulehandle=nullptr, ExtensionModuleLoadResult *result=nullptr)
Load the extension module with the given name.
Definition: ExtensionRegistry.cpp:127
Cogs::Logging::Log
Log implementation class.
Definition: LogManager.h:140
Cogs::StringView
Provides a weakly referenced view over the contents of a string.
Definition: StringView.h:24
Cogs::Core
Contains the Engine, Renderer, resource managers and other systems needed to run Cogs....
Definition: ComponentFunctions.h:10
Cogs::Logging::getLogger
constexpr Log getLogger(const char(&name)[LEN]) noexcept
Definition: LogManager.h:181
Cogs::Core::BatchSourceDestination
Pair of source/destination paths. Doing recursive processing adds directories to the destination.
Definition: Batch.cpp:87
Cogs::Core::BatchSourceDestination::destination
std::string destination
Matching destination directory. May vary for recursive source processing.
Definition: Batch.cpp:91
Cogs::Core::BatchSourceDestination::source
std::string source
Batch source file.
Definition: Batch.cpp:89
Cogs::Core::Batch::destination
std::string destination
Destination in batch script.
Definition: Batch.cpp:107
Cogs::Core::Batch::source
std::string source
Path of the batch script.
Definition: Batch.cpp:105
Cogs::Core::Batch::sources
std::vector< BatchSourceDestination > sources
All source/destination pairs.
Definition: Batch.cpp:110
Cogs::Core::Batch::commands
std::vector< BatchCommand > commands
list of commands to execute for each file.
Definition: Batch.cpp:113
Generated by doxygen 1.9.6