From 91ad7860b2d5924d08038d675174fefda225b54f Mon Sep 17 00:00:00 2001 From: octorock <79596758+octorock@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:58:25 +0100 Subject: [PATCH] Improve compilation time of asset_processor By splitting it up into object files and letting them be compiled in parallel. Also only rebuild the parts that changed. --- Makefile | 6 ++- tools/asset_processor/.gitignore | 1 + tools/asset_processor/Makefile | 21 ++++++-- tools/asset_processor/assets/asset.h | 6 +-- tools/asset_processor/assets/gfx.cpp | 9 ++-- tools/asset_processor/assets/midi.cpp | 4 +- tools/asset_processor/json_fwd.hpp | 78 +++++++++++++++++++++++++++ tools/asset_processor/util.cpp | 9 ---- tools/asset_processor/util.h | 4 +- tools/mid2agb/midi.h | 10 +--- 10 files changed, 115 insertions(+), 33 deletions(-) create mode 100644 tools/asset_processor/json_fwd.hpp diff --git a/Makefile b/Makefile index e9743138..b8296589 100644 --- a/Makefile +++ b/Makefile @@ -89,6 +89,7 @@ SCANINC := tools/scaninc/scaninc # TODO: use charmap? PREPROC := tools/preproc/preproc FIX := tools/gbafix/gbafix +ASSET_PROCESSOR := tools/asset_processor/asset_processor # Clear the default suffixes .SUFFIXES: @@ -168,13 +169,14 @@ compare: $(ROM) setup: $(TOOLDIRS) +# Automatically extract binary data build/extracted_assets_%: assets/assets.json assets/gfx.json assets/map.json assets/samples.json assets/sounds.json - tools/asset_processor/asset_processor extract $(GAME_VERSION) $(ASSET_BUILDDIR) + $(ASSET_PROCESSOR) extract $(GAME_VERSION) $(ASSET_BUILDDIR) touch $@ # Extract assets to human readable form extractassets: - tools/asset_processor/asset_processor convert $(GAME_VERSION) $(ASSET_BUILDDIR) + $(ASSET_PROCESSOR) convert $(GAME_VERSION) $(ASSET_BUILDDIR) $(TOOLDIRS): @$(MAKE) -C $@ diff --git a/tools/asset_processor/.gitignore b/tools/asset_processor/.gitignore index 183e753a..86d45694 100644 --- a/tools/asset_processor/.gitignore +++ b/tools/asset_processor/.gitignore @@ -1 +1,2 @@ asset_processor +build/ \ No newline at end of file diff --git a/tools/asset_processor/Makefile b/tools/asset_processor/Makefile index d5d0631a..98bf70a8 100644 --- a/tools/asset_processor/Makefile +++ b/tools/asset_processor/Makefile @@ -3,19 +3,32 @@ CXX = g++ CXXFLAGS = -O3 -Wall -Wextra -std=c++17 #CXXFLAGS += -g # debug +BUILD_FOLDER=build + SRCS = $(wildcard *.cpp) -HEADERS = $(wildcard *.h) SRCS += $(wildcard assets/*.cpp) -HEADERS += $(wildcard assets/*.h) + +OBJS := $(patsubst %.cpp,$(BUILD_FOLDER)/%.o,$(SRCS)) INCLUDES = -I./ +# Create build dirs +$(shell mkdir -p $(dir $(OBJS)) >/dev/null) + .PHONY: all clean all: asset_processor -asset_processor: $(SRCS) $(HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) -o asset_processor $(SRCS) -lstdc++fs +asset_processor: $(OBJS) + $(CXX) -o asset_processor $(OBJS) -lstdc++fs + +$(BUILD_FOLDER)/%.o: %.cpp + $(CXX) $(CXXFLAGS) $(INCLUDES) -c -o $@ $< clean: $(RM) asset_processor asset_processor.exe + $(RM) -r $(BUILD_FOLDER) + +# Automatic dependencies +CXXFLAGS += -MMD +-include $(OBJS:.o=.d) \ No newline at end of file diff --git a/tools/asset_processor/assets/asset.h b/tools/asset_processor/assets/asset.h index 7ccfb3e6..56289199 100644 --- a/tools/asset_processor/assets/asset.h +++ b/tools/asset_processor/assets/asset.h @@ -3,13 +3,13 @@ #include "util.h" #include -#include +#include #include #include class BaseAsset { public: - BaseAsset(std::filesystem::path path, int start, int size, nlohmann::json asset) + BaseAsset(std::filesystem::path path, int start, int size, const nlohmann::json& asset) : path(path), start(start), size(size), asset(asset) { } @@ -62,7 +62,7 @@ class BaseAsset { std::filesystem::path buildPath; int start; int size; - nlohmann::json asset; + const nlohmann::json& asset; private: virtual std::filesystem::path generateAssetPath() { diff --git a/tools/asset_processor/assets/gfx.cpp b/tools/asset_processor/assets/gfx.cpp index 956503af..8d6c11f1 100644 --- a/tools/asset_processor/assets/gfx.cpp +++ b/tools/asset_processor/assets/gfx.cpp @@ -1,5 +1,6 @@ #include "gfx.h" #include "util.h" +#include std::filesystem::path GfxAsset::generateAssetPath() { std::filesystem::path pngPath = this->path; @@ -18,9 +19,11 @@ void GfxAsset::convertToHumanReadable(const std::vector& baserom) { cmd.push_back(toolsPath / "gbagfx" / "gbagfx"); cmd.push_back(this->path); cmd.push_back(this->assetPath); - for (const auto& it : this->asset["options"].items()) { - cmd.push_back("-" + it.key()); - cmd.push_back(to_string(it.value())); + if (this->asset.contains("options")) { + for (const auto& it : this->asset["options"].items()) { + cmd.push_back("-" + it.key()); + cmd.push_back(to_string(it.value())); + } } check_call(cmd); } diff --git a/tools/asset_processor/assets/midi.cpp b/tools/asset_processor/assets/midi.cpp index 9b2c31e9..0e516067 100644 --- a/tools/asset_processor/assets/midi.cpp +++ b/tools/asset_processor/assets/midi.cpp @@ -1,10 +1,12 @@ #include "midi.h" -#include "main.h" #include "reader.h" #include "util.h" #include #include #include +#include + +extern std::string gBaseromPath; std::filesystem::path MidiAsset::generateAssetPath() { std::filesystem::path path = this->path; diff --git a/tools/asset_processor/json_fwd.hpp b/tools/asset_processor/json_fwd.hpp new file mode 100644 index 00000000..332227c1 --- /dev/null +++ b/tools/asset_processor/json_fwd.hpp @@ -0,0 +1,78 @@ +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; + +template +struct ordered_map; + +/*! +@brief ordered JSON class + +This type preserves the insertion order of object keys. + +@since version 3.9.0 +*/ +using ordered_json = basic_json; + +} // namespace nlohmann + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ diff --git a/tools/asset_processor/util.cpp b/tools/asset_processor/util.cpp index 6e8537b4..f6ac050f 100644 --- a/tools/asset_processor/util.cpp +++ b/tools/asset_processor/util.cpp @@ -18,12 +18,3 @@ void check_call(const std::vector& cmd) { std::exit(1); } } - -// https://github.com/nlohmann/json/issues/642#issuecomment-311937344 -std::string to_string(const nlohmann::json& j) { - if (j.type() == nlohmann::json::value_t::string) { - return j.get(); - } - - return j.dump(); -} diff --git a/tools/asset_processor/util.h b/tools/asset_processor/util.h index e49656b1..977c25ea 100644 --- a/tools/asset_processor/util.h +++ b/tools/asset_processor/util.h @@ -1,7 +1,7 @@ #ifndef UTIL_H #define UTIL_H -#include +#include #include #include #include @@ -20,8 +20,6 @@ typedef int64_t s64; void check_call(const std::vector& cmd); -std::string to_string(const nlohmann::json& j); - template std::string string_format(const std::string& format, Args... args) { int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1; // Extra space for '\0' if (size_s <= 0) { diff --git a/tools/mid2agb/midi.h b/tools/mid2agb/midi.h index c617ced2..28c98f8f 100755 --- a/tools/mid2agb/midi.h +++ b/tools/mid2agb/midi.h @@ -33,15 +33,9 @@ enum class EventType { EndOfTie = 0x01, Label = 0x11, - LoopEnd = 0x38,// To place it last if at the same time as other meta events, but before notes on the same frame + LoopEnd = 0x38, // To place it last if at the same time as other meta events, but before notes on the same frame LoopEndBegin = 0x13, - LoopBegin = 0x24, - // Original: 0x14 - // TODO sfx1AA wants a LoopBegin before a Volume Change -> < 0x22 - // bgmFestivalApproach wants a LoopBegin after a Tempo -> >0x19 -> 0x20 - // sfxSparkles wants a LoopBegin after a VOL --> NOT POSSIBLE - // bgmCuccoMinigame as well - // To place it last if at the same time as other meta events, but before notes on the same frame + LoopBegin = 0x24, // To place it last if at the same time as other meta events, but before notes on the same frame OriginalTimeSignature = 0x15, WholeNoteMark = 0x16, Pattern = 0x17,