From df1fb56d7edcaabc9d17fc77c37f865b03903f66 Mon Sep 17 00:00:00 2001 From: coco875 Date: Mon, 4 Mar 2024 17:00:42 +0100 Subject: [PATCH] change data format and simplify the thread queue --- tools/3d_models.json | 402 +++++++++++++--------- tools/blender_extension/blender_export.py | 3 +- tools/extract_models.py | 65 ++-- 3 files changed, 275 insertions(+), 195 deletions(-) diff --git a/tools/3d_models.json b/tools/3d_models.json index a62a8886d..4286626cb 100644 --- a/tools/3d_models.json +++ b/tools/3d_models.json @@ -1,167 +1,235 @@ -{ - "courses/dks_jungle_parkway/course_data.inc.c": [ - "d_course_dks_jungle_parkway_dl_E730", - "d_course_dks_jungle_parkway_dl_E058", - "d_course_dks_jungle_parkway_dl_FC28", - "d_course_dks_jungle_parkway_dl_kiwano_fruit", - "d_course_dks_jungle_parkway_dl_palm_tree", - "d_course_dks_jungle_parkway_dl_13878", - "d_course_dks_jungle_parkway_dl_tree3", - "d_course_dks_jungle_parkway_dl_tree2", - "d_course_dks_jungle_parkway_dl_tree1" - ], - "courses/banshee_boardwalk/course_data.inc.c": [ - "d_course_banshee_boardwalk_dl_B278", - "d_course_banshee_boardwalk_dl_trash_bin", - "d_course_banshee_boardwalk_dl_cheep_cheep" - ], - "courses/bowsers_castle/course_data.inc.c": [ - "d_course_bowsers_castle_dl_thwomp", - "d_course_bowsers_castle_dl_bush" - ], - "courses/choco_mountain/course_data.inc.c":[ - "d_course_choco_mountain_dl_falling_rock" - ], - "courses/frappe_snowland/course_data.inc.c":[ - "d_course_frappe_snowland_dl_tree" - ], - "courses/kalimari_desert/course_data.inc.c":[ - "d_course_kalimari_desert_dl_cactus1", - "d_course_kalimari_desert_dl_cactus2", - "d_course_kalimari_desert_dl_cactus3", - "d_course_kalimari_desert_dl_crossing_right_active", - "d_course_kalimari_desert_dl_crossing_left_active", - "d_course_kalimari_desert_dl_crossing_both_inactive", - "d_course_kalimari_desert_dl_1B978", - "d_course_kalimari_desert_dl_1C0F0", - "d_course_kalimari_desert_dl_1D160", - "d_course_kalimari_desert_dl_1D670", - "d_course_kalimari_desert_dl_1E480", - "d_course_kalimari_desert_dl_1E910", - "d_course_kalimari_desert_dl_1F228", - "d_course_kalimari_desert_dl_1F708", - "d_course_kalimari_desert_dl_1FAF8", - "d_course_kalimari_desert_dl_20A20", - "d_course_kalimari_desert_dl_21220", - "d_course_kalimari_desert_dl_21550", - "d_course_kalimari_desert_dl_21A80", - "d_course_kalimari_desert_dl_21C90", - "d_course_kalimari_desert_dl_22D28", - "d_course_kalimari_desert_dl_22D70", - "d_course_kalimari_desert_dl_22DB8" - ], - "courses/koopa_troopa_beach/course_data.inc.c":[ - "d_course_koopa_troopa_beach_dl_16630", - "d_course_koopa_troopa_beach_dl_tree_top1", - "d_course_koopa_troopa_beach_dl_tree_trunk1", - "d_course_koopa_troopa_beach_dl_tree_top2", - "d_course_koopa_troopa_beach_dl_tree_trunk2", - "d_course_koopa_troopa_beach_dl_tree_top3", - "d_course_koopa_troopa_beach_dl_tree_trunk3" - ], - "courses/luigi_raceway/course_data.inc.c": [ - "d_course_luigi_raceway_dl_F650", - "d_course_luigi_raceway_dl_F960", - "d_course_luigi_raceway_dl_FA20", - "d_course_luigi_raceway_dl_FBE0", - "d_course_luigi_raceway_dl_FC70" - ], - "courses/mario_raceway/course_data.inc.c": [ - "d_course_mario_raceway_dl_piranha_plant", - "d_course_mario_raceway_dl_tree", - "d_course_mario_raceway_dl_sign" - ], - "courses/moo_moo_farm/course_data.inc.c": [ - "d_course_moo_moo_farm_dl_cow1", - "d_course_moo_moo_farm_dl_cow2", - "d_course_moo_moo_farm_dl_cow3", - "d_course_moo_moo_farm_dl_cow4", - "d_course_moo_moo_farm_dl_cow5", - "d_course_moo_moo_farm_dl_tree" - ], - "courses/rainbow_road/course_data.inc.c": [ - "d_course_rainbow_road_dl_151A8", - "d_course_rainbow_road_dl_15550", - "d_course_rainbow_road_dl_158C0", - "d_course_rainbow_road_dl_15C68", - "d_course_rainbow_road_dl_15F18" - ], - "courses/royal_raceway/course_data.inc.c": [ - "d_course_royal_raceway_dl_tree", - "d_course_royal_raceway_dl_castle_tree", - "d_course_royal_raceway_dl_piranha_plant" - ], - "courses/sherbet_land/course_data.inc.c": [ - "d_course_sherbet_land_dl_ice_block", - "d_course_sherbet_land_dl_8E00" - ], - "courses/toads_turnpike/course_data.inc.c": [ - "d_course_toads_turnpike_dl_19020", - "d_course_toads_turnpike_dl_19518", - "d_course_toads_turnpike_dl_19DF0", - "d_course_toads_turnpike_dl_1A068", - "d_course_toads_turnpike_dl_1A5F8", - "d_course_toads_turnpike_dl_1A6C8", - "d_course_toads_turnpike_dl_1B778", - "d_course_toads_turnpike_dl_1BE48", - "d_course_toads_turnpike_dl_1C700", - "d_course_toads_turnpike_dl_1CAA8", - "d_course_toads_turnpike_dl_1CE70", - "d_course_toads_turnpike_dl_1D018", - "d_course_toads_turnpike_dl_1E458", - "d_course_toads_turnpike_dl_1EB48", - "d_course_toads_turnpike_dl_1F9D0", - "d_course_toads_turnpike_dl_20008", - "d_course_toads_turnpike_dl_20510", - "d_course_toads_turnpike_dl_205A8", - "d_course_toads_turnpike_dl_21780", - "d_course_toads_turnpike_dl_21E28", - "d_course_toads_turnpike_dl_22BA0", - "d_course_toads_turnpike_dl_23078", - "d_course_toads_turnpike_dl_237F8", - "d_course_toads_turnpike_dl_23848", - "d_course_toads_turnpike_dl_23858", - "d_course_toads_turnpike_dl_238A0", - "d_course_toads_turnpike_dl_238E8" - ], - "courses/wario_stadium/course_data.inc.c":[ - "d_course_wario_stadium_dl_sign" - ], - "courses/yoshi_valley/course_data.inc.c":[ - "d_course_yoshi_valley_dl_142C0", - "d_course_yoshi_valley_dl_143C8", - "d_course_yoshi_valley_dl_144A8", - "d_course_yoshi_valley_dl_14500", - "d_course_yoshi_valley_dl_14558", - "d_course_yoshi_valley_dl_145D0", - "d_course_yoshi_valley_dl_tree", - "d_course_yoshi_valley_dl_16D70", - "d_course_yoshi_valley_dl_egg_lod0" - ], - "src/data/common_textures.inc.c": [ - "D_0D001B90", - "D_0D001BD8", - "D_0D001C20", - "D_0D001C88", - "D_0D002EE8", - "common_model_fake_itembox", - "itemBoxQuestionMarkModel", - "D_0D0030F8", - "D_0D003128", - "D_0D003158", - "D_0D003188", - "D_0D0031B8", - "D_0D0031E8", - "D_0D003218", - "D_0D003248", - "D_0D003288", - "common_model_banana", - "common_model_flat_banana", - "D_0D0052B8", - "D_0D005338", - "D_0D005368", - "D_0D007B20", - "D_0D007B98", - "D_0D007C10" - ] -} \ No newline at end of file +[ + { + "file":"courses/banshee_boardwalk/course_data.inc.c", + "out_dir": "banshee_boardwalk", + "models": [ + "d_course_banshee_boardwalk_dl_B278", + "d_course_banshee_boardwalk_dl_trash_bin", + "d_course_banshee_boardwalk_dl_cheep_cheep" + ] + }, + { + "file": "courses/bowsers_castle/course_data.inc.c", + "out_dir": "bowsers_castle", + "models": [ + "d_course_bowsers_castle_dl_thwomp", + "d_course_bowsers_castle_dl_bush" + ] + }, + { + "file": "courses/choco_mountain/course_data.inc.c", + "out_dir": "choco_mountain", + "models": [ + "d_course_choco_mountain_dl_falling_rock" + ] + }, + { + "file": "courses/dks_jungle_parkway/course_data.inc.c", + "out_dir": "dks_jungle_parkway", + "models":[ + "d_course_dks_jungle_parkway_dl_E730", + "d_course_dks_jungle_parkway_dl_E058", + "d_course_dks_jungle_parkway_dl_FC28", + "d_course_dks_jungle_parkway_dl_kiwano_fruit", + "d_course_dks_jungle_parkway_dl_palm_tree", + "d_course_dks_jungle_parkway_dl_13878", + "d_course_dks_jungle_parkway_dl_tree3", + "d_course_dks_jungle_parkway_dl_tree2", + "d_course_dks_jungle_parkway_dl_tree1" + ] + }, + { + "file":"courses/frappe_snowland/course_data.inc.c", + "out_dir": "frappe_snowland", + "models": [ + "d_course_frappe_snowland_dl_tree" + ] + }, + { + "file": "courses/kalimari_desert/course_data.inc.c", + "out_dir": "kalimari_desert", + "models": [ + "d_course_kalimari_desert_dl_cactus1", + "d_course_kalimari_desert_dl_cactus2", + "d_course_kalimari_desert_dl_cactus3", + "d_course_kalimari_desert_dl_crossing_right_active", + "d_course_kalimari_desert_dl_crossing_left_active", + "d_course_kalimari_desert_dl_crossing_both_inactive", + "d_course_kalimari_desert_dl_1B978", + "d_course_kalimari_desert_dl_1C0F0", + "d_course_kalimari_desert_dl_1D160", + "d_course_kalimari_desert_dl_1D670", + "d_course_kalimari_desert_dl_1E480", + "d_course_kalimari_desert_dl_1E910", + "d_course_kalimari_desert_dl_1F228", + "d_course_kalimari_desert_dl_1F708", + "d_course_kalimari_desert_dl_1FAF8", + "d_course_kalimari_desert_dl_20A20", + "d_course_kalimari_desert_dl_21220", + "d_course_kalimari_desert_dl_21550", + "d_course_kalimari_desert_dl_21A80", + "d_course_kalimari_desert_dl_21C90", + "d_course_kalimari_desert_dl_22D28", + "d_course_kalimari_desert_dl_22D70", + "d_course_kalimari_desert_dl_22DB8" + ] + }, + { + "file": "courses/koopa_troopa_beach/course_data.inc.c", + "out_dir": "koopa_troopa_beach", + "models": [ + "d_course_koopa_troopa_beach_dl_16630", + "d_course_koopa_troopa_beach_dl_tree_top1", + "d_course_koopa_troopa_beach_dl_tree_trunk1", + "d_course_koopa_troopa_beach_dl_tree_top2", + "d_course_koopa_troopa_beach_dl_tree_trunk2", + "d_course_koopa_troopa_beach_dl_tree_top3", + "d_course_koopa_troopa_beach_dl_tree_trunk3" + ] + }, + { + "file": "courses/luigi_raceway/course_data.inc.c", + "out_dir": "luigi_raceway", + "models": [ + "d_course_luigi_raceway_dl_F650", + "d_course_luigi_raceway_dl_F960", + "d_course_luigi_raceway_dl_FA20", + "d_course_luigi_raceway_dl_FBE0", + "d_course_luigi_raceway_dl_FC70" + ] + }, + { + "file": "courses/mario_raceway/course_data.inc.c", + "out_dir": "mario_raceway", + "models": [ + "d_course_mario_raceway_dl_piranha_plant", + "d_course_mario_raceway_dl_tree", + "d_course_mario_raceway_dl_sign" + ] + }, + { + "file": "courses/moo_moo_farm/course_data.inc.c", + "out_dir": "moo_moo_farm", + "models": [ + "d_course_moo_moo_farm_dl_cow1", + "d_course_moo_moo_farm_dl_cow2", + "d_course_moo_moo_farm_dl_cow3", + "d_course_moo_moo_farm_dl_cow4", + "d_course_moo_moo_farm_dl_cow5", + "d_course_moo_moo_farm_dl_tree" + ] + }, + { + "file": "courses/rainbow_road/course_data.inc.c", + "out_dir": "rainbow_road", + "models": [ + "d_course_rainbow_road_dl_151A8", + "d_course_rainbow_road_dl_15550", + "d_course_rainbow_road_dl_158C0", + "d_course_rainbow_road_dl_15C68", + "d_course_rainbow_road_dl_15F18" + ] + }, + { + "file": "courses/royal_raceway/course_data.inc.c", + "out_dir": "royal_raceway", + "models": [ + "d_course_royal_raceway_dl_tree", + "d_course_royal_raceway_dl_castle_tree", + "d_course_royal_raceway_dl_piranha_plant" + ] + }, + { + "file": "courses/sherbet_land/course_data.inc.c", + "out_dir": "sherbet_land", + "models": [ + "d_course_sherbet_land_dl_ice_block", + "d_course_sherbet_land_dl_8E00" + ] + }, + { + "file": "courses/toads_turnpike/course_data.inc.c", + "out_dir": "toads_turnpike", + "models": [ + "d_course_toads_turnpike_dl_19020", + "d_course_toads_turnpike_dl_19518", + "d_course_toads_turnpike_dl_19DF0", + "d_course_toads_turnpike_dl_1A068", + "d_course_toads_turnpike_dl_1A5F8", + "d_course_toads_turnpike_dl_1A6C8", + "d_course_toads_turnpike_dl_1B778", + "d_course_toads_turnpike_dl_1BE48", + "d_course_toads_turnpike_dl_1C700", + "d_course_toads_turnpike_dl_1CAA8", + "d_course_toads_turnpike_dl_1CE70", + "d_course_toads_turnpike_dl_1D018", + "d_course_toads_turnpike_dl_1E458", + "d_course_toads_turnpike_dl_1EB48", + "d_course_toads_turnpike_dl_1F9D0", + "d_course_toads_turnpike_dl_20008", + "d_course_toads_turnpike_dl_20510", + "d_course_toads_turnpike_dl_205A8", + "d_course_toads_turnpike_dl_21780", + "d_course_toads_turnpike_dl_21E28", + "d_course_toads_turnpike_dl_22BA0", + "d_course_toads_turnpike_dl_23078", + "d_course_toads_turnpike_dl_237F8", + "d_course_toads_turnpike_dl_23848", + "d_course_toads_turnpike_dl_23858", + "d_course_toads_turnpike_dl_238A0", + "d_course_toads_turnpike_dl_238E8" + ] + }, + { + "file": "courses/wario_stadium/course_data.inc.c", + "out_dir": "wario_stadium", + "models": [ + "d_course_wario_stadium_dl_sign" + ] + }, + { + "file": "courses/yoshi_valley/course_data.inc.c", + "out_dir": "yoshi_valley", + "models": [ + "d_course_yoshi_valley_dl_142C0", + "d_course_yoshi_valley_dl_143C8", + "d_course_yoshi_valley_dl_144A8", + "d_course_yoshi_valley_dl_14500", + "d_course_yoshi_valley_dl_14558", + "d_course_yoshi_valley_dl_145D0", + "d_course_yoshi_valley_dl_tree", + "d_course_yoshi_valley_dl_16D70", + "d_course_yoshi_valley_dl_egg_lod0" + ] + }, + { + "file": "src/data/common_textures.inc.c", + "out_dir": "common_textures", + "models": [ + "D_0D001B90", + "D_0D001BD8", + "D_0D001C20", + "D_0D001C88", + "D_0D002EE8", + "common_model_fake_itembox", + "itemBoxQuestionMarkModel", + "D_0D0030F8", + "D_0D003128", + "D_0D003158", + "D_0D003188", + "D_0D0031B8", + "D_0D0031E8", + "D_0D003218", + "D_0D003248", + "D_0D003288", + "common_model_banana", + "common_model_flat_banana", + "D_0D0052B8", + "D_0D005338", + "D_0D005368", + "D_0D007B20", + "D_0D007B98", + "D_0D007C10" + ] + } +] \ No newline at end of file diff --git a/tools/blender_extension/blender_export.py b/tools/blender_extension/blender_export.py index ede2b04e8..0aa538c18 100644 --- a/tools/blender_extension/blender_export.py +++ b/tools/blender_extension/blender_export.py @@ -30,6 +30,7 @@ drawLayer = bpy.context.scene.DLImportDrawLayer file = argv[0] obj = argv[1] +out_file = argv[2] clean_scene() if bpy.context.mode != "OBJECT": @@ -48,4 +49,4 @@ importMeshC( drawLayer, F3DContext(get_F3D_GBI(), basePath, createF3DMat(None)), ) -bpy.ops.wm.save_as_mainfile(filepath="models/"+obj+".blend") \ No newline at end of file +bpy.ops.wm.save_as_mainfile(filepath=out_file) \ No newline at end of file diff --git a/tools/extract_models.py b/tools/extract_models.py index 69e527430..8f252b2aa 100644 --- a/tools/extract_models.py +++ b/tools/extract_models.py @@ -11,7 +11,7 @@ args = parser.parse_args() if not os.path.exists("models"): os.makedirs("models") -models = json.load(open("tools/3d_models.json")) +models:list[dict[str, str|list[str]]] = json.load(open("tools/3d_models.json")) if os.name == "nt": blender = "c:/Program Files/Blender Foundation/Blender 4.0/blender.exe" @@ -27,42 +27,53 @@ args_blender = [ failed = [] -threads:list[tuple[threading.Thread, str]] = [] - -for file in models: - for obj in models[file]: - if os.path.exists("models/"+obj+".blend"): - continue - while len(threads) >= args.threads: - for i, (thread, out) in enumerate(threads): +class QueueThread: + def __init__(self, max_threads:int, check): + self.threads:list[tuple[threading.Thread, str]] = [] + self.max_threads = max_threads + self.check = check + def check_available(self): + while len(self.threads) >= args.threads: + for i, (thread, out) in enumerate(self.threads): if not thread.is_alive(): thread.join() - if os.path.exists(out): - print(f"Exported {out}") - else: - print(f"Failed to export {out}") + if not self.check(out): failed.append(out) - del threads[i] + del self.threads[i] break - print(f"Exporting {obj} from {file}") + + def append(self, thread:threading.Thread, out:str): + self.check_available() + self.threads.append((thread, out)) + thread.start() + + def finish(self): + for thread, out in self.threads: + thread.join() + if not self.check(out): + failed.append(out) + +queue = QueueThread(args.threads, lambda out: os.path.exists(out)) + +for model in models: + file = model["file"] + out_dir = "models/"+model["out_dir"] if "out_dir" in model else "models" + if not os.path.exists(out_dir): + os.makedirs(out_dir) + for obj in model["models"]: + out_file = out_dir+"/"+obj+".blend" + if os.path.exists(out_file): + continue args_blender.append(file) args_blender.append(obj) + args_blender.append(out_file) thread = threading.Thread(target=lambda: subprocess.run([blender]+args_blender)) - thread.start() - threads.append(( - thread, - "models/"+obj+".blend" - )) + queue.append(thread, out_file) + args_blender.pop() args_blender.pop() args_blender.pop() -for thread, out in threads: - thread.join() - if os.path.exists(out): - print(f"Exported {out}") - else: - print(f"Failed to export {out}") - failed.append(out) +queue.finish() print("Failed to export:") print("\n".join(failed))