change data format and simplify the thread queue

This commit is contained in:
coco875
2024-03-04 17:00:42 +01:00
parent 36d4e33070
commit df1fb56d7e
3 changed files with 275 additions and 195 deletions
+235 -167
View File
@@ -1,167 +1,235 @@
{
"courses/dks_jungle_parkway/course_data.inc.c": [
"d_course_dks_jungle_parkway_dl_E730",
"d_course_dks_jungle_parkway_dl_E058",
"d_course_dks_jungle_parkway_dl_FC28",
"d_course_dks_jungle_parkway_dl_kiwano_fruit",
"d_course_dks_jungle_parkway_dl_palm_tree",
"d_course_dks_jungle_parkway_dl_13878",
"d_course_dks_jungle_parkway_dl_tree3",
"d_course_dks_jungle_parkway_dl_tree2",
"d_course_dks_jungle_parkway_dl_tree1"
],
"courses/banshee_boardwalk/course_data.inc.c": [
"d_course_banshee_boardwalk_dl_B278",
"d_course_banshee_boardwalk_dl_trash_bin",
"d_course_banshee_boardwalk_dl_cheep_cheep"
],
"courses/bowsers_castle/course_data.inc.c": [
"d_course_bowsers_castle_dl_thwomp",
"d_course_bowsers_castle_dl_bush"
],
"courses/choco_mountain/course_data.inc.c":[
"d_course_choco_mountain_dl_falling_rock"
],
"courses/frappe_snowland/course_data.inc.c":[
"d_course_frappe_snowland_dl_tree"
],
"courses/kalimari_desert/course_data.inc.c":[
"d_course_kalimari_desert_dl_cactus1",
"d_course_kalimari_desert_dl_cactus2",
"d_course_kalimari_desert_dl_cactus3",
"d_course_kalimari_desert_dl_crossing_right_active",
"d_course_kalimari_desert_dl_crossing_left_active",
"d_course_kalimari_desert_dl_crossing_both_inactive",
"d_course_kalimari_desert_dl_1B978",
"d_course_kalimari_desert_dl_1C0F0",
"d_course_kalimari_desert_dl_1D160",
"d_course_kalimari_desert_dl_1D670",
"d_course_kalimari_desert_dl_1E480",
"d_course_kalimari_desert_dl_1E910",
"d_course_kalimari_desert_dl_1F228",
"d_course_kalimari_desert_dl_1F708",
"d_course_kalimari_desert_dl_1FAF8",
"d_course_kalimari_desert_dl_20A20",
"d_course_kalimari_desert_dl_21220",
"d_course_kalimari_desert_dl_21550",
"d_course_kalimari_desert_dl_21A80",
"d_course_kalimari_desert_dl_21C90",
"d_course_kalimari_desert_dl_22D28",
"d_course_kalimari_desert_dl_22D70",
"d_course_kalimari_desert_dl_22DB8"
],
"courses/koopa_troopa_beach/course_data.inc.c":[
"d_course_koopa_troopa_beach_dl_16630",
"d_course_koopa_troopa_beach_dl_tree_top1",
"d_course_koopa_troopa_beach_dl_tree_trunk1",
"d_course_koopa_troopa_beach_dl_tree_top2",
"d_course_koopa_troopa_beach_dl_tree_trunk2",
"d_course_koopa_troopa_beach_dl_tree_top3",
"d_course_koopa_troopa_beach_dl_tree_trunk3"
],
"courses/luigi_raceway/course_data.inc.c": [
"d_course_luigi_raceway_dl_F650",
"d_course_luigi_raceway_dl_F960",
"d_course_luigi_raceway_dl_FA20",
"d_course_luigi_raceway_dl_FBE0",
"d_course_luigi_raceway_dl_FC70"
],
"courses/mario_raceway/course_data.inc.c": [
"d_course_mario_raceway_dl_piranha_plant",
"d_course_mario_raceway_dl_tree",
"d_course_mario_raceway_dl_sign"
],
"courses/moo_moo_farm/course_data.inc.c": [
"d_course_moo_moo_farm_dl_cow1",
"d_course_moo_moo_farm_dl_cow2",
"d_course_moo_moo_farm_dl_cow3",
"d_course_moo_moo_farm_dl_cow4",
"d_course_moo_moo_farm_dl_cow5",
"d_course_moo_moo_farm_dl_tree"
],
"courses/rainbow_road/course_data.inc.c": [
"d_course_rainbow_road_dl_151A8",
"d_course_rainbow_road_dl_15550",
"d_course_rainbow_road_dl_158C0",
"d_course_rainbow_road_dl_15C68",
"d_course_rainbow_road_dl_15F18"
],
"courses/royal_raceway/course_data.inc.c": [
"d_course_royal_raceway_dl_tree",
"d_course_royal_raceway_dl_castle_tree",
"d_course_royal_raceway_dl_piranha_plant"
],
"courses/sherbet_land/course_data.inc.c": [
"d_course_sherbet_land_dl_ice_block",
"d_course_sherbet_land_dl_8E00"
],
"courses/toads_turnpike/course_data.inc.c": [
"d_course_toads_turnpike_dl_19020",
"d_course_toads_turnpike_dl_19518",
"d_course_toads_turnpike_dl_19DF0",
"d_course_toads_turnpike_dl_1A068",
"d_course_toads_turnpike_dl_1A5F8",
"d_course_toads_turnpike_dl_1A6C8",
"d_course_toads_turnpike_dl_1B778",
"d_course_toads_turnpike_dl_1BE48",
"d_course_toads_turnpike_dl_1C700",
"d_course_toads_turnpike_dl_1CAA8",
"d_course_toads_turnpike_dl_1CE70",
"d_course_toads_turnpike_dl_1D018",
"d_course_toads_turnpike_dl_1E458",
"d_course_toads_turnpike_dl_1EB48",
"d_course_toads_turnpike_dl_1F9D0",
"d_course_toads_turnpike_dl_20008",
"d_course_toads_turnpike_dl_20510",
"d_course_toads_turnpike_dl_205A8",
"d_course_toads_turnpike_dl_21780",
"d_course_toads_turnpike_dl_21E28",
"d_course_toads_turnpike_dl_22BA0",
"d_course_toads_turnpike_dl_23078",
"d_course_toads_turnpike_dl_237F8",
"d_course_toads_turnpike_dl_23848",
"d_course_toads_turnpike_dl_23858",
"d_course_toads_turnpike_dl_238A0",
"d_course_toads_turnpike_dl_238E8"
],
"courses/wario_stadium/course_data.inc.c":[
"d_course_wario_stadium_dl_sign"
],
"courses/yoshi_valley/course_data.inc.c":[
"d_course_yoshi_valley_dl_142C0",
"d_course_yoshi_valley_dl_143C8",
"d_course_yoshi_valley_dl_144A8",
"d_course_yoshi_valley_dl_14500",
"d_course_yoshi_valley_dl_14558",
"d_course_yoshi_valley_dl_145D0",
"d_course_yoshi_valley_dl_tree",
"d_course_yoshi_valley_dl_16D70",
"d_course_yoshi_valley_dl_egg_lod0"
],
"src/data/common_textures.inc.c": [
"D_0D001B90",
"D_0D001BD8",
"D_0D001C20",
"D_0D001C88",
"D_0D002EE8",
"common_model_fake_itembox",
"itemBoxQuestionMarkModel",
"D_0D0030F8",
"D_0D003128",
"D_0D003158",
"D_0D003188",
"D_0D0031B8",
"D_0D0031E8",
"D_0D003218",
"D_0D003248",
"D_0D003288",
"common_model_banana",
"common_model_flat_banana",
"D_0D0052B8",
"D_0D005338",
"D_0D005368",
"D_0D007B20",
"D_0D007B98",
"D_0D007C10"
]
}
[
{
"file":"courses/banshee_boardwalk/course_data.inc.c",
"out_dir": "banshee_boardwalk",
"models": [
"d_course_banshee_boardwalk_dl_B278",
"d_course_banshee_boardwalk_dl_trash_bin",
"d_course_banshee_boardwalk_dl_cheep_cheep"
]
},
{
"file": "courses/bowsers_castle/course_data.inc.c",
"out_dir": "bowsers_castle",
"models": [
"d_course_bowsers_castle_dl_thwomp",
"d_course_bowsers_castle_dl_bush"
]
},
{
"file": "courses/choco_mountain/course_data.inc.c",
"out_dir": "choco_mountain",
"models": [
"d_course_choco_mountain_dl_falling_rock"
]
},
{
"file": "courses/dks_jungle_parkway/course_data.inc.c",
"out_dir": "dks_jungle_parkway",
"models":[
"d_course_dks_jungle_parkway_dl_E730",
"d_course_dks_jungle_parkway_dl_E058",
"d_course_dks_jungle_parkway_dl_FC28",
"d_course_dks_jungle_parkway_dl_kiwano_fruit",
"d_course_dks_jungle_parkway_dl_palm_tree",
"d_course_dks_jungle_parkway_dl_13878",
"d_course_dks_jungle_parkway_dl_tree3",
"d_course_dks_jungle_parkway_dl_tree2",
"d_course_dks_jungle_parkway_dl_tree1"
]
},
{
"file":"courses/frappe_snowland/course_data.inc.c",
"out_dir": "frappe_snowland",
"models": [
"d_course_frappe_snowland_dl_tree"
]
},
{
"file": "courses/kalimari_desert/course_data.inc.c",
"out_dir": "kalimari_desert",
"models": [
"d_course_kalimari_desert_dl_cactus1",
"d_course_kalimari_desert_dl_cactus2",
"d_course_kalimari_desert_dl_cactus3",
"d_course_kalimari_desert_dl_crossing_right_active",
"d_course_kalimari_desert_dl_crossing_left_active",
"d_course_kalimari_desert_dl_crossing_both_inactive",
"d_course_kalimari_desert_dl_1B978",
"d_course_kalimari_desert_dl_1C0F0",
"d_course_kalimari_desert_dl_1D160",
"d_course_kalimari_desert_dl_1D670",
"d_course_kalimari_desert_dl_1E480",
"d_course_kalimari_desert_dl_1E910",
"d_course_kalimari_desert_dl_1F228",
"d_course_kalimari_desert_dl_1F708",
"d_course_kalimari_desert_dl_1FAF8",
"d_course_kalimari_desert_dl_20A20",
"d_course_kalimari_desert_dl_21220",
"d_course_kalimari_desert_dl_21550",
"d_course_kalimari_desert_dl_21A80",
"d_course_kalimari_desert_dl_21C90",
"d_course_kalimari_desert_dl_22D28",
"d_course_kalimari_desert_dl_22D70",
"d_course_kalimari_desert_dl_22DB8"
]
},
{
"file": "courses/koopa_troopa_beach/course_data.inc.c",
"out_dir": "koopa_troopa_beach",
"models": [
"d_course_koopa_troopa_beach_dl_16630",
"d_course_koopa_troopa_beach_dl_tree_top1",
"d_course_koopa_troopa_beach_dl_tree_trunk1",
"d_course_koopa_troopa_beach_dl_tree_top2",
"d_course_koopa_troopa_beach_dl_tree_trunk2",
"d_course_koopa_troopa_beach_dl_tree_top3",
"d_course_koopa_troopa_beach_dl_tree_trunk3"
]
},
{
"file": "courses/luigi_raceway/course_data.inc.c",
"out_dir": "luigi_raceway",
"models": [
"d_course_luigi_raceway_dl_F650",
"d_course_luigi_raceway_dl_F960",
"d_course_luigi_raceway_dl_FA20",
"d_course_luigi_raceway_dl_FBE0",
"d_course_luigi_raceway_dl_FC70"
]
},
{
"file": "courses/mario_raceway/course_data.inc.c",
"out_dir": "mario_raceway",
"models": [
"d_course_mario_raceway_dl_piranha_plant",
"d_course_mario_raceway_dl_tree",
"d_course_mario_raceway_dl_sign"
]
},
{
"file": "courses/moo_moo_farm/course_data.inc.c",
"out_dir": "moo_moo_farm",
"models": [
"d_course_moo_moo_farm_dl_cow1",
"d_course_moo_moo_farm_dl_cow2",
"d_course_moo_moo_farm_dl_cow3",
"d_course_moo_moo_farm_dl_cow4",
"d_course_moo_moo_farm_dl_cow5",
"d_course_moo_moo_farm_dl_tree"
]
},
{
"file": "courses/rainbow_road/course_data.inc.c",
"out_dir": "rainbow_road",
"models": [
"d_course_rainbow_road_dl_151A8",
"d_course_rainbow_road_dl_15550",
"d_course_rainbow_road_dl_158C0",
"d_course_rainbow_road_dl_15C68",
"d_course_rainbow_road_dl_15F18"
]
},
{
"file": "courses/royal_raceway/course_data.inc.c",
"out_dir": "royal_raceway",
"models": [
"d_course_royal_raceway_dl_tree",
"d_course_royal_raceway_dl_castle_tree",
"d_course_royal_raceway_dl_piranha_plant"
]
},
{
"file": "courses/sherbet_land/course_data.inc.c",
"out_dir": "sherbet_land",
"models": [
"d_course_sherbet_land_dl_ice_block",
"d_course_sherbet_land_dl_8E00"
]
},
{
"file": "courses/toads_turnpike/course_data.inc.c",
"out_dir": "toads_turnpike",
"models": [
"d_course_toads_turnpike_dl_19020",
"d_course_toads_turnpike_dl_19518",
"d_course_toads_turnpike_dl_19DF0",
"d_course_toads_turnpike_dl_1A068",
"d_course_toads_turnpike_dl_1A5F8",
"d_course_toads_turnpike_dl_1A6C8",
"d_course_toads_turnpike_dl_1B778",
"d_course_toads_turnpike_dl_1BE48",
"d_course_toads_turnpike_dl_1C700",
"d_course_toads_turnpike_dl_1CAA8",
"d_course_toads_turnpike_dl_1CE70",
"d_course_toads_turnpike_dl_1D018",
"d_course_toads_turnpike_dl_1E458",
"d_course_toads_turnpike_dl_1EB48",
"d_course_toads_turnpike_dl_1F9D0",
"d_course_toads_turnpike_dl_20008",
"d_course_toads_turnpike_dl_20510",
"d_course_toads_turnpike_dl_205A8",
"d_course_toads_turnpike_dl_21780",
"d_course_toads_turnpike_dl_21E28",
"d_course_toads_turnpike_dl_22BA0",
"d_course_toads_turnpike_dl_23078",
"d_course_toads_turnpike_dl_237F8",
"d_course_toads_turnpike_dl_23848",
"d_course_toads_turnpike_dl_23858",
"d_course_toads_turnpike_dl_238A0",
"d_course_toads_turnpike_dl_238E8"
]
},
{
"file": "courses/wario_stadium/course_data.inc.c",
"out_dir": "wario_stadium",
"models": [
"d_course_wario_stadium_dl_sign"
]
},
{
"file": "courses/yoshi_valley/course_data.inc.c",
"out_dir": "yoshi_valley",
"models": [
"d_course_yoshi_valley_dl_142C0",
"d_course_yoshi_valley_dl_143C8",
"d_course_yoshi_valley_dl_144A8",
"d_course_yoshi_valley_dl_14500",
"d_course_yoshi_valley_dl_14558",
"d_course_yoshi_valley_dl_145D0",
"d_course_yoshi_valley_dl_tree",
"d_course_yoshi_valley_dl_16D70",
"d_course_yoshi_valley_dl_egg_lod0"
]
},
{
"file": "src/data/common_textures.inc.c",
"out_dir": "common_textures",
"models": [
"D_0D001B90",
"D_0D001BD8",
"D_0D001C20",
"D_0D001C88",
"D_0D002EE8",
"common_model_fake_itembox",
"itemBoxQuestionMarkModel",
"D_0D0030F8",
"D_0D003128",
"D_0D003158",
"D_0D003188",
"D_0D0031B8",
"D_0D0031E8",
"D_0D003218",
"D_0D003248",
"D_0D003288",
"common_model_banana",
"common_model_flat_banana",
"D_0D0052B8",
"D_0D005338",
"D_0D005368",
"D_0D007B20",
"D_0D007B98",
"D_0D007C10"
]
}
]
+2 -1
View File
@@ -30,6 +30,7 @@ drawLayer = bpy.context.scene.DLImportDrawLayer
file = argv[0]
obj = argv[1]
out_file = argv[2]
clean_scene()
if bpy.context.mode != "OBJECT":
@@ -48,4 +49,4 @@ importMeshC(
drawLayer,
F3DContext(get_F3D_GBI(), basePath, createF3DMat(None)),
)
bpy.ops.wm.save_as_mainfile(filepath="models/"+obj+".blend")
bpy.ops.wm.save_as_mainfile(filepath=out_file)
+38 -27
View File
@@ -11,7 +11,7 @@ args = parser.parse_args()
if not os.path.exists("models"):
os.makedirs("models")
models = json.load(open("tools/3d_models.json"))
models:list[dict[str, str|list[str]]] = json.load(open("tools/3d_models.json"))
if os.name == "nt":
blender = "c:/Program Files/Blender Foundation/Blender 4.0/blender.exe"
@@ -27,42 +27,53 @@ args_blender = [
failed = []
threads:list[tuple[threading.Thread, str]] = []
for file in models:
for obj in models[file]:
if os.path.exists("models/"+obj+".blend"):
continue
while len(threads) >= args.threads:
for i, (thread, out) in enumerate(threads):
class QueueThread:
def __init__(self, max_threads:int, check):
self.threads:list[tuple[threading.Thread, str]] = []
self.max_threads = max_threads
self.check = check
def check_available(self):
while len(self.threads) >= args.threads:
for i, (thread, out) in enumerate(self.threads):
if not thread.is_alive():
thread.join()
if os.path.exists(out):
print(f"Exported {out}")
else:
print(f"Failed to export {out}")
if not self.check(out):
failed.append(out)
del threads[i]
del self.threads[i]
break
print(f"Exporting {obj} from {file}")
def append(self, thread:threading.Thread, out:str):
self.check_available()
self.threads.append((thread, out))
thread.start()
def finish(self):
for thread, out in self.threads:
thread.join()
if not self.check(out):
failed.append(out)
queue = QueueThread(args.threads, lambda out: os.path.exists(out))
for model in models:
file = model["file"]
out_dir = "models/"+model["out_dir"] if "out_dir" in model else "models"
if not os.path.exists(out_dir):
os.makedirs(out_dir)
for obj in model["models"]:
out_file = out_dir+"/"+obj+".blend"
if os.path.exists(out_file):
continue
args_blender.append(file)
args_blender.append(obj)
args_blender.append(out_file)
thread = threading.Thread(target=lambda: subprocess.run([blender]+args_blender))
thread.start()
threads.append((
thread,
"models/"+obj+".blend"
))
queue.append(thread, out_file)
args_blender.pop()
args_blender.pop()
args_blender.pop()
for thread, out in threads:
thread.join()
if os.path.exists(out):
print(f"Exported {out}")
else:
print(f"Failed to export {out}")
failed.append(out)
queue.finish()
print("Failed to export:")
print("\n".join(failed))