Add support for clangd in the python tools (#120)

* initial clangd support through python

* update install instructions

* invert configure.py behavior

* fix oopsie

* fix few issues from clangd

* fix related build issues
This commit is contained in:
Yanis
2026-07-21 19:37:21 +02:00
committed by GitHub
parent 04d20febb1
commit bc04021bc0
15 changed files with 322 additions and 51 deletions
+39
View File
@@ -0,0 +1,39 @@
CompileFlags:
Add: [
"-Wno-c++11-compat-deprecated-writable-strings",
"-Wno-multichar",
"-fdeclspec",
"-Wno-c++11-extensions",
"-Wuninitialized",
"-Wsometimes-uninitialized",
"-Wlogical-op-parentheses",
"-Wbitwise-op-parentheses",
# "-Wunused-variable",
# "-Wunused-but-set-variable",
"-Wunused-parameter",
"-Wunused-but-set-parameter",
"-Wself-assign",
]
Diagnostics:
Suppress:
- "warn_char_constant_too_large"
- "illegal_union_or_anon_struct_member"
- "main_returns_nonint"
- "template_spec_needs_header"
Documentation:
CommentFormat: Doxygen
---
If:
PathMatch: .*/*.inc
Diagnostics:
Suppress:
- "undeclared_var_use"
- "undeclared_var_use_suggest"
- "bound_member_function"
- "typecheck_subscript_value"
- "unknown_typename"
---
If:
PathMatch: .*\.pch
CompileFlags:
Add: ["--language=c++", "--std=c++98"]
-1
View File
@@ -1,6 +1,5 @@
build/
.vscode/
.clangd
cmake/
compile_commands.json
ph_*/
+4
View File
@@ -54,6 +54,10 @@ ARM7 BIOS in the root directory of this repository, and verify that your dumped
## LSP setup
By default `configure.py` will create the config file for clangd (at the root folder of the project), if you want to use the alternative setup you can run `configure.py` with `--noclangd | -c` to disable the creation of the file.
If you wish to use CMake:
**This is likely not necessary.** Most C++ editors usually have their one LSP (Language Server Protocol, a tool for code completion and more) configuration that should recognize the project structure and work out of the box. This section is about how to setup your LSP yourself **if the need be**.
The repository contains a [`CMakeLists.txt`](CMakeLists.txt) that allows generating a compilation database. For now, the `CMakeLists.txt` can only be used to generate `compile_commands.json` and similar files, not compiling the project.
+1 -1
View File
@@ -27,7 +27,7 @@ typedef struct FileInfos {
// .cib
typedef struct CourseInitEntry {
/* 00 */ const char name[16];
/* 00 */ char name[16];
/* 10 */ STRUCT_PAD(0x10, 0x24);
} CourseInitEntry; // size = 0x24
+1 -1
View File
@@ -2,7 +2,7 @@
#define GLOBAL_H
// Prevent the IDE from reporting errors that the compiler/linker won't report
#ifdef __INTELLISENSE__
#ifndef __MWERKS__
#endif
// start of thumb region, using thumb instructions
+9 -9
View File
@@ -50,20 +50,20 @@ public:
~AutoInstance() {}
};
#define DECL_INSTANCE_CTOR(T, gpInstance) \
#define DECL_INSTANCE_CTOR(type, gpInstance) \
template <typename T> Instance<T>::Instance() { \
gpInstance = (T *) this; \
gpInstance = (type *) this; \
} \
template class Instance<T>;
template class Instance<type>;
#define DECL_INSTANCE_DTOR(T, gpInstance) \
Instance<T>::~Instance() { \
gpInstance = NULL; \
#define DECL_INSTANCE_DTOR(type, gpInstance) \
template <> Instance<type>::~Instance() { \
gpInstance = NULL; \
}
#define DECL_INSTANCE(T, gpInstance) \
DECL_INSTANCE_CTOR(T, gpInstance) \
DECL_INSTANCE_DTOR(T, gpInstance)
#define DECL_INSTANCE(type, gpInstance) \
DECL_INSTANCE_CTOR(type, gpInstance) \
DECL_INSTANCE_DTOR(type, gpInstance)
template <typename T> struct StaticInstance {
static T sInstance;
+3 -3
View File
@@ -11,7 +11,9 @@ namespace Metrowerks {
namespace detail {
#ifdef DECOMP_IDE_FLAG
#if __MWERKS__
typedef short double short_double;
#else
#define __builtin_ntype(T) (detail::ntype) 0
#define __builtin_align(T) 0
#define __builtin_is_pod(T) 0
@@ -29,8 +31,6 @@ namespace Metrowerks {
#define __builtin_has_virtual_destructor(T) 0
typedef double short_double;
typedef double __vec2x32float__;
#else
typedef short double short_double;
#endif
enum trivial_member {
@@ -16,10 +16,10 @@ namespace std {
static true_type test(To) { __MWERKS_NOEVAL; }
static false_type test(...) { __MWERKS_NOEVAL; }
#ifdef DECOMP_IDE_FLAG
typedef false_type type;
#if __MWERKS__
typedef __decltype__(test(declval<From>())) type;
#else
typedef __decltype__(test(declval<From>())) type;
typedef false_type type;
#endif
};
@@ -28,10 +28,10 @@ namespace std {
static true_type test(const volatile Base*) { __MWERKS_NOEVAL; }
static false_type test(const volatile void*) { __MWERKS_NOEVAL; }
#ifdef DECOMP_IDE_FLAG
typedef false_type type;
#if __MWERKS__
typedef __decltype__(test(declval<Derived*>())) type;
#else
typedef __decltype__(test(declval<Derived*>())) type;
typedef false_type type;
#endif
};
+3 -1
View File
@@ -34,6 +34,7 @@ bool ActorUnkSWOB::vfunc_18(unk32 param1) {
this->SetState(ActorUnkSWOBState_0);
}
#pragma unused(param1)
return true;
}
@@ -70,7 +71,7 @@ void ActorUnkSWOB::func_ov000_0209aa30(void) {
case 1:
data_ov000_020b5214.func_ov000_0206db44(0xA3);
break;
case 3:
case 3: {
data_ov000_020b5214.func_ov000_0206db44(0xA3);
s16 unk_78 = this->mUnk_5C.mUnk_1A[1];
@@ -78,6 +79,7 @@ void ActorUnkSWOB::func_ov000_0209aa30(void) {
data_027e0cd8->func_ov000_02081d7c((s16) (unk_78 - 1), this->mUnk_5C.mUnk_18[1], true);
}
break;
}
case 2: {
VecFx32 temp;
VecFx32 vec2;
+5 -5
View File
@@ -75,8 +75,8 @@ void UnkStruct_027e0cd8::func_ov001_020b7830(const EntranceInfo *param1) {
pUnk1 = data_027e09a0->func_ov000_020702a8(param1->sceneIndex);
wchar_t sp80[16];
sp80[0] = L'\0';
sp80[sizeof(sp80) / 2 - 1] = L'\0';
sp80[0] = L'\0';
sp80[ARRAY_LEN(sp80) - 1] = L'\0';
snprintf((char *) sp80, sizeof(sp80), "Map/%s/course.bin", pEntry->name);
UnkStruct2 sp70((char *) sp80, 0x01);
@@ -123,8 +123,8 @@ void UnkStruct_027e0cd8::func_ov001_020b7830(const EntranceInfo *param1) {
if (pEntry->unk_21 != -1) {
wchar_t sp30[16];
sp30[0] = L'\0';
sp30[sizeof(sp30) / 2 - 1] = L'\0';
sp30[0] = L'\0';
sp30[ARRAY_LEN(sp30) - 1] = L'\0';
snprintf((char *) sp30, sizeof(sp30), "Map/%s/course.bin", data_027e09a0->GetCourseEntry(pEntry->unk_21)->name);
@@ -254,7 +254,7 @@ void UnkStruct_027e0cd8::func_ov001_020b7c08(const EntranceInfo *param1, const U
this->func_ov001_020b7e50();
if (sceneChange.roomIndex != ROOM_INDEX_NONE) {
if (this->mUnk_30 == sceneChange.sceneIndex && DSProt_DetectNotEmulator(func_ov084_0216122c) == 0) {
if (this->mUnk_30 == sceneChange.sceneIndex && DSProt_DetectNotEmulator((void *) func_ov084_0216122c) == 0) {
func_ov084_021612ac();
}
+2 -2
View File
@@ -134,7 +134,7 @@ void Game::Run() {
{
int enabled = OS_DisableInterrupts_Irq();
this->mUnk_1C.func_02013e18(func_020132dc, 0);
this->mUnk_1C.func_02013e18((void *) func_020132dc, 0);
REG_GFX_FIFO_SWAP_BUFFERS = 3;
OS_RestoreInterrupts(enabled);
}
@@ -143,7 +143,7 @@ void Game::Run() {
if (this->mUnk_18 != NULL) {
while (this->mUnk_18() != 0) {
while (this->mUnk_1C.func_02013e18(func_02013354, 0) == 0) {
while (this->mUnk_1C.func_02013e18((void *) func_02013354, 0) == 0) {
}
func_020132c8();
+2 -1
View File
@@ -85,7 +85,7 @@ void SysFault::func_020127f0(unk32 param1) {
switch (param1) {
case 0:
case 1:
case 1: {
data_02049b18.mButtons.func_02013b24(data_02049b18.mButtons.func_02013bbc());
u16 expected = data_0203e0c4[this->mUnk_04];
@@ -108,6 +108,7 @@ void SysFault::func_020127f0(unk32 param1) {
doDraw = false;
break;
}
case 2:
case 3:
default:
+1 -2
View File
@@ -1,5 +1,4 @@
#include "System/SysNew.hpp"
#include "global.h"
extern "C" {
void *func_02001654(void *);
@@ -68,7 +67,7 @@ void *func_02011f30(s32 length) {
return func_02011f10(length);
}
void *operator new(unsigned long length, u32 id, u32 idLength) {
void *operator new(size_t length, u32 id, u32 idLength) {
void *pvVar1;
UnkStruct_02011e10_Sub1 *pUVar5;
+1
View File
@@ -16,6 +16,7 @@ parser.add_argument("--compiler", type=Path, required=False, help="Path to pre-i
parser.add_argument("--no-extract", action="store_true", help="Skip extract step")
parser.add_argument("--dsd", type=Path, required=False, help="Path to pre-installed dsd CLI")
parser.add_argument("--version", "-v", help='Game version', required=False)
parser.add_argument("--noclangd", "-c", help='Do not create clangd config', required=False, action="store_true")
args = parser.parse_args()
config = ProjectConfig("st", args.compiler, "dsi/1.2p1", args.wine, args.dsd, Path(__file__).resolve())
+245 -19
View File
@@ -5,7 +5,7 @@ import subprocess
import ninja_syntax
from pathlib import Path
from typing import Any, Optional, Dict, List
from typing import Iterable, Any, Dict, List, Optional, Set, Tuple
from get_platform import get_platform
@@ -48,12 +48,15 @@ def get_c_cpp_files(dirs: list[Path]):
def is_cpp(name: str | Path):
return Path(name).suffix in [".cpp"]
return Path(name).suffix in [".cc", ".cp", ".cpp", ".cxx", ".pch++"]
def is_c(name: str | Path):
return Path(name).suffix in [".c"]
def is_c_cpp(name: str | Path):
return is_c(name) or is_cpp(name)
class Object:
def __init__(self, name: str, **options: Any):
self.name = name
@@ -65,6 +68,7 @@ class Object:
"extra_asflags": [],
"cflags": None,
"extra_cflags": [],
"extra_clang_flags": [],
"asm_dir": None,
"src_dir": None,
}
@@ -171,7 +175,7 @@ class ProjectConfig:
if dir == "include":
includes.append(Path(root) / dir)
self.includes = " ".join(f"-i {include}" for include in includes)
self.includes = [f"-i {include}" for include in includes]
"""C/C++ includes"""
self.auto_add_sources: bool = False
@@ -180,6 +184,12 @@ class ProjectConfig:
self.warn_missing_source: bool = True
"""Warn on missing source file"""
self.generate_compile_commands: bool = True
"""Generate compile_commands.json for clangd"""
self.extra_clang_flags: List[str] = []
"""Extra flags for clangd"""
def get_game_config(self, version: str):
"""Root directory for dsd configs"""
config_path = self.config_path / version
@@ -297,13 +307,14 @@ class ProjectConfig:
def arm9_disassembly_dir(self, version: str) -> Path:
return self.get_game_build(version) / "asm"
def objdiff_report(self, version: str) -> Path:
def objdiff_report(self, version: str) -> str:
return f"report_{version}.json"
def files(self, version: str) -> list[dict[str, str]]:
if self.delinks_jsons[version] is None:
delinks = self.delinks_jsons[version]
if delinks is None:
return []
return self.delinks_jsons[version]['files']
return delinks['files']
def delink_files(self, version: str) -> list[str]:
delink_files = [file['delink_file'] for file in self.files(version)]
@@ -312,14 +323,16 @@ class ProjectConfig:
return delink_files
def arm9_lcf_file(self, version: str) -> str:
if self.delinks_jsons[version] is None:
delinks = self.delinks_jsons[version]
if delinks is None:
return ""
return self.delinks_jsons[version]['arm9_lcf_file']
return delinks['arm9_lcf_file']
def arm9_objects_file(self, version: str) -> str:
if self.delinks_jsons[version] is None:
delinks = self.delinks_jsons[version]
if delinks is None:
return ""
return self.delinks_jsons[version]['arm9_objects_file']
return delinks['arm9_objects_file']
def get_config_files(self, version: str, name: str) -> list[str]:
files = [
@@ -372,7 +385,7 @@ def add_download_tool_builds(cfg: ProjectConfig, n: ninja_syntax.Writer, args: A
variables={
"tool": "dsd",
"tag": cfg.dsd_version,
"path": cfg.dsd_path,
"path": str(cfg.dsd_path),
},
)
n.newline()
@@ -402,7 +415,7 @@ def add_download_tool_builds(cfg: ProjectConfig, n: ninja_syntax.Writer, args: A
)
n.newline()
if cfg.platform.system != "windows" and cfg.wine_path == cfg.default_wibo_path:
if cfg.platform is not None and cfg.platform.system != "windows" and cfg.wine_path == cfg.default_wibo_path:
downloads.append(str(cfg.wine_path))
n.build(
rule="download_tool",
@@ -519,7 +532,7 @@ def add_disassemble_builds(cfg: ProjectConfig, version: str, n: ninja_syntax.Wri
def add_mwcc_builds(cfg: ProjectConfig, version: str, objects: Dict[str, Object], n: ninja_syntax.Writer, mwcc_implicit: list[str]):
file_map: dict[str, list[str]] = {}
file_map: dict[str, list[str] | None] = {}
for object in objects.values():
file_map[str(object.src_path)] = object.options["cflags"] + object.options["extra_cflags"]
@@ -536,7 +549,8 @@ def add_mwcc_builds(cfg: ProjectConfig, version: str, objects: Dict[str, Object]
if cfg.warn_missing_source and not source_file.exists():
print(f"WARNING: path not found for `{source_file}`")
if "-lang=c++" not in cc_flags or "-lang=c" not in cc_flags:
assert cc_flags is not None, "cc_flags is None"
if "-lang=c++" not in cc_flags or "-lang=c" not in cc_flags:
if is_cpp(source_file):
cc_flags.append("-lang=c++")
elif is_c(source_file):
@@ -757,13 +771,223 @@ def create_objdiff_fixup_config(cfg: ProjectConfig, objects: Dict[str, Object]):
json.dump(out_json, f, indent=2)
def create_compile_commands(cfg: ProjectConfig):
if not cfg.generate_compile_commands:
return
objects: Dict[str, Object] = cfg.objects()
# The following code attempts to convert mwcc flags to clang flags
# for use with clangd.
#
# Adapted from: https://github.com/encounter/dtk-template/blob/95a941f755919ebe50c1725a4ce73524470e7a02/tools/project.py#L1781
# Flags to ignore explicitly
CFLAG_IGNORE: Set[str] = {
# Search order modifier
# Has a different meaning to Clang, and would otherwise
# be picked up by the include passthrough prefix
"-I-",
"-i-",
}
CFLAG_IGNORE_PREFIX: Tuple[str, ...] = (
# Recursive includes are not supported by modern compilers
"-ir ",
)
# Flags to replace
CFLAG_REPLACE: Dict[str, str] = {}
CFLAG_REPLACE_PREFIX: Tuple[Tuple[str, str], ...] = (
# Includes
("-i ", "-I"),
("-I ", "-I"),
("-I+", "-I"),
# Defines
("-d ", "-D"),
("-D ", "-D"),
("-D+", "-D"),
)
# Flags with a finite set of options
CFLAG_REPLACE_OPTIONS: Tuple[Tuple[str, Dict[str, Tuple[str, ...]]], ...] = (
# Exceptions
(
"-Cpp_exceptions",
{
"off": ("-fno-cxx-exceptions",),
"on": ("-fcxx-exceptions",),
},
),
# RTTI
(
"-RTTI",
{
"off": ("-fno-rtti",),
"on": ("-frtti",),
},
),
# Language configuration
(
"-lang",
{
"c": ("--language=c", "--std=c99"),
"c99": ("--language=c", "--std=c99"),
"c++": ("--language=c++", "--std=c++98"),
"cplus": ("--language=c++", "--std=c++98"),
},
),
# Enum size
(
"-enum",
{
"min": ("-fshort-enums",),
"int": ("-fno-short-enums",),
},
),
# Common BSS
(
"-common",
{
"off": ("-fno-common",),
"on": ("-fcommon",),
},
),
)
# Flags to pass through
CFLAG_PASSTHROUGH: Set[str] = set()
CFLAG_PASSTHROUGH_PREFIX: Tuple[str, ...] = (
"-I", # includes
"-D", # defines
)
clangd_config = []
def add_unit(obj: Object):
# Skip unresolved objects
if (
obj.src_path is None
or obj.src_obj_path is None
or not is_c_cpp(obj.src_path)
):
return
# Gather cflags for source file
cflags: list[str] = []
def append_cflags(flags: Iterable[str]) -> None:
# Match a flag against either a set of concrete flags, or a set of prefixes.
def flag_match(
flag: str, concrete: Set[str], prefixes: Tuple[str, ...]
) -> bool:
if flag in concrete:
return True
for prefix in prefixes:
if flag.startswith(prefix):
return True
return False
# Determine whether a flag should be ignored.
def should_ignore(flag: str) -> bool:
return flag_match(flag, CFLAG_IGNORE, CFLAG_IGNORE_PREFIX)
# Determine whether a flag should be passed through.
def should_passthrough(flag: str) -> bool:
return flag_match(flag, CFLAG_PASSTHROUGH, CFLAG_PASSTHROUGH_PREFIX)
# Attempts replacement for the given flag.
def try_replace(flag: str) -> bool:
replacement = CFLAG_REPLACE.get(flag)
if replacement is not None:
cflags.append(replacement)
return True
for prefix, replacement in CFLAG_REPLACE_PREFIX:
if flag.startswith(prefix):
cflags.append(flag.replace(prefix, replacement, 1))
return True
for prefix, options in CFLAG_REPLACE_OPTIONS:
if not flag.startswith(prefix):
continue
# "-lang c99" and "-lang=c99" are both generally valid option forms
option = flag.removeprefix(prefix).removeprefix("=").lstrip()
replacements = options.get(option)
if replacements is not None:
cflags.extend(replacements)
return True
return False
for flag in flags:
# Ignore flags first
if should_ignore(flag):
continue
# Then find replacements
if try_replace(flag):
continue
# Pass flags through last
if should_passthrough(flag):
cflags.append(flag)
continue
append_cflags(cfg.includes)
append_cflags(obj.options["cflags"])
append_cflags(obj.options["extra_cflags"])
cflags.extend(cfg.extra_clang_flags)
cflags.extend(obj.options["extra_clang_flags"])
unit_config = {
"directory": Path.cwd(),
"file": obj.src_path,
"output": obj.src_obj_path,
"arguments": [
"clang",
"-nostdinc",
"-fno-builtin",
"--target=arm-none-eabi",
"-march=armv5te",
"-mfloat-abi=soft",
*cflags,
"-c",
obj.src_path,
"-o",
obj.src_obj_path,
],
}
clangd_config.append(unit_config)
# Add units
for name, object in objects.items():
add_unit(object)
# Write compile_commands.json
with open("compile_commands.json", "w", encoding="utf-8") as w:
def default_format(o):
if isinstance(o, Path):
return o.resolve().as_posix()
return str(o)
json.dump(clangd_config, w, indent=2, default=default_format)
def process_project(cfg: ProjectConfig, args: Any):
objects = cfg.objects()
if not args.noclangd:
create_compile_commands(cfg)
create_objdiff_fixup_config(cfg, objects)
rust_log = "RUST_LOG=ds_rom::rom::rom=warn"
if cfg.platform.system == "windows":
if cfg.platform is not None and cfg.platform.system == "windows":
rust_log = f"set {rust_log} &&"
with cfg.build_ninja_path.open("w") as file:
@@ -800,9 +1024,10 @@ def process_project(cfg: ProjectConfig, args: Any):
n.newline()
# -MMD excludes all includes instead of just system includes for some reason, so use -MD instead.
mwcc_cmd = f'{cfg.wine_path} {cfg.sjiswrap_path} "{cfg.cc_path}" $cc_flags {cfg.includes} -DVERSION=$game_version -MD -c $in -o $basedir'
includes = " ".join(include for include in cfg.includes)
mwcc_cmd = f'{cfg.wine_path} {cfg.sjiswrap_path} "{cfg.cc_path}" $cc_flags {includes} -DVERSION=$game_version -MD -c $in -o $basedir'
mwcc_implicit = [str(cfg.cc_path), str(cfg.sjiswrap_path)]
if cfg.platform.system != "windows":
if cfg.platform and cfg.platform.system != "windows":
transform_dep = "tools/transform_dep.py"
mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d"
mwcc_implicit.append(transform_dep)
@@ -821,9 +1046,10 @@ def process_project(cfg: ProjectConfig, args: Any):
)
n.newline()
ldflags = " ".join(cfg.ldflags) if cfg.ldflags is not None else ""
n.rule(
name="mwld",
command=f'{cfg.wine_path} "{cfg.ld_path}" {' '.join(cfg.ldflags)} $extra_ld_flags @$objects_file $lcf_file -o $out'
command=f'{cfg.wine_path} "{cfg.ld_path}" {ldflags} $extra_ld_flags @$objects_file $lcf_file -o $out'
)
n.newline()
@@ -839,7 +1065,7 @@ def process_project(cfg: ProjectConfig, args: Any):
)
n.newline()
cflags = " ".join(cfg.cflags_base)
cflags = " ".join(cfg.cflags_base) if cfg.cflags_base is not None else ""
dsd_objdiff_args = " ".join([
"--scratch", # Metadata for creating decomp.me scratches
f"--compiler {cfg.get_decompme_compiler()}", # decomp.me compiler name