mirror of https://github.com/mongodb/mongo
SERVER-93557 bazel header list generator should find gen headers from lib deps (#26104)
GitOrigin-RevId: 308b614aaf56cfbf6f8567c4c3c0d6fec7b1240b
This commit is contained in:
Daniel Moody
MongoDB Bot
parent
2713bbe201
commit
ae2ecce6e7
|
|
@ -280,6 +280,7 @@ buildifier
|
|||
.ninja_last_command_line_targets.txt
|
||||
bazel/coverity/analysis/BUILD.bazel
|
||||
.bazel_include_info.json
|
||||
.bazel_header_list_cache
|
||||
# generated configs for external fixture suites
|
||||
docker_compose/
|
||||
|
||||
|
|
|
|||
166
SConstruct
166
SConstruct
|
|
@ -6556,171 +6556,7 @@ clang_tidy_config = env.Substfile(
|
|||
env.Alias("generated-sources", clang_tidy_config)
|
||||
|
||||
if get_option("bazel-includes-info"):
|
||||
target_library = get_option("bazel-includes-info").replace("\\", "/")
|
||||
|
||||
header_map = {}
|
||||
bazel_query = ["aquery"] + env["BAZEL_FLAGS_STR"] + ['mnemonic("CppArchive", //src/...)']
|
||||
|
||||
results = env.RunBazelQuery(bazel_query, "getting all bazel libraries")
|
||||
targets = set()
|
||||
for line in results.stdout.split("\n"):
|
||||
if " Target: //src" in line:
|
||||
target = line.split(" Target: ")[-1]
|
||||
targets.add(target)
|
||||
|
||||
for target in targets:
|
||||
print(f"getting headers for {target}")
|
||||
bazel_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f'labels(hdrs, "@{target}")',
|
||||
"--output",
|
||||
"files",
|
||||
]
|
||||
)
|
||||
results = env.RunBazelQuery(bazel_query, f"getting {target} headers")
|
||||
|
||||
bazel_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f'kind("extract_debuginfo", rdeps(@//src/mongo/..., "@{target}", 1))',
|
||||
]
|
||||
)
|
||||
target_results = env.RunBazelQuery(bazel_query, "getting real target")
|
||||
target = target_results.stdout.split(" ")[0]
|
||||
header_map[target] = []
|
||||
for line in results.stdout.split("\n"):
|
||||
if not line.endswith("src/mongo/config.h"):
|
||||
header_map[target] += [line]
|
||||
|
||||
bazel_query = (
|
||||
["aquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ ['mnemonic("IdlcGenerator|TemplateRenderer|ConfigHeaderGen", //src/...)']
|
||||
)
|
||||
|
||||
source_gen_targets = set()
|
||||
results = env.RunBazelQuery(bazel_query, "getting all source gen targets")
|
||||
for line in results.stdout.split("\n"):
|
||||
if " Target: //src" in line:
|
||||
target = line.split(" Target: ")[-1]
|
||||
source_gen_targets.add(target)
|
||||
|
||||
for target in source_gen_targets:
|
||||
print(f"getting generated headers for {target}")
|
||||
bazel_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f"@{target}",
|
||||
"--output",
|
||||
"files",
|
||||
]
|
||||
)
|
||||
results = env.RunBazelQuery(bazel_query, f"getting {target} headers")
|
||||
header_map[target] = []
|
||||
for line in results.stdout.split("\n"):
|
||||
if line.endswith(".h") and not line.endswith("src/mongo/config.h"):
|
||||
header_map[target] += [line]
|
||||
|
||||
bazel_include_info = {
|
||||
"header_map": header_map,
|
||||
"bazel_exec": env["SCONS2BAZEL_TARGETS"].bazel_executable,
|
||||
"config": env["BAZEL_FLAGS_STR"] + ["--config=local"],
|
||||
}
|
||||
|
||||
with open(".bazel_include_info.json", "w") as f:
|
||||
json.dump(bazel_include_info, f)
|
||||
|
||||
def bazel_includes_emitter(target, source, env):
|
||||
rel_target = os.path.relpath(str(target[0].abspath), start=env.Dir("#").abspath).replace(
|
||||
"\\", "/"
|
||||
)
|
||||
|
||||
if rel_target == target_library:
|
||||
objsuffix = (
|
||||
env.subst("$OBJSUFFIX")
|
||||
if not env.TargetOSIs("linux")
|
||||
else env.subst("$SHOBJSUFFIX")
|
||||
)
|
||||
builder_name = (
|
||||
"StaticLibrary" if not env.TargetOSIs("linux") == "nt" else "SharedLibrary"
|
||||
)
|
||||
os.makedirs(os.path.dirname(str(target[0].abspath)), exist_ok=True)
|
||||
with open(str(target[0].abspath) + ".obj_files", "w") as f:
|
||||
for s in source:
|
||||
if str(s).endswith(objsuffix):
|
||||
f.write(os.path.relpath(str(s.abspath), start=env.Dir("#").abspath) + "\n")
|
||||
with open(str(target[0].abspath) + ".env_vars", "w") as f:
|
||||
json.dump(env["ENV"], f)
|
||||
|
||||
with (
|
||||
open(str(target[0].abspath) + ".bazel_headers", "w") as fheaders,
|
||||
open(str(target[0].abspath) + ".bazel_deps", "w") as fdeps,
|
||||
):
|
||||
# note we can't know about LIBDEPS_DEPDENDENTS (reverse deps) in an emitter
|
||||
# however we do co-opt the libdeps linter to check for these at the end of reading
|
||||
# sconscripts
|
||||
|
||||
deps = []
|
||||
for s in (
|
||||
env.get("LIBDEPS", [])
|
||||
+ env.get("LIBDEPS_PRIVATE", [])
|
||||
+ env.get("LIBDEPS_INTERFACE", [])
|
||||
):
|
||||
if not s:
|
||||
continue
|
||||
|
||||
libnode = libdeps._get_node_with_ixes(env, s, builder_name)
|
||||
|
||||
libnode_path = os.path.relpath(
|
||||
str(libnode.abspath), start=env.Dir("#").abspath
|
||||
).replace("\\", "/")
|
||||
if (
|
||||
libnode.has_builder()
|
||||
and libnode.get_builder().get_name(env) != "ThinTarget"
|
||||
):
|
||||
print(
|
||||
f"ERROR: can generate correct bazel header list because {target[0]} has non-bazel dependency: {libnode}"
|
||||
)
|
||||
sys.exit(1)
|
||||
if str(libnode_path) in env["SCONS2BAZEL_TARGETS"].scons2bazel_targets:
|
||||
bazel_target = env["SCONS2BAZEL_TARGETS"].bazel_target(str(libnode_path))
|
||||
# new query to run, run and cache it
|
||||
deps.append(bazel_target)
|
||||
bazel_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f'filter("[\\.h,\\.ipp,\\.hpp].*$", kind("source", deps("@{bazel_target}")))',
|
||||
"--output",
|
||||
"files",
|
||||
]
|
||||
)
|
||||
results = env.RunBazelQuery(bazel_query, "getting bazel headers")
|
||||
|
||||
if results.returncode != 0:
|
||||
print("ERROR: bazel libdeps query failed:")
|
||||
print(results)
|
||||
sys.exit(1)
|
||||
results = set(
|
||||
[line for line in results.stdout.split("\n") if line.startswith("src/")]
|
||||
)
|
||||
|
||||
for header in results:
|
||||
fheaders.write(header + "\n")
|
||||
for dep in deps:
|
||||
fdeps.write(dep + "\n")
|
||||
|
||||
return target, source
|
||||
|
||||
for builder_name in ["SharedLibrary", "StaticLibrary", "Program"]:
|
||||
builder = env["BUILDERS"][builder_name]
|
||||
base_emitter = builder.emitter
|
||||
new_emitter = SCons.Builder.ListEmitter([base_emitter, bazel_includes_emitter])
|
||||
builder.emitter = new_emitter
|
||||
env.Tool("bazel_includes_info")
|
||||
|
||||
env.SConscript(
|
||||
dirs=[
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ def main(
|
|||
bazel_include_info = json.load(f)
|
||||
|
||||
header_map = bazel_include_info["header_map"]
|
||||
gen_header_map = bazel_include_info["gen_header_map"]
|
||||
bazel_exec = bazel_include_info["bazel_exec"]
|
||||
bazel_config = bazel_include_info["config"]
|
||||
|
||||
|
|
@ -175,7 +176,6 @@ def main(
|
|||
)
|
||||
|
||||
reverse_header_map = {}
|
||||
reverse_header_gen_map = {}
|
||||
for k, v in header_map.items():
|
||||
for hdr in v:
|
||||
if not hdr or hdr.endswith(global_headers):
|
||||
|
|
@ -186,8 +186,6 @@ def main(
|
|||
reverse_header_map[bazel_header] = "//src/third_party/SafeInt:headers"
|
||||
elif bazel_header.startswith("//src/third_party/immer"):
|
||||
reverse_header_map[bazel_header] = "//src/third_party/immer:headers"
|
||||
elif bazel_header.startswith("//bazel-out/"):
|
||||
reverse_header_gen_map[bazel_header] = k
|
||||
elif bazel_header in reverse_header_map:
|
||||
if bazel_header.startswith("//src/third_party/"):
|
||||
continue
|
||||
|
|
@ -197,16 +195,26 @@ def main(
|
|||
else:
|
||||
reverse_header_map[bazel_header] = k
|
||||
|
||||
for k, v in gen_header_map.items():
|
||||
for hdr in v:
|
||||
if not hdr or hdr.endswith(global_headers):
|
||||
continue
|
||||
bazel_header = "//" + hdr.replace("\\", "/")
|
||||
bazel_header = ":".join(bazel_header.rsplit("/", 1))
|
||||
if bazel_header not in reverse_header_map:
|
||||
reverse_header_map[bazel_header] = k
|
||||
|
||||
recommended_deps = set()
|
||||
minimal_headers = []
|
||||
minimal_headers = set()
|
||||
for header in headers:
|
||||
if header in reverse_header_map:
|
||||
recommended_deps.add(reverse_header_map[header])
|
||||
elif header in reverse_header_gen_map:
|
||||
minimal_headers.append(reverse_header_gen_map[header])
|
||||
if reverse_header_map[header] in gen_header_map:
|
||||
minimal_headers.add(reverse_header_map[header])
|
||||
else:
|
||||
recommended_deps.add(reverse_header_map[header])
|
||||
else:
|
||||
if not header.endswith(global_headers):
|
||||
minimal_headers.append(header)
|
||||
minimal_headers.add(header)
|
||||
|
||||
working_deps = recommended_deps.copy()
|
||||
for dep in recommended_deps:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,254 @@
|
|||
import os
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import SCons
|
||||
import libdeps_tool
|
||||
|
||||
|
||||
def exists(env):
|
||||
return True
|
||||
|
||||
|
||||
def get_md5(file_path):
|
||||
h = hashlib.md5()
|
||||
|
||||
with open(file_path, "rb") as file:
|
||||
while True:
|
||||
# Reading is buffered, so we can read smaller chunks.
|
||||
chunk = file.read(h.block_size)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def get_target_headers(env, target, header_query, symlink_query=None):
|
||||
header_list_cache_dir = Path(".bazel_header_list_cache")
|
||||
target_path = "/".join(target.rsplit(":", 1))[2:]
|
||||
bazel_file = Path(os.path.dirname(target_path)) / "BUILD.bazel"
|
||||
cache_file = str(header_list_cache_dir / bazel_file) + ".json"
|
||||
build_file_hash = get_md5(bazel_file)
|
||||
cache_data = None
|
||||
|
||||
if os.path.exists(cache_file):
|
||||
with open(cache_file) as f:
|
||||
cache_data = json.load(f)
|
||||
if cache_data["MD5"] == build_file_hash:
|
||||
if target in cache_data:
|
||||
return cache_data[target]["headers"], cache_data[target]["macro_name"]
|
||||
else:
|
||||
# invalidate the cache
|
||||
cache_data = None
|
||||
|
||||
if cache_data is None:
|
||||
print(f"{bazel_file} changed, invalidating cache")
|
||||
os.makedirs(os.path.dirname(cache_file), exist_ok=True)
|
||||
cache_data = dict()
|
||||
cache_data["MD5"] = build_file_hash
|
||||
|
||||
print(f"getting {target} headers")
|
||||
results = env.RunBazelQuery(header_query, f"getting {target} headers")
|
||||
cache_data[target] = {"headers": [], "macro_name": target}
|
||||
for line in results.stdout.split("\n"):
|
||||
cache_data[target]["headers"] += [line]
|
||||
|
||||
if symlink_query is not None:
|
||||
target_results = env.RunBazelQuery(symlink_query, f"getting macro name for {target}")
|
||||
cache_data[target]["macro_name"] = target_results.stdout.split(" ")[0]
|
||||
|
||||
with open(cache_file, "w") as f:
|
||||
json.dump(cache_data, f)
|
||||
|
||||
return cache_data[target]["headers"], cache_data[target]["macro_name"]
|
||||
|
||||
|
||||
def add_headers_from_all_libraries(env, header_map):
|
||||
bazel_query = ["aquery"] + env["BAZEL_FLAGS_STR"] + ['mnemonic("CppArchive", //src/...)']
|
||||
results = env.RunBazelQuery(bazel_query, "getting all bazel libraries")
|
||||
targets = set()
|
||||
for line in results.stdout.split("\n"):
|
||||
if " Target: //src" in line:
|
||||
target = line.split(" Target: ")[-1]
|
||||
targets.add(target)
|
||||
|
||||
for target in targets:
|
||||
header_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f'labels(hdrs, "@{target}")',
|
||||
"--output",
|
||||
"files",
|
||||
]
|
||||
)
|
||||
macro_name_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f'kind("extract_debuginfo", rdeps(@//src/mongo/..., "@{target}", 1))',
|
||||
]
|
||||
)
|
||||
headers, macro_name = get_target_headers(env, target, header_query, macro_name_query)
|
||||
header_map[macro_name] = [hdr for hdr in headers if not hdr.endswith("src/mongo/config.h")]
|
||||
|
||||
|
||||
def add_headers_from_gen_code(env, header_map):
|
||||
source_generators_query = (
|
||||
["aquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ ['mnemonic("IdlcGenerator|TemplateRenderer|ConfigHeaderGen", //src/...)']
|
||||
)
|
||||
|
||||
idl_gen_targets = set()
|
||||
results = env.RunBazelQuery(source_generators_query, "getting all idl gen targets")
|
||||
for line in results.stdout.split("\n"):
|
||||
if " Target: //src" in line:
|
||||
target = line.split(" Target: ")[-1]
|
||||
idl_gen_targets.add(target)
|
||||
|
||||
for target in idl_gen_targets:
|
||||
header_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f"@{target}",
|
||||
"--output",
|
||||
"files",
|
||||
]
|
||||
)
|
||||
headers, macro_name = get_target_headers(env, target, header_query)
|
||||
header_map[macro_name] = [
|
||||
hdr for hdr in headers if hdr.endswith(target.split(":")[-1] + ".h")
|
||||
]
|
||||
|
||||
source_generators_query = (
|
||||
["aquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ ['mnemonic("TemplateRenderer|ConfigHeaderGen", //src/...)']
|
||||
)
|
||||
|
||||
source_gen_targets = set()
|
||||
results = env.RunBazelQuery(source_generators_query, "getting all source gen targets")
|
||||
for line in results.stdout.split("\n"):
|
||||
if " Target: //src" in line:
|
||||
target = line.split(" Target: ")[-1]
|
||||
source_gen_targets.add(target)
|
||||
|
||||
for target in source_gen_targets:
|
||||
header_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f"@{target}",
|
||||
"--output",
|
||||
"files",
|
||||
]
|
||||
)
|
||||
headers, macro_name = get_target_headers(env, target, header_query)
|
||||
header_map[macro_name] = [
|
||||
hdr for hdr in headers if hdr.endswith(".h") and not hdr.endswith("src/mongo/config.h")
|
||||
]
|
||||
|
||||
|
||||
def bazel_includes_emitter(target, source, env):
|
||||
target_library = env.GetOption("bazel-includes-info").replace("\\", "/")
|
||||
rel_target = os.path.relpath(str(target[0].abspath), start=env.Dir("#").abspath).replace(
|
||||
"\\", "/"
|
||||
)
|
||||
|
||||
if rel_target == target_library:
|
||||
objsuffix = (
|
||||
env.subst("$OBJSUFFIX") if not env.TargetOSIs("linux") else env.subst("$SHOBJSUFFIX")
|
||||
)
|
||||
builder_name = "StaticLibrary" if not env.TargetOSIs("linux") == "nt" else "SharedLibrary"
|
||||
os.makedirs(os.path.dirname(str(target[0].abspath)), exist_ok=True)
|
||||
with open(str(target[0].abspath) + ".obj_files", "w") as f:
|
||||
for s in source:
|
||||
if str(s).endswith(objsuffix):
|
||||
f.write(os.path.relpath(str(s.abspath), start=env.Dir("#").abspath) + "\n")
|
||||
with open(str(target[0].abspath) + ".env_vars", "w") as f:
|
||||
json.dump(env["ENV"], f)
|
||||
|
||||
with (
|
||||
open(str(target[0].abspath) + ".bazel_headers", "w") as fheaders,
|
||||
open(str(target[0].abspath) + ".bazel_deps", "w") as fdeps,
|
||||
):
|
||||
# note we can't know about LIBDEPS_DEPDENDENTS (reverse deps) in an emitter
|
||||
# however we do co-opt the libdeps linter to check for these at the end of reading
|
||||
# sconscripts
|
||||
|
||||
deps = []
|
||||
for s in (
|
||||
env.get("LIBDEPS", [])
|
||||
+ env.get("LIBDEPS_PRIVATE", [])
|
||||
+ env.get("LIBDEPS_INTERFACE", [])
|
||||
):
|
||||
if not s:
|
||||
continue
|
||||
|
||||
libnode = libdeps_tool._get_node_with_ixes(env, s, builder_name)
|
||||
|
||||
libnode_path = os.path.relpath(
|
||||
str(libnode.abspath), start=env.Dir("#").abspath
|
||||
).replace("\\", "/")
|
||||
if libnode.has_builder() and libnode.get_builder().get_name(env) != "ThinTarget":
|
||||
print(
|
||||
f"ERROR: can generate correct bazel header list because {target[0]} has non-bazel dependency: {libnode}"
|
||||
)
|
||||
sys.exit(1)
|
||||
if str(libnode_path) in env["SCONS2BAZEL_TARGETS"].scons2bazel_targets:
|
||||
bazel_target = env["SCONS2BAZEL_TARGETS"].bazel_target(str(libnode_path))
|
||||
# new query to run, run and cache it
|
||||
deps.append(bazel_target)
|
||||
bazel_query = (
|
||||
["cquery"]
|
||||
+ env["BAZEL_FLAGS_STR"]
|
||||
+ [
|
||||
f'filter("[\\.h,\\.ipp,\\.hpp].*$", kind("source", deps("@{bazel_target}")))',
|
||||
"--output",
|
||||
"files",
|
||||
]
|
||||
)
|
||||
results = env.RunBazelQuery(bazel_query, "getting bazel headers")
|
||||
|
||||
if results.returncode != 0:
|
||||
print("ERROR: bazel libdeps query failed:")
|
||||
print(results)
|
||||
sys.exit(1)
|
||||
results = set(
|
||||
[line for line in results.stdout.split("\n") if line.startswith("src/")]
|
||||
)
|
||||
|
||||
for header in results:
|
||||
fheaders.write(header + "\n")
|
||||
for dep in deps:
|
||||
fdeps.write(dep + "\n")
|
||||
|
||||
return target, source
|
||||
|
||||
|
||||
def generate(env):
|
||||
header_map = {}
|
||||
add_headers_from_all_libraries(env, header_map)
|
||||
gen_header_map = {}
|
||||
add_headers_from_gen_code(env, gen_header_map)
|
||||
|
||||
bazel_include_info = {
|
||||
"header_map": header_map,
|
||||
"gen_header_map": gen_header_map,
|
||||
"bazel_exec": env["SCONS2BAZEL_TARGETS"].bazel_executable,
|
||||
"config": env["BAZEL_FLAGS_STR"] + ["--config=local"],
|
||||
}
|
||||
|
||||
with open(".bazel_include_info.json", "w") as f:
|
||||
json.dump(bazel_include_info, f)
|
||||
|
||||
for builder_name in ["SharedLibrary", "StaticLibrary", "Program"]:
|
||||
builder = env["BUILDERS"][builder_name]
|
||||
base_emitter = builder.emitter
|
||||
new_emitter = SCons.Builder.ListEmitter([base_emitter, bazel_includes_emitter])
|
||||
builder.emitter = new_emitter
|
||||
Loading…
Reference in New Issue