diff --git a/manager/cmd/manager.py b/manager/cmd/manager.py index 8f088fbf..3562fd61 100644 --- a/manager/cmd/manager.py +++ b/manager/cmd/manager.py @@ -8,6 +8,7 @@ binary_finder, cache_query, cli_config, + compile_commands, create_dev_env, create_env, develop, @@ -46,6 +47,7 @@ def setup_parser(subparser): analyze.add_command(sp, _subcommands) binary_finder.add_command(sp, _subcommands) cache_query.add_command(sp, _subcommands) + compile_commands.add_command(sp, _subcommands) create_env.add_command(sp, _subcommands) create_dev_env.add_command(sp, _subcommands) develop.add_command(sp, _subcommands) diff --git a/manager/manager_cmds/compile_commands.py b/manager/manager_cmds/compile_commands.py new file mode 100644 index 00000000..5b72e217 --- /dev/null +++ b/manager/manager_cmds/compile_commands.py @@ -0,0 +1,179 @@ +import concurrent.futures +import json +import os +import re +import sys +import time +from contextlib import contextmanager +from functools import partial + +import spack +import spack.builder +from spack import environment + +""" +This extension was contributed by vbrunini +""" + + +command_name = "compile-commands" +description = ( + "Copy compile_commands.json for all develop packages in the active environment," + " then remove cross-package -isystem flags and substitute source/build -I flags." +) +section = "sierra" +level = "long" +aliases = [] + + +def setup_parser(subparser): + subparser.add_argument( + "--serial", action="store_true", help="Process compile_commands.json files in serial." + ) + + +@contextmanager +def timer(name: str, args): + start = time.perf_counter() + yield + end = time.perf_counter() + if args.verbose: + print(f"[{name}] elapsed: {end-start:.3f} s") + + +def build_dir(builder, pkg): + if hasattr(builder, "build_directory"): + return os.path.normpath(os.path.join(pkg.stage.path, builder.build_directory)) + return pkg.stage.source_path + + +def source_root(pkg): + if hasattr(pkg, "root_cmakelists_dir"): + return os.path.join(pkg.stage.source_path, pkg.root_cmakelists_dir) + return pkg.stage.source_path + + +def _process_spec(spec, args): + """Worker that handles a single spec and returns (info, (regex, includes)).""" + + if not spec.is_develop: + return None + + pkg = spec.package + if args.verbose: + sys.stdout.write(f"\nProcessing {pkg.name}\n") + try: + builder = spack.builder.create(pkg) + src_cc_path = os.path.join(build_dir(builder, pkg), "compile_commands.json") + except Exception: + return None + + if not os.path.exists(src_cc_path): + return None + + dst_dir = pkg.stage.source_path + if getattr(pkg, "root_cmakelists_dir", None): + dst_dir = os.path.join(dst_dir, pkg.root_cmakelists_dir) + os.makedirs(dst_dir, exist_ok=True) + dest_cc_path = os.path.join(dst_dir, "compile_commands.json") + + if os.path.islink(dest_cc_path): + if args.verbose: + sys.stdout.write(f" removing existing symlink {dest_cc_path}\n") + os.unlink(dest_cc_path) + + try: + with open(src_cc_path) as f: + cc_entries = json.load(f) + except Exception: + cc_entries = [] + + pkg_build_dir = build_dir(builder, pkg) + pkg_src_root = source_root(pkg) + + source_include_flags = [] + source_include_re = re.compile(rf"-I(?:{pkg_src_root}|{pkg_build_dir})\S*") + for entry in cc_entries: + if "command" not in entry: + continue + for match in source_include_re.findall(entry["command"]): + if match not in source_include_flags: + source_include_flags.append(match) + + replacement_pattern = rf"-isystem\s+{pkg.prefix}\S*" + if args.verbose: + sys.stdout.write(f" replacement pattern = {replacement_pattern}\n") + sys.stdout.write(f" source include flags = {source_include_flags}\n") + + info = {"cc_entries": cc_entries, "dest_cc_path": dest_cc_path} + repl = (re.compile(replacement_pattern), " ".join(source_include_flags)) + return (info, repl) + + +def _apply_replacements(info, include_path_replacements): + cc_entries = info["cc_entries"] + dest_cc_path = info["dest_cc_path"] + + for entry in cc_entries: + if "command" not in entry: + continue + command = entry["command"] + for regex, source_includes in include_path_replacements: + # First replace with the gathered -I flags, then strip any leftover -isystem + command = regex.sub(source_includes, command, 1) + command = regex.sub("", command) + entry["command"] = command + + # Write out the (now cleaned) compile_commands.json + with open(dest_cc_path, "w") as f: + json.dump(cc_entries, f, indent=2) + f.write("\n") + return dest_cc_path + + +def _get_executor(args): + if args.serial: + return concurrent.futures.ThreadPoolExecutor(max_workers=1) + + return concurrent.futures.ProcessPoolExecutor() + + +def compile_commands(parser, args): + env = environment.active_environment() + + # First pass: copy each compile_commands.json into its source dir + infos = [] + include_path_replacements = [] + with timer("Determine include path replacements", args): + with _get_executor(args) as exe: + futures = [exe.submit(_process_spec, spec, args) for spec in env.all_specs()] + for fut in concurrent.futures.as_completed(futures): + result = fut.result() + if result is None: + continue + info, repl = result + infos.append(info) + include_path_replacements.append(repl) + + # Second pass: Apply isystem removal regexes and insert corresponding source include flags if + # they were applied + with timer("Apply include path replacements", args): + with _get_executor(args) as exe: + list( + exe.map( + partial( + _apply_replacements, include_path_replacements=include_path_replacements + ), + infos, + ) + ) + + +def add_command(parser, command_dict): + sub_parser = parser.add_parser( + command_name, help=description, description=description, aliases=aliases + ) + setup_parser(sub_parser) + command_dict[command_name] = compile_commands + for alias in aliases: + command_dict[alias] = compile_commands diff --git a/tests/test_compile_commands.py b/tests/test_compile_commands.py new file mode 100644 index 00000000..302adfff --- /dev/null +++ b/tests/test_compile_commands.py @@ -0,0 +1,253 @@ +import json +import os +import shlex +from unittest import mock + +import pytest + +import spack.builder +import spack.environment +import spack.main + +manager = spack.main.SpackCommand("manager") + + +class DummyPkg: + def __init__(self, name, build_root, src_root, prefix, root_cmakelists_dir=None): + self.name = name + self.stage = mock.Mock() + self.stage.path = str(build_root) + self.stage.source_path = str(src_root) + self.prefix = str(prefix) + if root_cmakelists_dir is not None: + self.root_cmakelists_dir = root_cmakelists_dir + + +class DummyBuilder: + def __init__(self, build_directory): + self.build_directory = build_directory + + +@pytest.fixture +def mock_environment_and_package(tmp_path, monkeypatch): + """ + Single‐package fixture, with root_cmakelists_dir='bar'. + Yields (pkg, path_to_original_compile_commands.json). + """ + build_root = tmp_path / "build" + src_root = tmp_path / "source" + inst_root = tmp_path / "install" + + # make build_dir and source/bar + (build_root / "build_dir").mkdir(parents=True) + (src_root / "bar").mkdir(parents=True) + inst_root.mkdir() + + # write an empty compile_commands.json in build_dir + cc_in = build_root / "build_dir" / "compile_commands.json" + with open(cc_in, "w") as f: + f.write("{}") + + # create pkg/spec/env + pkg = DummyPkg("foo", build_root, src_root, inst_root, root_cmakelists_dir="bar") + spec = mock.Mock(package=pkg) + fake_env = mock.Mock(all_specs=mock.Mock(return_value=[spec])) + + # patch active_environment() and builder.create(...) + monkeypatch.setattr(spack.environment, "active_environment", lambda: fake_env) + monkeypatch.setattr(spack.builder, "create", lambda pkg: DummyBuilder("build_dir")) + + return pkg, str(cc_in) + + +def test_compile_commands_copies_file(mock_environment_and_package): + pkg, cc_in = mock_environment_and_package + manager("compile-commands", "--serial") + dest = os.path.join(pkg.stage.source_path, "bar", "compile_commands.json") + assert os.path.exists(dest) + + # cleanup + os.remove(cc_in) + os.remove(dest) + os.rmdir(os.path.dirname(cc_in)) + + +def test_compile_commands_symlink_replacement(mock_environment_and_package): + pkg, cc_in = mock_environment_and_package + dest = os.path.join(pkg.stage.source_path, "bar", "compile_commands.json") + + # create a symlink at the destination + os.symlink("/some/other/path", dest) + manager("compile-commands", "--serial") + + assert os.path.exists(dest) + assert not os.path.islink(dest) + + # cleanup + os.remove(cc_in) + os.remove(dest) + os.rmdir(os.path.dirname(cc_in)) + + +def test_compile_commands_no_source_file(tmp_path, monkeypatch): + # pkg with no compile_commands.json in build_dir + build_root = tmp_path / "build" + src_root = tmp_path / "source" + inst_root = tmp_path / "install" + + (src_root / "bar").mkdir(parents=True) + inst_root.mkdir() + + pkg = DummyPkg("foo", build_root, src_root, inst_root, root_cmakelists_dir="bar") + spec = mock.Mock(package=pkg) + fake_env = mock.Mock(all_specs=mock.Mock(return_value=[spec])) + + monkeypatch.setattr(spack.environment, "active_environment", lambda: fake_env) + monkeypatch.setattr(spack.builder, "create", lambda pkg: DummyBuilder("build_dir")) + + manager("compile-commands", "--serial") + dest = os.path.join(pkg.stage.source_path, "bar", "compile_commands.json") + assert not os.path.exists(dest) + + +def test_compile_commands_no_root_cmakelists_dir(mock_environment_and_package): + pkg, cc_in = mock_environment_and_package + # remove the attribute + if hasattr(pkg, "root_cmakelists_dir"): + del pkg.root_cmakelists_dir + + manager("compile-commands", "--serial") + dest = os.path.join(pkg.stage.source_path, "compile_commands.json") + assert os.path.exists(dest) + + # cleanup + os.remove(cc_in) + os.remove(dest) + os.rmdir(os.path.dirname(cc_in)) + + +@pytest.fixture +def two_pkg_env(tmp_path, monkeypatch): + """ + Two‐package fixture: pkg_foo and pkg_bar, + each with a compile_commands.json in build_dir. + bar’s file refers via -isystem to something under foo.prefix. + """ + # foo + build_foo = tmp_path / "build_foo" + (build_foo / "build_dir").mkdir(parents=True) + src_foo = tmp_path / "src_foo" + src_foo.mkdir() + inst_foo = tmp_path / "inst_foo" + inst_foo.mkdir() + (src_foo / "inc").mkdir() + (build_foo / "build_dir" / "bld_inc").mkdir(parents=True) + + foo_cmd = "clang -I{} -I{} -c foo.c".format( + src_foo / "inc", build_foo / "build_dir" / "bld_inc" + ) + foo_entries = [{"directory": str(build_foo), "command": foo_cmd, "file": "foo.c"}] + foo_cc = build_foo / "build_dir" / "compile_commands.json" + with open(foo_cc, "w") as f: + json.dump(foo_entries, f) + + # bar + build_bar = tmp_path / "build_bar" + (build_bar / "build_dir").mkdir(parents=True) + src_bar = tmp_path / "src_bar" + src_bar.mkdir() + inst_bar = tmp_path / "inst_bar" + inst_bar.mkdir() + + # create an include dir under foo’s install prefix + foo_inc_inst = inst_foo / "foo_inc_dir" + foo_inc_inst.mkdir() + + bar_cmd = ( + f"clang -I{src_bar} -isystem {foo_inc_inst} -isystem {foo_inc_inst}/subdir_1 " + "-isystem {foo_inc_inst}/subdir_2 -c bar.c" + ) + bar_entries = [{"directory": str(build_bar), "command": bar_cmd, "file": "bar.c"}] + bar_cc = build_bar / "build_dir" / "compile_commands.json" + with open(bar_cc, "w") as f: + json.dump(bar_entries, f) + + # create pkgs/specs/env + pkg_foo = DummyPkg("foo", build_foo, src_foo, inst_foo) + pkg_bar = DummyPkg("bar", build_bar, src_bar, inst_bar) + spec_foo = mock.Mock(package=pkg_foo) + spec_bar = mock.Mock(package=pkg_bar) + fake_env = mock.Mock(all_specs=mock.Mock(return_value=[spec_foo, spec_bar])) + + monkeypatch.setattr(spack.environment, "active_environment", lambda: fake_env) + monkeypatch.setattr(spack.builder, "create", lambda pkg: DummyBuilder("build_dir")) + + return pkg_foo, pkg_bar + + +def test_strip_isystem_and_inject_I(two_pkg_env): + pkg_foo, pkg_bar = two_pkg_env + manager("compile-commands", "--serial") + + # foo’s file unchanged (except no -isystem): + foo_dst = os.path.join(pkg_foo.stage.source_path, "compile_commands.json") + data = json.load(open(foo_dst)) + toklist = shlex.split(data[0]["command"]) + assert "-I" + os.path.join(pkg_foo.stage.source_path, "inc") in toklist + assert "-I" + os.path.join(pkg_foo.stage.path, "build_dir", "bld_inc") in toklist + assert all(not t.startswith("-isystem") for t in toklist) + + # bar’s file: -isystem removed, foo’s -I flags injected + bar_dst = os.path.join(pkg_bar.stage.source_path, "compile_commands.json") + data = json.load(open(bar_dst)) + toklist = shlex.split(data[0]["command"]) + assert all(not t.startswith("-isystem") for t in toklist) + assert "-I" + os.path.join(pkg_foo.stage.source_path, "inc") in toklist + assert "-I" + os.path.join(pkg_foo.stage.path, "build_dir", "bld_inc") in toklist + assert "-I" + os.path.join(pkg_foo.stage.source_path, "inc") in toklist + + +def test_no_cross_inject_when_no_isystem(tmp_path, monkeypatch): + # Build two packages, each only with its own -I under source + build1 = tmp_path / "b1" + (build1 / "build_dir").mkdir(parents=True) + src1 = tmp_path / "s1" + src1.mkdir() + inst1 = tmp_path / "i1" + inst1.mkdir() + inc1 = src1 / "inc1" + inc1.mkdir() + cc1 = build1 / "build_dir" / "compile_commands.json" + with open(cc1, "w") as f: + json.dump([{"directory": str(build1), "command": f"cc -I{inc1} -c a.c", "file": "a.c"}], f) + + build2 = tmp_path / "b2" + (build2 / "build_dir").mkdir(parents=True) + src2 = tmp_path / "s2" + src2.mkdir() + inst2 = tmp_path / "i2" + inst2.mkdir() + inc2 = src2 / "inc2" + inc2.mkdir() + cc2 = build2 / "build_dir" / "compile_commands.json" + with open(cc2, "w") as f: + json.dump([{"directory": str(build2), "command": f"cc -I{inc2} -c b.c", "file": "b.c"}], f) + + pkg1 = DummyPkg("one", build1, src1, inst1) + pkg2 = DummyPkg("two", build2, src2, inst2) + spec1 = mock.Mock(package=pkg1) + spec2 = mock.Mock(package=pkg2) + fake_env = mock.Mock(all_specs=mock.Mock(return_value=[spec1, spec2])) + + monkeypatch.setattr(spack.environment, "active_environment", lambda: fake_env) + monkeypatch.setattr(spack.builder, "create", lambda pkg: DummyBuilder("build_dir")) + + manager("compile-commands", "--serial") + + for pkg in (pkg1, pkg2): + dst = os.path.join(pkg.stage.source_path, "compile_commands.json") + data = json.load(open(dst)) + toks = shlex.split(data[0]["command"]) + # must have its own -I but no -isystem, and no other injections + assert any(t.startswith("-I" + pkg.stage.source_path) for t in toks) + assert all(not t.startswith("-isystem") for t in toks)