From 49880e5f23f2687e82c66218e1ea947f3ceab3d0 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Wed, 22 Dec 2021 14:14:47 -0500 Subject: [PATCH] Squashed 'wrap/' changes from 2cbaf7a3a..3e076c9ac 3e076c9ac Merge pull request #141 from borglab/feature/compile-on-change 055d6ad3f add dependencies to specialization and preamble files 497b330e8 address review comments 8e233b644 use custom command instead of custom target 238954eaa minor fix b709f512b update cmake to make multiple wrapping calls for each interface file a1b9ffaaf pybind wrapper updates to handle submodules separately from the main python module git-subtree-dir: wrap git-subtree-split: 3e076c9ace04fea946124d586a01c2e9b8a32bdc --- cmake/PybindWrap.cmake | 54 ++++++++++++++++++++++++++-------------- gtwrap/pybind_wrapper.py | 45 +++++++++++++++++++++++---------- scripts/pybind_wrap.py | 19 +++++++++----- 3 files changed, 80 insertions(+), 38 deletions(-) diff --git a/cmake/PybindWrap.cmake b/cmake/PybindWrap.cmake index 2149c7195..2008bf2dd 100644 --- a/cmake/PybindWrap.cmake +++ b/cmake/PybindWrap.cmake @@ -55,15 +55,44 @@ function( set(GTWRAP_PATH_SEPARATOR ";") endif() + # Create a copy of interface_headers so we can freely manipulate it + set(interface_files ${interface_headers}) + + # Pop the main interface file so that interface_files has only submodules. + list(POP_FRONT interface_files main_interface) + # Convert .i file names to .cpp file names. - foreach(filepath ${interface_headers}) - get_filename_component(interface ${filepath} NAME) - string(REPLACE ".i" ".cpp" cpp_file ${interface}) + foreach(interface_file ${interface_files}) + # This block gets the interface file name and does the replacement + get_filename_component(interface ${interface_file} NAME_WLE) + set(cpp_file "${interface}.cpp") list(APPEND cpp_files ${cpp_file}) + + # Wrap the specific interface header + # This is done so that we can create CMake dependencies in such a way so that when changing a single .i file, + # the others don't need to be regenerated. + # NOTE: We have to use `add_custom_command` so set the dependencies correctly. + # https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes + add_custom_command( + OUTPUT ${cpp_file} + COMMAND + ${CMAKE_COMMAND} -E env + "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}" + ${PYTHON_EXECUTABLE} ${PYBIND_WRAP_SCRIPT} --src "${interface_file}" + --out "${cpp_file}" --module_name ${module_name} + --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} + --template ${module_template} --is_submodule ${_WRAP_BOOST_ARG} + DEPENDS "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h" + VERBATIM) + endforeach() + get_filename_component(main_interface_name ${main_interface} NAME_WLE) + set(main_cpp_file "${main_interface_name}.cpp") + list(PREPEND cpp_files ${main_cpp_file}) + add_custom_command( - OUTPUT ${cpp_files} + OUTPUT ${main_cpp_file} COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}" @@ -71,23 +100,10 @@ function( --out "${generated_cpp}" --module_name ${module_name} --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} --template ${module_template} ${_WRAP_BOOST_ARG} - DEPENDS "${interface_headers}" ${module_template} + DEPENDS "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h" VERBATIM) - add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${cpp_files}) - - # Late dependency injection, to make sure this gets called whenever the - # interface header or the wrap library are updated. - # ~~~ - # See: https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes - # ~~~ - add_custom_command( - OUTPUT ${cpp_files} - DEPENDS ${interface_headers} - # @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py - # @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py - # @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py - APPEND) + add_custom_target(pybind_wrap_${module_name} DEPENDS ${cpp_files}) pybind11_add_module(${target} "${cpp_files}") diff --git a/gtwrap/pybind_wrapper.py b/gtwrap/pybind_wrapper.py index cf89251b5..7b7512e4d 100755 --- a/gtwrap/pybind_wrapper.py +++ b/gtwrap/pybind_wrapper.py @@ -631,28 +631,47 @@ class PybindWrapper: submodules_init="\n".join(submodules_init), ) - def wrap(self, sources, main_output): + def wrap_submodule(self, source): """ - Wrap all the source interface files. + Wrap a list of submodule files, i.e. a set of interface files which are + in support of a larger wrapping project. + + E.g. This is used in GTSAM where we have a main gtsam.i, but various smaller .i files + which are the submodules. + The benefit of this scheme is that it reduces compute and memory usage during compilation. + + Args: + source: Interface file which forms the submodule. + """ + filename = Path(source).name + module_name = Path(source).stem + + # Read in the complete interface (.i) file + with open(source, "r") as f: + content = f.read() + # Wrap the read-in content + cc_content = self.wrap_file(content, module_name=module_name) + + # Generate the C++ code which Pybind11 will use. + with open(filename.replace(".i", ".cpp"), "w") as f: + f.write(cc_content) + + def wrap(self, sources, main_module_name): + """ + Wrap all the main interface file. Args: sources: List of all interface files. - main_output: The name for the main module. + The first file should be the main module. + main_module_name: The name for the main module. """ main_module = sources[0] + + # Get all the submodule names. submodules = [] for source in sources[1:]: - filename = Path(source).name module_name = Path(source).stem - # Read in the complete interface (.i) file - with open(source, "r") as f: - content = f.read() submodules.append(module_name) - cc_content = self.wrap_file(content, module_name=module_name) - - # Generate the C++ code which Pybind11 will use. - with open(filename.replace(".i", ".cpp"), "w") as f: - f.write(cc_content) with open(main_module, "r") as f: content = f.read() @@ -661,5 +680,5 @@ class PybindWrapper: submodules=submodules) # Generate the C++ code which Pybind11 will use. - with open(main_output, "w") as f: + with open(main_module_name, "w") as f: f.write(cc_content) diff --git a/scripts/pybind_wrap.py b/scripts/pybind_wrap.py index c82a1d24c..577060243 100644 --- a/scripts/pybind_wrap.py +++ b/scripts/pybind_wrap.py @@ -19,7 +19,7 @@ def main(): arg_parser.add_argument("--src", type=str, required=True, - help="Input interface .i/.h file") + help="Input interface .i/.h file(s)") arg_parser.add_argument( "--module_name", type=str, @@ -31,7 +31,7 @@ def main(): "--out", type=str, required=True, - help="Name of the output pybind .cc file", + help="Name of the output pybind .cc file(s)", ) arg_parser.add_argument( "--use-boost", @@ -60,7 +60,10 @@ def main(): ) arg_parser.add_argument("--template", type=str, - help="The module template file") + help="The module template file (e.g. module.tpl).") + arg_parser.add_argument("--is_submodule", + default=False, + action="store_true") args = arg_parser.parse_args() top_module_namespaces = args.top_module_namespaces.split("::") @@ -78,9 +81,13 @@ def main(): module_template=template_content, ) - # Wrap the code and get back the cpp/cc code. - sources = args.src.split(';') - wrapper.wrap(sources, args.out) + if args.is_submodule: + wrapper.wrap_submodule(args.src) + + else: + # Wrap the code and get back the cpp/cc code. + sources = args.src.split(';') + wrapper.wrap(sources, args.out) if __name__ == "__main__":