diff --git a/wrap/cmake/PybindWrap.cmake b/wrap/cmake/PybindWrap.cmake index 331dfff8c..f341c2f98 100644 --- a/wrap/cmake/PybindWrap.cmake +++ b/wrap/cmake/PybindWrap.cmake @@ -13,15 +13,14 @@ gtwrap_get_python_version(${WRAP_PYTHON_VERSION}) message(STATUS "Setting Python version for wrapper") set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION}) -# User-friendly Pybind11 wrapping and installing function. -# Builds a Pybind11 module from the provided interface_header. -# For example, for the interface header gtsam.h, this will -# build the wrap module 'gtsam_py.cc'. +# User-friendly Pybind11 wrapping and installing function. Builds a Pybind11 +# module from the provided interface_headers. For example, for the interface +# header gtsam.h, this will build the wrap module 'gtsam_py.cc'. # # Arguments: # ~~~ # target: The Make target -# interface_header: The relative path to the wrapper interface definition file. +# interface_headers: List of paths to the wrapper interface definition files. The top level interface file should be first. # generated_cpp: The name of the cpp file which is generated from the tpl file. # module_name: The name of the Python module to use. # top_namespace: The C++ namespace under which the code to be wrapped exists. @@ -31,16 +30,17 @@ set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION}) # libs: Libraries to link with. # dependencies: Dependencies which need to be built before the wrapper. # use_boost (optional): Flag indicating whether to include Boost. -function(pybind_wrap - target - interface_header - generated_cpp - module_name - top_namespace - ignore_classes - module_template - libs - dependencies) +function( + pybind_wrap + target + interface_headers + generated_cpp + module_name + top_namespace + ignore_classes + module_template + libs + dependencies) set(ExtraMacroArgs ${ARGN}) list(GET ExtraMacroArgs 0 USE_BOOST) if(USE_BOOST) @@ -49,57 +49,62 @@ function(pybind_wrap set(_WRAP_BOOST_ARG "") endif(USE_BOOST) - if (UNIX) + if(UNIX) set(GTWRAP_PATH_SEPARATOR ":") else() set(GTWRAP_PATH_SEPARATOR ";") endif() - add_custom_command(OUTPUT ${generated_cpp} - COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}" - ${PYTHON_EXECUTABLE} - ${PYBIND_WRAP_SCRIPT} - --src - ${interface_header} - --out - ${generated_cpp} - --module_name - ${module_name} - --top_module_namespaces - "${top_namespace}" - --ignore - ${ignore_classes} - --template - ${module_template} - ${_WRAP_BOOST_ARG} - DEPENDS ${interface_header} ${module_template} - VERBATIM) - add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${generated_cpp}) + # Convert .i file names to .cpp file names. + foreach(filepath ${interface_headers}) + get_filename_component(interface ${filepath} NAME) + string(REPLACE ".i" ".cpp" cpp_file ${interface}) + list(APPEND cpp_files ${cpp_file}) + endforeach() + + add_custom_command( + OUTPUT ${cpp_files} + COMMAND + ${CMAKE_COMMAND} -E env + "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}" + ${PYTHON_EXECUTABLE} ${PYBIND_WRAP_SCRIPT} --src "${interface_headers}" + --out "${generated_cpp}" --module_name ${module_name} + --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} + --template ${module_template} ${_WRAP_BOOST_ARG} + DEPENDS "${interface_headers}" ${module_template} + VERBATIM) + + add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${cpp_files}) # Late dependency injection, to make sure this gets called whenever the # interface header or the wrap library are updated. # ~~~ # See: https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes # ~~~ - add_custom_command(OUTPUT ${generated_cpp} - DEPENDS ${interface_header} - # @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py - # @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py - # @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py - APPEND) + add_custom_command( + OUTPUT ${cpp_files} + DEPENDS ${interface_headers} + # @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py + # @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py + # @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py + APPEND) - pybind11_add_module(${target} ${generated_cpp}) + pybind11_add_module(${target} "${cpp_files}") if(APPLE) - # `type_info` objects will become "weak private external" if the templated class is initialized implicitly even if we explicitly - # export them with `WRAP_EXPORT`. If that happens, the `type_info` for the same templated class will diverge between shared - # libraries, causing `dynamic_cast` to fail. This is mitigated by telling Clang to mimic the MSVC behavior. - # See https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2 + # `type_info` objects will become "weak private external" if the templated + # class is initialized implicitly even if we explicitly export them with + # `WRAP_EXPORT`. If that happens, the `type_info` for the same templated + # class will diverge between shared libraries, causing `dynamic_cast` to + # fail. This is mitigated by telling Clang to mimic the MSVC behavior. See + # https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2 # https://github.com/CppMicroServices/CppMicroServices/pull/82/files # https://www.russellmcc.com/posts/2013-08-03-rtti.html target_compile_options(${target} PRIVATE "-fvisibility-ms-compat") endif() + add_dependencies(${target} pybind_wrap_${module_name}) + if(NOT "${libs}" STREQUAL "") target_link_libraries(${target} PRIVATE "${libs}") endif() @@ -121,10 +126,7 @@ endfunction() # dest_directory: The destination directory to install to. # patterns: list of file patterns to install # ~~~ -function(install_python_scripts - source_directory - dest_directory - patterns) +function(install_python_scripts source_directory dest_directory patterns) set(patterns_args "") set(exclude_patterns "") @@ -144,17 +146,19 @@ function(install_python_scripts # there is one get_filename_component(location "${dest_directory}" PATH) get_filename_component(name "${dest_directory}" NAME) - install(DIRECTORY "${source_directory}" - DESTINATION "${location}/${name}${build_type_tag}" - CONFIGURATIONS "${build_type}" - FILES_MATCHING ${patterns_args} - PATTERN "${exclude_patterns}" EXCLUDE) + install( + DIRECTORY "${source_directory}" + DESTINATION "${location}/${name}${build_type_tag}" + CONFIGURATIONS "${build_type}" + FILES_MATCHING ${patterns_args} + PATTERN "${exclude_patterns}" EXCLUDE) endforeach() else() - install(DIRECTORY "${source_directory}" - DESTINATION "${dest_directory}" - FILES_MATCHING ${patterns_args} - PATTERN "${exclude_patterns}" EXCLUDE) + install( + DIRECTORY "${source_directory}" + DESTINATION "${dest_directory}" + FILES_MATCHING ${patterns_args} + PATTERN "${exclude_patterns}" EXCLUDE) endif() endfunction() @@ -172,13 +176,14 @@ function(install_python_files source_files dest_directory) foreach(build_type ${CMAKE_CONFIGURATION_TYPES}) string(TOUPPER "${build_type}" build_type_upper) set(build_type_tag "") - # Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if - # there is one + # Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if there + # is one get_filename_component(location "${dest_directory}" PATH) get_filename_component(name "${dest_directory}" NAME) - install(FILES "${source_files}" - DESTINATION "${location}/${name}${build_type_tag}" - CONFIGURATIONS "${build_type}") + install( + FILES "${source_files}" + DESTINATION "${location}/${name}${build_type_tag}" + CONFIGURATIONS "${build_type}") endforeach() else() install(FILES "${source_files}" DESTINATION "${dest_directory}") @@ -194,18 +199,19 @@ function(create_symlinks source_folder dest_folder) return() endif() - file(GLOB files - LIST_DIRECTORIES true - RELATIVE "${source_folder}" - "${source_folder}/*") + file( + GLOB files + LIST_DIRECTORIES true + RELATIVE "${source_folder}" + "${source_folder}/*") foreach(path_file ${files}) get_filename_component(folder ${path_file} PATH) get_filename_component(ext ${path_file} EXT) set(ignored_ext ".tpl" ".h") - list (FIND ignored_ext "${ext}" _index) - if (${_index} GREATER -1) + list(FIND ignored_ext "${ext}" _index) + if(${_index} GREATER -1) continue() - endif () + endif() # Create REAL folder file(MAKE_DIRECTORY "${dest_folder}") @@ -224,9 +230,10 @@ function(create_symlinks source_folder dest_folder) endif() # cmake-format: on - execute_process(COMMAND ${command} - RESULT_VARIABLE result - ERROR_VARIABLE output) + execute_process( + COMMAND ${command} + RESULT_VARIABLE result + ERROR_VARIABLE output) if(NOT ${result} EQUAL 0) message( diff --git a/wrap/gtwrap/pybind_wrapper.py b/wrap/gtwrap/pybind_wrapper.py index 0e1b3c7ea..40571263a 100755 --- a/wrap/gtwrap/pybind_wrapper.py +++ b/wrap/gtwrap/pybind_wrapper.py @@ -13,6 +13,7 @@ Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellae # pylint: disable=too-many-arguments, too-many-instance-attributes, no-self-use, no-else-return, too-many-arguments, unused-format-string-argument, line-too-long import re +from pathlib import Path import gtwrap.interface_parser as parser import gtwrap.template_instantiator as instantiator @@ -32,7 +33,7 @@ class PybindWrapper: self.top_module_namespaces = top_module_namespaces self.use_boost = use_boost self.ignore_classes = ignore_classes - self._serializing_classes = list() + self._serializing_classes = [] self.module_template = module_template self.python_keywords = [ 'lambda', 'False', 'def', 'if', 'raise', 'None', 'del', 'import', @@ -160,7 +161,7 @@ class PybindWrapper: 'self->print', 'py::scoped_ostream_redirect output; self->print') - # Make __repr__() call print() internally + # Make __repr__() call .print() internally ret += '''{prefix}.def("__repr__", [](const {cpp_class}& self{opt_comma}{args_signature_with_names}){{ gtsam::RedirectCout redirect; @@ -557,8 +558,15 @@ class PybindWrapper: ) return wrapped, includes - def wrap(self, content): - """Wrap the code in the interface file.""" + def wrap_file(self, content, module_name=None, submodules=None): + """ + Wrap the code in the interface file. + + Args: + content: The contents of the interface file. + module_name: The name of the module. + submodules: List of other interface file names that should be linked to. + """ # Parse the contents of the interface file module = parser.Module.parseString(content) # Instantiate all templates @@ -574,23 +582,74 @@ class PybindWrapper: if ',' in cpp_class: new_name = re.sub("[,:<> ]", "", cpp_class) boost_class_export += "typedef {cpp_class} {new_name};\n".format( # noqa - cpp_class=cpp_class, - new_name=new_name, - ) + cpp_class=cpp_class, new_name=new_name) + boost_class_export += "BOOST_CLASS_EXPORT({new_name})\n".format( new_name=new_name, ) + # Reset the serializing classes list + self._serializing_classes = [] + holder_type = "PYBIND11_DECLARE_HOLDER_TYPE(TYPE_PLACEHOLDER_DONOTUSE, " \ "{shared_ptr_type}::shared_ptr);" include_boost = "#include " if self.use_boost else "" + submodules_init = [] + + if submodules is not None: + module_def = "PYBIND11_MODULE({0}, m_)".format(module_name) + + for idx, submodule in enumerate(submodules): + submodules[idx] = "void {0}(py::module_ &);".format(submodule) + submodules_init.append("{0}(m_);".format(submodule)) + + else: + module_def = "void {0}(py::module_ &m_)".format(module_name) + submodules = [] + return self.module_template.format( include_boost=include_boost, - module_name=self.module_name, + module_def=module_def, + module_name=module_name, includes=includes, holder_type=holder_type.format( shared_ptr_type=('boost' if self.use_boost else 'std')) if self.use_boost else "", wrapped_namespace=wrapped_namespace, boost_class_export=boost_class_export, + submodules="\n".join(submodules), + submodules_init="\n".join(submodules_init), ) + + def wrap(self, sources, main_output): + """ + Wrap all the source interface files. + + Args: + sources: List of all interface files. + main_output: The name for the main module. + """ + main_module = sources[0] + submodules = [] + for source in sources[1:]: + filename = Path(source).name + module_name = Path(source).stem + # Read in the complete interface (.i) file + with open(source, "r") as f: + content = f.read() + submodules.append(module_name) + cc_content = self.wrap_file(content, module_name=module_name) + + # Generate the C++ code which Pybind11 will use. + with open(filename.replace(".i", ".cpp"), "w") as f: + f.write(cc_content) + + with open(main_module, "r") as f: + content = f.read() + cc_content = self.wrap_file(content, + module_name=self.module_name, + submodules=submodules) + + # Generate the C++ code which Pybind11 will use. + with open(main_output, "w") as f: + f.write(cc_content) diff --git a/wrap/scripts/pybind_wrap.py b/wrap/scripts/pybind_wrap.py index 7f2f8d419..c82a1d24c 100644 --- a/wrap/scripts/pybind_wrap.py +++ b/wrap/scripts/pybind_wrap.py @@ -67,10 +67,6 @@ def main(): if top_module_namespaces[0]: top_module_namespaces = [''] + top_module_namespaces - # Read in the complete interface (.i) file - with open(args.src, "r") as f: - content = f.read() - with open(args.template, "r") as f: template_content = f.read() @@ -83,11 +79,8 @@ def main(): ) # Wrap the code and get back the cpp/cc code. - cc_content = wrapper.wrap(content) - - # Generate the C++ code which Pybind11 will use. - with open(args.out, "w") as f: - f.write(cc_content) + sources = args.src.split(';') + wrapper.wrap(sources, args.out) if __name__ == "__main__": diff --git a/wrap/templates/pybind_wrapper.tpl.example b/wrap/templates/pybind_wrapper.tpl.example index bf5b33490..485aa8d00 100644 --- a/wrap/templates/pybind_wrapper.tpl.example +++ b/wrap/templates/pybind_wrapper.tpl.example @@ -7,7 +7,7 @@ #include #include #include "gtsam/base/serialization.h" -#include "gtsam/nonlinear/utilities.h" // for RedirectCout. +#include "gtsam/base/utilities.h" // for RedirectCout. {includes} #include @@ -22,9 +22,13 @@ using namespace std; namespace py = pybind11; -PYBIND11_MODULE({module_name}, m_) {{ +{submodules} + +{module_def} {{ m_.doc() = "pybind11 wrapper of {module_name}"; +{submodules_init} + {wrapped_namespace} #include "python/specializations.h"