wrapper updates

wrap/cmake/PybindWrap.cmake

@@ -13,15 +13,14 @@ gtwrap_get_python_version(${WRAP_PYTHON_VERSION})
 message(STATUS "Setting Python version for wrapper")
 set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION})
 
-# User-friendly Pybind11 wrapping and installing function.
+# User-friendly Pybind11 wrapping and installing function. Builds a Pybind11
-# Builds a Pybind11 module from the provided interface_header.
+# module from the provided interface_headers. For example, for the interface
-# For example, for the interface header gtsam.h, this will
+# header gtsam.h, this will build the wrap module 'gtsam_py.cc'.
-# build the wrap module 'gtsam_py.cc'.
 #
 # Arguments:
 # ~~~
 # target: The Make target
-# interface_header:  The relative path to the wrapper interface definition file.
+# interface_headers:  List of paths to the wrapper interface definition files. The top level interface file should be first.
 # generated_cpp: The name of the cpp file which is generated from the tpl file.
 # module_name: The name of the Python module to use.
 # top_namespace: The C++ namespace under which the code to be wrapped exists.
@@ -31,16 +30,17 @@ set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION})
 # libs: Libraries to link with.
 # dependencies: Dependencies which need to be built before the wrapper.
 # use_boost (optional): Flag indicating whether to include Boost.
-function(pybind_wrap
+function(
-         target
+  pybind_wrap
-         interface_header
+  target
-         generated_cpp
+  interface_headers
-         module_name
+  generated_cpp
-         top_namespace
+  module_name
-         ignore_classes
+  top_namespace
-         module_template
+  ignore_classes
-         libs
+  module_template
-         dependencies)
+  libs
+  dependencies)
   set(ExtraMacroArgs ${ARGN})
   list(GET ExtraMacroArgs 0 USE_BOOST)
   if(USE_BOOST)
@@ -49,57 +49,62 @@ function(pybind_wrap
     set(_WRAP_BOOST_ARG "")
   endif(USE_BOOST)
 
-  if (UNIX)
+  if(UNIX)
     set(GTWRAP_PATH_SEPARATOR ":")
   else()
     set(GTWRAP_PATH_SEPARATOR ";")
   endif()
 
-  add_custom_command(OUTPUT ${generated_cpp}
+  # Convert .i file names to .cpp file names.
-          COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
+  foreach(filepath ${interface_headers})
-          ${PYTHON_EXECUTABLE}
+    get_filename_component(interface ${filepath} NAME)
-                             ${PYBIND_WRAP_SCRIPT}
+    string(REPLACE ".i" ".cpp" cpp_file ${interface})
-                             --src
+    list(APPEND cpp_files ${cpp_file})
-                             ${interface_header}
+  endforeach()
-                             --out
+
-                             ${generated_cpp}
+  add_custom_command(
-                             --module_name
+    OUTPUT ${cpp_files}
-                             ${module_name}
+    COMMAND
-                             --top_module_namespaces
+      ${CMAKE_COMMAND} -E env
-                             "${top_namespace}"
+      "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
-                             --ignore
+      ${PYTHON_EXECUTABLE} ${PYBIND_WRAP_SCRIPT} --src "${interface_headers}"
-                             ${ignore_classes}
+      --out "${generated_cpp}" --module_name ${module_name}
-                             --template
+      --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes}
-                             ${module_template}
+      --template ${module_template} ${_WRAP_BOOST_ARG}
-                             ${_WRAP_BOOST_ARG}
+    DEPENDS "${interface_headers}" ${module_template}
-                     DEPENDS ${interface_header} ${module_template}
+    VERBATIM)
-                     VERBATIM)
+
-  add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${generated_cpp})
+  add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${cpp_files})
 
   # Late dependency injection, to make sure this gets called whenever the
   # interface header or the wrap library are updated.
   # ~~~
   # See: https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes
   # ~~~
-  add_custom_command(OUTPUT ${generated_cpp}
+  add_custom_command(
-                     DEPENDS ${interface_header}
+    OUTPUT ${cpp_files}
-                            #  @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py
+    DEPENDS ${interface_headers}
-                            #  @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py
+    # @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py
-                            #  @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py
+    # @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py
-                     APPEND)
+    # @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py
+    APPEND)
 
-  pybind11_add_module(${target} ${generated_cpp})
+  pybind11_add_module(${target} "${cpp_files}")
 
   if(APPLE)
-    # `type_info` objects will become "weak private external" if the templated class is initialized implicitly even if we explicitly
+    # `type_info` objects will become "weak private external" if the templated
-    # export them with `WRAP_EXPORT`. If that happens, the `type_info` for the same templated class will diverge between shared
+    # class is initialized implicitly even if we explicitly export them with
-    # libraries, causing `dynamic_cast` to fail. This is mitigated by telling Clang to mimic the MSVC behavior.
+    # `WRAP_EXPORT`. If that happens, the `type_info` for the same templated
-    # See https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2
+    # class will diverge between shared libraries, causing `dynamic_cast` to
+    # fail. This is mitigated by telling Clang to mimic the MSVC behavior. See
+    # https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2
     # https://github.com/CppMicroServices/CppMicroServices/pull/82/files
     # https://www.russellmcc.com/posts/2013-08-03-rtti.html
     target_compile_options(${target} PRIVATE "-fvisibility-ms-compat")
   endif()
+
   add_dependencies(${target} pybind_wrap_${module_name})
+
   if(NOT "${libs}" STREQUAL "")
     target_link_libraries(${target} PRIVATE "${libs}")
   endif()
@@ -121,10 +126,7 @@ endfunction()
 # dest_directory: The destination directory to install to.
 # patterns: list of file patterns to install
 # ~~~
-function(install_python_scripts
+function(install_python_scripts source_directory dest_directory patterns)
-         source_directory
-         dest_directory
-         patterns)
   set(patterns_args "")
   set(exclude_patterns "")
 
@@ -144,17 +146,19 @@ function(install_python_scripts
       # there is one
       get_filename_component(location "${dest_directory}" PATH)
       get_filename_component(name "${dest_directory}" NAME)
-      install(DIRECTORY "${source_directory}"
+      install(
-              DESTINATION "${location}/${name}${build_type_tag}"
+        DIRECTORY "${source_directory}"
-              CONFIGURATIONS "${build_type}"
+        DESTINATION "${location}/${name}${build_type_tag}"
-              FILES_MATCHING ${patterns_args}
+        CONFIGURATIONS "${build_type}"
-              PATTERN "${exclude_patterns}" EXCLUDE)
+        FILES_MATCHING ${patterns_args}
+        PATTERN "${exclude_patterns}" EXCLUDE)
     endforeach()
   else()
-    install(DIRECTORY "${source_directory}"
+    install(
-            DESTINATION "${dest_directory}"
+      DIRECTORY "${source_directory}"
-            FILES_MATCHING ${patterns_args}
+      DESTINATION "${dest_directory}"
-            PATTERN "${exclude_patterns}" EXCLUDE)
+      FILES_MATCHING ${patterns_args}
+      PATTERN "${exclude_patterns}" EXCLUDE)
   endif()
 
 endfunction()
@@ -172,13 +176,14 @@ function(install_python_files source_files dest_directory)
     foreach(build_type ${CMAKE_CONFIGURATION_TYPES})
       string(TOUPPER "${build_type}" build_type_upper)
       set(build_type_tag "")
-      # Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if
+      # Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if there
-      # there is one
+      # is one
       get_filename_component(location "${dest_directory}" PATH)
       get_filename_component(name "${dest_directory}" NAME)
-      install(FILES "${source_files}"
+      install(
-              DESTINATION "${location}/${name}${build_type_tag}"
+        FILES "${source_files}"
-              CONFIGURATIONS "${build_type}")
+        DESTINATION "${location}/${name}${build_type_tag}"
+        CONFIGURATIONS "${build_type}")
     endforeach()
   else()
     install(FILES "${source_files}" DESTINATION "${dest_directory}")
@@ -194,18 +199,19 @@ function(create_symlinks source_folder dest_folder)
     return()
   endif()
 
-  file(GLOB files
+  file(
-       LIST_DIRECTORIES true
+    GLOB files
-       RELATIVE "${source_folder}"
+    LIST_DIRECTORIES true
-       "${source_folder}/*")
+    RELATIVE "${source_folder}"
+    "${source_folder}/*")
   foreach(path_file ${files})
     get_filename_component(folder ${path_file} PATH)
     get_filename_component(ext ${path_file} EXT)
     set(ignored_ext ".tpl" ".h")
-    list (FIND ignored_ext "${ext}" _index)
+    list(FIND ignored_ext "${ext}" _index)
-    if (${_index} GREATER -1)
+    if(${_index} GREATER -1)
       continue()
-    endif ()
+    endif()
     # Create REAL folder
     file(MAKE_DIRECTORY "${dest_folder}")
 
@@ -224,9 +230,10 @@ function(create_symlinks source_folder dest_folder)
     endif()
     # cmake-format: on
 
-    execute_process(COMMAND ${command}
+    execute_process(
-                    RESULT_VARIABLE result
+      COMMAND ${command}
-                    ERROR_VARIABLE output)
+      RESULT_VARIABLE result
+      ERROR_VARIABLE output)
 
     if(NOT ${result} EQUAL 0)
       message(

wrap/gtwrap/pybind_wrapper.py

@@ -13,6 +13,7 @@ Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellae
 # pylint: disable=too-many-arguments, too-many-instance-attributes, no-self-use, no-else-return, too-many-arguments, unused-format-string-argument, line-too-long
 
 import re
+from pathlib import Path
 
 import gtwrap.interface_parser as parser
 import gtwrap.template_instantiator as instantiator
@@ -32,7 +33,7 @@ class PybindWrapper:
         self.top_module_namespaces = top_module_namespaces
         self.use_boost = use_boost
         self.ignore_classes = ignore_classes
-        self._serializing_classes = list()
+        self._serializing_classes = []
         self.module_template = module_template
         self.python_keywords = [
             'lambda', 'False', 'def', 'if', 'raise', 'None', 'del', 'import',
@@ -160,7 +161,7 @@ class PybindWrapper:
                 'self->print',
                 'py::scoped_ostream_redirect output; self->print')
 
-            # Make __repr__() call print() internally
+            # Make __repr__() call .print() internally
             ret += '''{prefix}.def("__repr__",
                     [](const {cpp_class}& self{opt_comma}{args_signature_with_names}){{
                         gtsam::RedirectCout redirect;
@@ -557,8 +558,15 @@ class PybindWrapper:
             )
         return wrapped, includes
 
-    def wrap(self, content):
+    def wrap_file(self, content, module_name=None, submodules=None):
-        """Wrap the code in the interface file."""
+        """
+        Wrap the code in the interface file.
+
+        Args:
+            content: The contents of the interface file.
+            module_name: The name of the module.
+            submodules: List of other interface file names that should be linked to.
+        """
         # Parse the contents of the interface file
         module = parser.Module.parseString(content)
         # Instantiate all templates
@@ -574,23 +582,74 @@ class PybindWrapper:
             if ',' in cpp_class:
                 new_name = re.sub("[,:<> ]", "", cpp_class)
                 boost_class_export += "typedef {cpp_class} {new_name};\n".format(  # noqa
-                    cpp_class=cpp_class,
+                    cpp_class=cpp_class, new_name=new_name)
-                    new_name=new_name,
+
-                )
             boost_class_export += "BOOST_CLASS_EXPORT({new_name})\n".format(
                 new_name=new_name, )
 
+        # Reset the serializing classes list
+        self._serializing_classes = []
+
         holder_type = "PYBIND11_DECLARE_HOLDER_TYPE(TYPE_PLACEHOLDER_DONOTUSE, " \
                       "{shared_ptr_type}::shared_ptr<TYPE_PLACEHOLDER_DONOTUSE>);"
         include_boost = "#include <boost/shared_ptr.hpp>" if self.use_boost else ""
 
+        submodules_init = []
+
+        if submodules is not None:
+            module_def = "PYBIND11_MODULE({0}, m_)".format(module_name)
+
+            for idx, submodule in enumerate(submodules):
+                submodules[idx] = "void {0}(py::module_ &);".format(submodule)
+                submodules_init.append("{0}(m_);".format(submodule))
+
+        else:
+            module_def = "void {0}(py::module_ &m_)".format(module_name)
+            submodules = []
+
         return self.module_template.format(
             include_boost=include_boost,
-            module_name=self.module_name,
+            module_def=module_def,
+            module_name=module_name,
             includes=includes,
             holder_type=holder_type.format(
                 shared_ptr_type=('boost' if self.use_boost else 'std'))
             if self.use_boost else "",
             wrapped_namespace=wrapped_namespace,
             boost_class_export=boost_class_export,
+            submodules="\n".join(submodules),
+            submodules_init="\n".join(submodules_init),
         )
+
+    def wrap(self, sources, main_output):
+        """
+        Wrap all the source interface files.
+
+        Args:
+            sources: List of all interface files.
+            main_output: The name for the main module.
+        """
+        main_module = sources[0]
+        submodules = []
+        for source in sources[1:]:
+            filename = Path(source).name
+            module_name = Path(source).stem
+            # Read in the complete interface (.i) file
+            with open(source, "r") as f:
+                content = f.read()
+            submodules.append(module_name)
+            cc_content = self.wrap_file(content, module_name=module_name)
+
+            # Generate the C++ code which Pybind11 will use.
+            with open(filename.replace(".i", ".cpp"), "w") as f:
+                f.write(cc_content)
+
+        with open(main_module, "r") as f:
+            content = f.read()
+        cc_content = self.wrap_file(content,
+                                    module_name=self.module_name,
+                                    submodules=submodules)
+
+        # Generate the C++ code which Pybind11 will use.
+        with open(main_output, "w") as f:
+            f.write(cc_content)

wrap/scripts/pybind_wrap.py

@@ -67,10 +67,6 @@ def main():
     if top_module_namespaces[0]:
         top_module_namespaces = [''] + top_module_namespaces
 
-    # Read in the complete interface (.i) file
-    with open(args.src, "r") as f:
-        content = f.read()
-
     with open(args.template, "r") as f:
         template_content = f.read()
 
@@ -83,11 +79,8 @@ def main():
     )
 
     # Wrap the code and get back the cpp/cc code.
-    cc_content = wrapper.wrap(content)
+    sources = args.src.split(';')
-
+    wrapper.wrap(sources, args.out)
-    # Generate the C++ code which Pybind11 will use.
-    with open(args.out, "w") as f:
-        f.write(cc_content)
 
 
 if __name__ == "__main__":

wrap/templates/pybind_wrapper.tpl.example

@@ -7,7 +7,7 @@
 #include <pybind11/iostream.h>
 #include <pybind11/functional.h>
 #include "gtsam/base/serialization.h"
-#include "gtsam/nonlinear/utilities.h"  // for RedirectCout.
+#include "gtsam/base/utilities.h"  // for RedirectCout.
 
 {includes}
 #include <boost/serialization/export.hpp>
@@ -22,9 +22,13 @@ using namespace std;
 
 namespace py = pybind11;
 
-PYBIND11_MODULE({module_name}, m_) {{
+{submodules}
+
+{module_def} {{
     m_.doc() = "pybind11 wrapper of {module_name}";
 
+{submodules_init}
+
 {wrapped_namespace}
 
 #include "python/specializations.h"