Merging 'master' into 'wrap'

wrap/cmake/PybindWrap.cmake

@@ -55,15 +55,44 @@ function(
     set(GTWRAP_PATH_SEPARATOR ";")
   endif()
 
+  # Create a copy of interface_headers so we can freely manipulate it
+  set(interface_files ${interface_headers})
+
+  # Pop the main interface file so that interface_files has only submodules.
+  list(POP_FRONT interface_files main_interface)
+
   # Convert .i file names to .cpp file names.
-  foreach(filepath ${interface_headers})
+  foreach(interface_file ${interface_files})
-    get_filename_component(interface ${filepath} NAME)
+    # This block gets the interface file name and does the replacement
-    string(REPLACE ".i" ".cpp" cpp_file ${interface})
+    get_filename_component(interface ${interface_file} NAME_WLE)
+    set(cpp_file "${interface}.cpp")
     list(APPEND cpp_files ${cpp_file})
+
+    # Wrap the specific interface header
+    # This is done so that we can create CMake dependencies in such a way so that when changing a single .i file,
+    # the others don't need to be regenerated.
+    # NOTE: We have to use `add_custom_command` so set the dependencies correctly.
+    # https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes
+    add_custom_command(
+      OUTPUT ${cpp_file}
+      COMMAND
+        ${CMAKE_COMMAND} -E env
+        "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
+        ${PYTHON_EXECUTABLE} ${PYBIND_WRAP_SCRIPT} --src "${interface_file}"
+          --out "${cpp_file}"  --module_name ${module_name}
+          --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes}
+          --template ${module_template} --is_submodule ${_WRAP_BOOST_ARG}
+      DEPENDS "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h"
+      VERBATIM)
+
   endforeach()
 
+  get_filename_component(main_interface_name ${main_interface} NAME_WLE)
+  set(main_cpp_file "${main_interface_name}.cpp")
+  list(PREPEND cpp_files ${main_cpp_file})
+
   add_custom_command(
-    OUTPUT ${cpp_files}
+    OUTPUT ${main_cpp_file}
     COMMAND
       ${CMAKE_COMMAND} -E env
       "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
@@ -71,23 +100,10 @@ function(
       --out "${generated_cpp}" --module_name ${module_name}
       --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes}
       --template ${module_template} ${_WRAP_BOOST_ARG}
-    DEPENDS "${interface_headers}" ${module_template}
+    DEPENDS "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h"
     VERBATIM)
 
-  add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${cpp_files})
+    add_custom_target(pybind_wrap_${module_name} DEPENDS ${cpp_files})
-
-  # Late dependency injection, to make sure this gets called whenever the
-  # interface header or the wrap library are updated.
-  # ~~~
-  # See: https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes
-  # ~~~
-  add_custom_command(
-    OUTPUT ${cpp_files}
-    DEPENDS ${interface_headers}
-    # @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py
-    # @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py
-    # @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py
-    APPEND)
 
   pybind11_add_module(${target} "${cpp_files}")
 

wrap/gtwrap/pybind_wrapper.py

@@ -631,28 +631,47 @@ class PybindWrapper:
             submodules_init="\n".join(submodules_init),
         )
 
-    def wrap(self, sources, main_output):
+    def wrap_submodule(self, source):
         """
-        Wrap all the source interface files.
+        Wrap a list of submodule files, i.e. a set of interface files which are
+        in support of a larger wrapping project.
+
+        E.g. This is used in GTSAM where we have a main gtsam.i, but various smaller .i files
+        which are the submodules.
+        The benefit of this scheme is that it reduces compute and memory usage during compilation.
+
+        Args:
+            source: Interface file which forms the submodule.
+        """
+        filename = Path(source).name
+        module_name = Path(source).stem
+
+        # Read in the complete interface (.i) file
+        with open(source, "r") as f:
+            content = f.read()
+        # Wrap the read-in content
+        cc_content = self.wrap_file(content, module_name=module_name)
+
+        # Generate the C++ code which Pybind11 will use.
+        with open(filename.replace(".i", ".cpp"), "w") as f:
+            f.write(cc_content)
+
+    def wrap(self, sources, main_module_name):
+        """
+        Wrap all the main interface file.
 
         Args:
             sources: List of all interface files.
-            main_output: The name for the main module.
+                The first file should be the main module.
+            main_module_name: The name for the main module.
         """
         main_module = sources[0]
+
+        # Get all the submodule names.
         submodules = []
         for source in sources[1:]:
-            filename = Path(source).name
             module_name = Path(source).stem
-            # Read in the complete interface (.i) file
-            with open(source, "r") as f:
-                content = f.read()
             submodules.append(module_name)
-            cc_content = self.wrap_file(content, module_name=module_name)
-
-            # Generate the C++ code which Pybind11 will use.
-            with open(filename.replace(".i", ".cpp"), "w") as f:
-                f.write(cc_content)
 
         with open(main_module, "r") as f:
             content = f.read()
@@ -661,5 +680,5 @@ class PybindWrapper:
                                     submodules=submodules)
 
         # Generate the C++ code which Pybind11 will use.
-        with open(main_output, "w") as f:
+        with open(main_module_name, "w") as f:
             f.write(cc_content)

wrap/scripts/pybind_wrap.py

@@ -19,7 +19,7 @@ def main():
     arg_parser.add_argument("--src",
                             type=str,
                             required=True,
-                            help="Input interface .i/.h file")
+                            help="Input interface .i/.h file(s)")
     arg_parser.add_argument(
         "--module_name",
         type=str,
@@ -31,7 +31,7 @@ def main():
         "--out",
         type=str,
         required=True,
-        help="Name of the output pybind .cc file",
+        help="Name of the output pybind .cc file(s)",
     )
     arg_parser.add_argument(
         "--use-boost",
@@ -60,7 +60,10 @@ def main():
     )
     arg_parser.add_argument("--template",
                             type=str,
-                            help="The module template file")
+                            help="The module template file (e.g. module.tpl).")
+    arg_parser.add_argument("--is_submodule",
+                            default=False,
+                            action="store_true")
     args = arg_parser.parse_args()
 
     top_module_namespaces = args.top_module_namespaces.split("::")
@@ -78,9 +81,13 @@ def main():
         module_template=template_content,
     )
 
-    # Wrap the code and get back the cpp/cc code.
+    if args.is_submodule:
-    sources = args.src.split(';')
+        wrapper.wrap_submodule(args.src)
-    wrapper.wrap(sources, args.out)
+
+    else:
+        # Wrap the code and get back the cpp/cc code.
+        sources = args.src.split(';')
+        wrapper.wrap(sources, args.out)
 
 
 if __name__ == "__main__":