Major update to generate proper Cython pxd header files which could be included in other projects/modules

All cdef (class, functions, variables) declarations are moved to pxd. Implementations of those cdefs and normal Python def are in pyx.
See: http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html#sharing-extension-types

wrap/Argument.cpp

@@ -17,6 +17,7 @@
  **/
 
 #include "Argument.h"
+#include "Class.h"
 
 #include <boost/lexical_cast.hpp>
 
@@ -98,15 +99,19 @@ void Argument::proxy_check(FileWriter& proxyFile, const string& s) const {
 }
 
 /* ************************************************************************* */
-void Argument::emit_cython_pxd(FileWriter& file, const std::string& className) const {
+void Argument::emit_cython_pxd(
-  string typeName = type.pxdClassName();
+    FileWriter& file, const std::string& className,
-  if (typeName == "This") typeName = className;
+    const std::vector<std::string>& templateArgs) const {
+  string cythonType = type.pxdClassName();
+  if (cythonType == "This") cythonType = className;
+  else if (type.isEigen())
+    cythonType = "const " + cythonType + "&";
+  else if (type.match(templateArgs))
+    cythonType = type.name();
 
-  string cythonType = typeName;
+  // add modifier
-  if (type.isEigen()) {
+  if (!type.isEigen()) {
-    cythonType = "const " + typeName + "&";
+    if (is_ptr) cythonType = "shared_ptr[" + cythonType + "]&";
-  } else {
-    if (is_ptr) cythonType = "shared_ptr[" + typeName + "]&";
     if (is_ref) cythonType = cythonType + "&";
     if (is_const) cythonType = "const " + cythonType;
   }
@@ -214,9 +219,11 @@ void ArgumentList::emit_prototype(FileWriter& file, const string& name) const {
 }
 
 /* ************************************************************************* */
-void ArgumentList::emit_cython_pxd(FileWriter& file, const std::string& className) const {
+void ArgumentList::emit_cython_pxd(
+    FileWriter& file, const std::string& className,
+    const std::vector<std::string>& templateArgs) const {
   for (size_t j = 0; j<size(); ++j) {
-    at(j).emit_cython_pxd(file, className);
+    at(j).emit_cython_pxd(file, className, templateArgs);
     if (j<size()-1) file.oss << ", ";
   }
 }

wrap/Argument.h

@@ -69,7 +69,8 @@ struct Argument {
    * emit arguments for cython pxd
    * @param file output stream
    */
-  void emit_cython_pxd(FileWriter& file, const std::string& className) const;
+  void emit_cython_pxd(FileWriter& file, const std::string& className,
+                       const std::vector<std::string>& templateArgs) const;
   void emit_cython_pyx(FileWriter& file) const;
   std::string pyx_asParam() const;
 
@@ -124,7 +125,8 @@ struct ArgumentList: public std::vector<Argument> {
    * emit arguments for cython pxd
    * @param file output stream
    */
-  void emit_cython_pxd(FileWriter& file, const std::string& className) const;
+  void emit_cython_pxd(FileWriter& file, const std::string& className,
+                       const std::vector<std::string>& templateArgs) const;
   void emit_cython_pyx(FileWriter& file) const;
   std::string pyx_asParams() const;
   std::string pyx_paramsList() const;

wrap/Class.cpp

@@ -744,7 +744,7 @@ void Class::emit_cython_pxd(FileWriter& pxdFile, const std::vector<Class>& allCl
   if (parentClass) pxdFile.oss << "(" <<  parentClass->pxdClassName() << ")";
   pxdFile.oss << ":\n";
 
-  constructor.emit_cython_pxd(pxdFile, pxdClassName());
+  constructor.emit_cython_pxd(pxdFile, *this);
   if (constructor.nrOverloads()>0) pxdFile.oss << "\n";
 
   for(const StaticMethod& m: static_methods | boost::adaptors::map_values)
@@ -759,7 +759,7 @@ void Class::emit_cython_pxd(FileWriter& pxdFile, const std::vector<Class>& allCl
   size_t numMethods = constructor.nrOverloads() + static_methods.size() +
                       methods_.size() + templateMethods_.size();
   if (numMethods == 0)
-      pxdFile.oss << "        pass";
+      pxdFile.oss << "        pass\n";
 }
 
 /* ************************************************************************* */
@@ -820,7 +820,7 @@ void Class::emit_cython_pyx(FileWriter& pyxFile, const std::vector<Class>& allCl
   pyxFile.oss << ":\n";
 
   // shared variable of the corresponding cython object 
-  pyxFile.oss << "    cdef " << shared_pxd_class_in_pyx() << " " << shared_pxd_obj_in_pyx() << "\n";
+  // pyxFile.oss << "    cdef " << shared_pxd_class_in_pyx() << " " << shared_pxd_obj_in_pyx() << "\n";
 
   // __cinit___
   pyxFile.oss << "    def __init__(self, *args, **kwargs):\n"

wrap/Class.h

@@ -93,6 +93,7 @@ public:
   void assignParent(const Qualified& parent);
 
   boost::optional<std::string> qualifiedParent() const;
+  boost::optional<Qualified> getParent() const { return parentClass; }
 
   size_t nrMethods() const {
     return methods_.size();

wrap/Constructor.cpp

@@ -129,13 +129,13 @@ bool Constructor::hasDefaultConstructor() const {
 }
 
 /* ************************************************************************* */
-void Constructor::emit_cython_pxd(FileWriter& pxdFile, Str className) const {
+void Constructor::emit_cython_pxd(FileWriter& pxdFile, const Class& cls) const {
   for (size_t i = 0; i < nrOverloads(); i++) {
     ArgumentList args = argumentList(i);
 
     // generate the constructor
-    pxdFile.oss << "        " << className << "(";
+    pxdFile.oss << "        " << cls.pxdClassName() << "(";
-    args.emit_cython_pxd(pxdFile, className);
+    args.emit_cython_pxd(pxdFile, cls.pxdClassName(), cls.templateArgs);
     pxdFile.oss << ") "
                 << "except +\n";
   }

wrap/Constructor.h

@@ -85,7 +85,7 @@ struct Constructor: public OverloadedFunction {
   void python_wrapper(FileWriter& wrapperFile, Str className) const;
 
   // emit cython wrapper
-  void emit_cython_pxd(FileWriter& pxdFile, Str className) const;
+  void emit_cython_pxd(FileWriter& pxdFile, const Class& cls) const;
   void emit_cython_pyx(FileWriter& pyxFile, const Class& cls) const;
 
   friend std::ostream& operator<<(std::ostream& os, const Constructor& m) {

wrap/GlobalFunction.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "GlobalFunction.h"
+#include "Class.h"
 #include "utilities.h"
 
 #include <boost/lexical_cast.hpp>
@@ -139,10 +140,10 @@ void GlobalFunction::emit_cython_pxd(FileWriter& file) const {
                 << "\":" << endl;
   for (size_t i = 0; i < nrOverloads(); ++i) {
     file.oss << "        ";
-    returnVals_[i].emit_cython_pxd(file, "");
+    returnVals_[i].emit_cython_pxd(file, "", vector<string>());
-    file.oss << pyRename(name_) + " \"" + overloads[0].qualifiedName("::") +
+    file.oss << pxdName() + " \"" + overloads[0].qualifiedName("::") +
                     "\"(";
-    argumentList(i).emit_cython_pxd(file, "");
+    argumentList(i).emit_cython_pxd(file, "", vector<string>());
     file.oss << ")";
     file.oss << "\n";
   }
@@ -150,7 +151,7 @@ void GlobalFunction::emit_cython_pxd(FileWriter& file) const {
 
 /* ************************************************************************* */
 void GlobalFunction::emit_cython_pyx_no_overload(FileWriter& file) const {
-  string funcName = pyRename(name_);
+  string funcName = pyxName();
 
   // Function definition
   file.oss << "def " << funcName;
@@ -163,7 +164,7 @@ void GlobalFunction::emit_cython_pyx_no_overload(FileWriter& file) const {
   file.oss << "):\n";
 
   /// Call cython corresponding function and return
-  string ret = pyx_functionCall("pxd", funcName, 0);
+  string ret = pyx_functionCall("", pxdName(), 0);
   if (!returnVals_[0].isVoid()) {
     file.oss << "    cdef " << returnVals_[0].pyx_returnType()
              << " ret = " << ret << "\n";
@@ -175,7 +176,7 @@ void GlobalFunction::emit_cython_pyx_no_overload(FileWriter& file) const {
 
 /* ************************************************************************* */
 void GlobalFunction::emit_cython_pyx(FileWriter& file) const {
-  string funcName = pyRename(name_);
+  string funcName = pyxName();
 
   size_t N = nrOverloads();
   if (N == 1) {
@@ -199,7 +200,7 @@ void GlobalFunction::emit_cython_pyx(FileWriter& file) const {
     file.oss << pyx_resolveOverloadParams(args, false, 1); // lazy: always return None even if it's a void function
 
     /// Call cython corresponding function
-    string ret = pyx_functionCall("pxd", funcName, i);
+    string ret = pyx_functionCall("", pxdName(), i);
     if (!returnVals_[i].isVoid()) {
       file.oss << "    cdef " << returnVals_[i].pyx_returnType()
               << " ret = " << ret << "\n";

wrap/GlobalFunction.h

@@ -51,6 +51,11 @@ struct GlobalFunction: public FullyOverloadedFunction {
   // emit python wrapper
   void python_wrapper(FileWriter& wrapperFile) const;
 
+  // function name in Cython pxd
+  std::string pxdName() const { return "pxd_" + pyRename(name_); }
+  // function name in Python pyx
+  std::string pyxName() const { return pyRename(name_); }
+
   // emit cython wrapper
   void emit_cython_pxd(FileWriter& pxdFile) const;
   void emit_cython_pyx(FileWriter& pyxFile) const;

wrap/Method.cpp

@@ -84,10 +84,10 @@ string Method::wrapper_call(FileWriter& wrapperFile, Str cppClassName,
 void Method::emit_cython_pxd(FileWriter& file, const Class& cls) const {
   for (size_t i = 0; i < nrOverloads(); ++i) {
     file.oss << "        ";
-    returnVals_[i].emit_cython_pxd(file, cls.pxdClassName());
+    returnVals_[i].emit_cython_pxd(file, cls.pxdClassName(), cls.templateArgs);
     file.oss << pyRename(name_) + " \"" + name_ + "\""
              << "(";
-    argumentList(i).emit_cython_pxd(file, cls.pxdClassName());
+    argumentList(i).emit_cython_pxd(file, cls.pxdClassName(), cls.templateArgs);
     file.oss << ")";
     if (is_const_) file.oss << " const";
     file.oss << "\n";

wrap/Module.cpp

@@ -314,10 +314,11 @@ void Module::matlab_code(const string& toolboxPath) const {
 }
 
 /* ************************************************************************* */
-void Module::cython_wrapper(const string& toolboxPath) const {
+void Module::cython_wrapper(const string& toolboxPath, const std::string& pxdImports) const {
   fs::create_directories(toolboxPath);
-  string pxdFileName = toolboxPath + "/" + name + "_wrapper" + ".pxd";
+  string pxdFileName = toolboxPath + "/" + name + ".pxd";
   FileWriter pxdFile(pxdFileName, verbose, "#");
+  pxdFile.oss << pxdImports << "\n";
   emit_cython_pxd(pxdFile);
   string pyxFileName = toolboxPath + "/" + name + ".pyx";
   FileWriter pyxFile(pyxFileName, verbose, "#");
@@ -350,12 +351,11 @@ void Module::emit_cython_pxd(FileWriter& pxdFile) const {
   //... wrap all classes
   for (const Class& cls : uninstantiatedClasses) {
       cls.emit_cython_pxd(pxdFile, uninstantiatedClasses);
-      pxdFile.oss << "\n";
 
-      //... ctypedef for template instantiations
       for (const Class& expCls : expandedClasses) {
-          if (!expCls.templateClass || expCls.templateClass->name_ != cls.name_)
+        //... ctypedef for template instantiations
-              continue;
+        if (expCls.templateClass &&
+            expCls.templateClass->pxdClassName() == cls.pxdClassName()) {
           pxdFile.oss << "ctypedef " << expCls.templateClass->pxdClassName()
                       << "[";
           for (size_t i = 0; i < expCls.templateInstTypeList.size(); ++i)
@@ -365,6 +365,25 @@ void Module::emit_cython_pxd(FileWriter& pxdFile) const {
                                 : ", ");
           pxdFile.oss << "] " << expCls.pxdClassName() << "\n";
         }
+
+        if ((!expCls.templateClass &&
+             expCls.pxdClassName() == cls.pxdClassName()) ||
+            (expCls.templateClass &&
+             expCls.templateClass->pxdClassName() == cls.pxdClassName())) {
+            pxdFile.oss << "\n";
+            pxdFile.oss << "cdef class " << expCls.pyxClassName();
+            if (expCls.getParent())
+              pxdFile.oss << "(" << expCls.getParent()->pyxClassName() << ")";
+            pxdFile.oss << ":\n";
+            pxdFile.oss << "    cdef " << expCls.shared_pxd_class_in_pyx()
+                        << " " << expCls.shared_pxd_obj_in_pyx() << "\n";
+            // cyCreateFromShared
+            pxdFile.oss << "    @staticmethod\n";
+            pxdFile.oss << "    cdef " << expCls.pyxClassName()
+                        << " cyCreateFromShared(const "
+                        << expCls.shared_pxd_class_in_pyx() << "& other)\n";
+          }
+      }
       pxdFile.oss << "\n\n";
   }
 
@@ -378,10 +397,10 @@ void Module::emit_cython_pxd(FileWriter& pxdFile) const {
 /* ************************************************************************* */
 void Module::emit_cython_pyx(FileWriter& pyxFile) const {
   // headers...
-  string pxdHeader = name + "_wrapper";
+  string pxdHeader = name;
   pyxFile.oss << "cimport numpy as np\n"
                  "import numpy as npp\n"
-                 "cimport " << pxdHeader << " as " << "pxd" << "\n"
+                 "cimport " << pxdHeader << "\n"
                  "from "<< pxdHeader << " cimport shared_ptr\n"
                  "from "<< pxdHeader << " cimport dynamic_pointer_cast\n";
   // import all typedefs, e.g. from gtsam_wrapper cimport Key, so we don't need to say gtsam.Key

wrap/Module.h

@@ -66,7 +66,7 @@ struct Module {
   void matlab_code(const std::string& path) const;
 
   /// Cython code generation:
-  void cython_wrapper(const std::string& path) const;
+  void cython_wrapper(const std::string& path, const std::string& pxdImports) const;
   void emit_cython_pxd(FileWriter& file) const;
   void emit_cython_pyx(FileWriter& file) const;
 

wrap/Qualified.h

@@ -86,6 +86,12 @@ public:
     return (name_ == templateArg && namespaces_.empty()); //TODO && category == CLASS);
   }
 
+  bool match(const std::vector<std::string>& templateArgs) const {
+    for(const std::string& s: templateArgs) 
+      if (match(s)) return true;
+    return false;
+  }
+
   void rename(const Qualified& q) {
     namespaces_ = q.namespaces_;
     name_ = q.name_;
@@ -136,8 +142,8 @@ public:
   }
 
   bool isNonBasicType() const {
-    return !isString() && !isScalar() && !isEigen() && !isVoid() &&
+    return name() != "This" && !isString() && !isScalar() && !isEigen() &&
-           !isBasicTypedef();
+           !isVoid() && !isBasicTypedef();
   }
 
 public:
@@ -191,8 +197,9 @@ public:
   std::string pxdClassName() const {
     if (isEigen())
       return name_ + "Xd";
-    else
+    else if (isNonBasicType())
-      return qualifiedName("_", 1);
+      return "C" + qualifiedName("_", 1);
+    else return name_;
   }
 
   /// name of Python classes in pyx
@@ -222,7 +229,7 @@ public:
   std::string pxd_class_in_pyx() const {
       if (isNonBasicType()) {
         if (namespaces_.size() > 0)
-          return "pxd." + pxdClassName();
+          return pxdClassName();
         else {
           std::cerr << "Class without namespace: " << pxdClassName() << std::endl;
           throw std::runtime_error("Error: User type without namespace!!");
@@ -236,7 +243,7 @@ public:
 
   /// the internal Cython shared obj in a Python class wrappper
   std::string shared_pxd_obj_in_pyx() const {
-    return "shared_pxd_" + pxdClassName() + "_";
+    return "shared_" + pxdClassName() + "_";
   }
 
   std::string shared_pxd_class_in_pyx() const {

wrap/ReturnType.cpp

@@ -66,15 +66,17 @@ void ReturnType::wrapTypeUnwrap(FileWriter& wrapperFile) const {
 }
 
 /* ************************************************************************* */
-void ReturnType::emit_cython_pxd(FileWriter& file,
+void ReturnType::emit_cython_pxd(
-                                 const std::string& className) const {
+    FileWriter& file, const std::string& className,
-  string typeName = pxdClassName();
+    const std::vector<std::string>& templateArgs) const {
   string cythonType;
-  if (isPtr)
+  if (name() == "This")
-    cythonType = "shared_ptr[" + typeName + "]";
+    cythonType = className;
+  else if (match(templateArgs))
+    cythonType = name();
   else
-    cythonType = typeName;
+    cythonType = pxdClassName();
-  if (typeName == "This") cythonType = className;
+  if (isPtr) cythonType = "shared_ptr[" + cythonType + "]";
   file.oss << cythonType;
 }
 

wrap/ReturnType.h

@@ -44,7 +44,8 @@ struct ReturnType : public Qualified {
   }
 
   /// @param className the actual class name to use when "This" is specified
-  void emit_cython_pxd(FileWriter& file, const std::string& className) const;
+  void emit_cython_pxd(FileWriter& file, const std::string& className,
+                       const std::vector<std::string>& templateArgs) const;
 
   std::string pyx_returnType(bool addShared = true) const;
   std::string pyx_casting(const std::string& var,

wrap/ReturnValue.cpp

@@ -67,16 +67,17 @@ void ReturnValue::emit_matlab(FileWriter& proxyFile) const {
 }
 
 /* ************************************************************************* */
-void ReturnValue::emit_cython_pxd(FileWriter& file,
+void ReturnValue::emit_cython_pxd(
-                                  const std::string& className) const {
+    FileWriter& file, const std::string& className,
+    const std::vector<std::string>& templateArgs) const {
   if (isPair) {
     file.oss << "pair[";
-    type1.emit_cython_pxd(file, className);
+    type1.emit_cython_pxd(file, className, templateArgs);
     file.oss << ",";
-    type2.emit_cython_pxd(file, className);
+    type2.emit_cython_pxd(file, className, templateArgs);
     file.oss << "] ";
   } else {
-    type1.emit_cython_pxd(file, className);
+    type1.emit_cython_pxd(file, className, templateArgs);
     file.oss << " ";
   }
 }

wrap/ReturnValue.h

@@ -72,7 +72,8 @@ struct ReturnValue {
   void emit_matlab(FileWriter& proxyFile) const;
 
   /// @param className the actual class name to use when "This" is specified
-  void emit_cython_pxd(FileWriter& file, const std::string& className) const;
+  void emit_cython_pxd(FileWriter& file, const std::string& className,
+                       const std::vector<std::string>& templateArgs) const;
   std::string pyx_returnType() const;
   std::string pyx_casting(const std::string& var) const;
 

wrap/StaticMethod.cpp

@@ -62,9 +62,9 @@ void StaticMethod::emit_cython_pxd(FileWriter& file, const Class& cls) const {
   for(size_t i = 0; i < nrOverloads(); ++i) {
     file.oss << "        @staticmethod\n";
     file.oss << "        ";
-    returnVals_[i].emit_cython_pxd(file, cls.pxdClassName());
+    returnVals_[i].emit_cython_pxd(file, cls.pxdClassName(), cls.templateArgs);
     file.oss << name_ + ((i>0)?"_" + to_string(i):"") << " \"" << name_ << "\"" << "(";
-    argumentList(i).emit_cython_pxd(file, cls.pxdClassName());
+    argumentList(i).emit_cython_pxd(file, cls.pxdClassName(), cls.templateArgs);
     file.oss << ")\n";
   }
 }

wrap/TemplateMethod.cpp

@@ -22,11 +22,13 @@ using namespace wrap;
 
 /* ************************************************************************* */
 void TemplateMethod::emit_cython_pxd(FileWriter& file, const Class& cls) const {
+  std::vector<std::string> templateArgs = cls.templateArgs;
+  templateArgs.push_back(argName);
   for(size_t i = 0; i < nrOverloads(); ++i) {
     file.oss << "        ";
-    returnVals_[i].emit_cython_pxd(file, cls.pxdClassName());
+    returnVals_[i].emit_cython_pxd(file, cls.pxdClassName(), templateArgs);
     file.oss << name_ << "[" << argName << "]" << "(";
-    argumentList(i).emit_cython_pxd(file, cls.pxdClassName());
+    argumentList(i).emit_cython_pxd(file, cls.pxdClassName(), templateArgs);
     file.oss << ")\n";
   }
 }

wrap/TemplateMethod.h

@@ -10,8 +10,8 @@
  * -------------------------------------------------------------------------- */
 
 /**
- * @file StaticMethod.h
+ * @file TemplateMethod.h
- * @brief describes and generates code for static methods
+ * @brief describes and generates code for template methods
  * @author Duy-Nguyen Ta
  **/
 
@@ -36,8 +36,6 @@ struct TemplateMethod: public Method {
     return os;
   }
 
-  void emit_cython_pxd(FileWriter& file) const;
-
 };
 
 } // \namespace wrap

wrap/wrap.cpp

@@ -24,12 +24,12 @@ using namespace std;
 
 /** Displays usage information */
 void usage() {
-  cerr << "wrap parses an interface file and produces a MATLAB toolbox" << endl;
+  cerr << "wrap parses an interface file and produces a MATLAB or Cython toolbox" << endl;
-  cerr << "usage: wrap [--matlab|--cython] interfacePath moduleName toolboxPath headerPath" << endl;
+  cerr << "usage: wrap [--matlab|--cython] interfacePath moduleName toolboxPath cythonImports" << endl;
-  cerr << "  interfacePath : *absolute* path to directory of module interface file" << endl;
+  cerr << "  interfacePath : path to directory of module interface file" << endl;
   cerr << "  moduleName    : the name of the module, interface file must be called moduleName.h" << endl;
   cerr << "  toolboxPath   : the directory in which to generate the wrappers" << endl;
-  cerr << "  headerPath    : path to matlab.h" << endl;
+  cerr << "  cythonImports : extra imports for Cython pxd header file" << endl;
 }
 
 /**
@@ -44,7 +44,7 @@ void generate_toolbox(
            const string& interfacePath,
            const string& moduleName,
            const string& toolboxPath,
-           const string& headerPath)
+           const string& cythonImports)
 {
   // Parse interface file into class object
   // This recursively creates Class objects, Method objects, etc...
@@ -54,7 +54,7 @@ void generate_toolbox(
   // Then emit MATLAB code
     module.matlab_code(toolboxPath);
   else if (language == "--cython") {
-    module.cython_wrapper(toolboxPath);
+    module.cython_wrapper(toolboxPath, cythonImports);
   }
   else {
       cerr << "First argument invalid" << endl;