From 66d3b95a6db2c261b2d280c11d19768d16c8874c Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Wed, 24 Mar 2021 16:14:55 -0400 Subject: [PATCH] Squashed 'wrap/' changes from 3eff76f60..548e61b1f 548e61b1f Merge pull request #57 from borglab/fix/configurable-matlab-include b58eabaf1 set correct template file path 483cdab9c fix 1f393516d fix CI syntax 8f0a3543f more concise cmake command because we don't care about the extra files generated 641ad1326 update CI to run cmake de6b9260f added CMake variable to configure the include directory for matlab.h cbe5f18bc Merge pull request #54 from borglab/feature/refactor2 cc78ee3bb test formatting 046a50b01 break down interface_parser into a submodule of smaller parts git-subtree-dir: wrap git-subtree-split: 548e61b1fbf02759d2e4a52435c2f1b3cbde98f0 --- .github/workflows/linux-ci.yml | 1 + .github/workflows/macos-ci.yml | 2 + .gitignore | 4 +- CMakeLists.txt | 7 + gtwrap/interface_parser.py | 951 ------------------------- gtwrap/interface_parser/__init__.py | 43 ++ gtwrap/interface_parser/classes.py | 282 ++++++++ gtwrap/interface_parser/declaration.py | 60 ++ gtwrap/interface_parser/function.py | 166 +++++ gtwrap/interface_parser/module.py | 55 ++ gtwrap/interface_parser/namespace.py | 128 ++++ gtwrap/interface_parser/template.py | 90 +++ gtwrap/interface_parser/tokens.py | 48 ++ gtwrap/interface_parser/type.py | 232 ++++++ gtwrap/matlab_wrapper.py | 10 +- gtwrap/pybind_wrapper.py | 1 - templates/matlab_wrapper.tpl.in | 2 + tests/test_interface_parser.py | 8 +- tests/test_pybind_wrapper.py | 28 +- 19 files changed, 1144 insertions(+), 974 deletions(-) delete mode 100644 gtwrap/interface_parser.py create mode 100644 gtwrap/interface_parser/__init__.py create mode 100644 gtwrap/interface_parser/classes.py create mode 100644 gtwrap/interface_parser/declaration.py create mode 100644 gtwrap/interface_parser/function.py create mode 100644 gtwrap/interface_parser/module.py create mode 100644 gtwrap/interface_parser/namespace.py create mode 100644 gtwrap/interface_parser/template.py create mode 100644 gtwrap/interface_parser/tokens.py create mode 100644 gtwrap/interface_parser/type.py create mode 100644 templates/matlab_wrapper.tpl.in diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 3d7232acd..34623385e 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -33,6 +33,7 @@ jobs: - name: Build and Test run: | + cmake . cd tests # Use Pytest to run all the tests. pytest diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index cd0571b34..3910d28d8 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -31,6 +31,8 @@ jobs: - name: Build and Test run: | + cmake . cd tests # Use Pytest to run all the tests. pytest + diff --git a/.gitignore b/.gitignore index 4bc4f119e..ed9bd8621 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ __pycache__/ *.egg-info # Files related to code coverage stats -**/.coverage \ No newline at end of file +**/.coverage + +gtwrap/matlab_wrapper.tpl diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b1bbc1fe..883f438e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,13 @@ else() set(SCRIPT_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/cmake") endif() +# Configure the include directory for matlab.h +# This allows the #include to be either gtwrap/matlab.h, wrap/matlab.h or something custom. +if(NOT DEFINED GTWRAP_INCLUDE_NAME) + set(GTWRAP_INCLUDE_NAME "gtwrap" CACHE INTERNAL "Directory name for Matlab includes") +endif() +configure_file(${PROJECT_SOURCE_DIR}/templates/matlab_wrapper.tpl.in ${PROJECT_SOURCE_DIR}/gtwrap/matlab_wrapper.tpl) + # Install CMake scripts to the standard CMake script directory. install(FILES cmake/gtwrapConfig.cmake cmake/MatlabWrap.cmake cmake/PybindWrap.cmake cmake/GtwrapUtils.cmake diff --git a/gtwrap/interface_parser.py b/gtwrap/interface_parser.py deleted file mode 100644 index 157de555a..000000000 --- a/gtwrap/interface_parser.py +++ /dev/null @@ -1,951 +0,0 @@ -""" -GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, -Atlanta, Georgia 30332-0415 -All Rights Reserved - -See LICENSE for the license information - -Parser to get the interface of a C++ source file -Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert -""" - -# pylint: disable=unnecessary-lambda, unused-import, expression-not-assigned, no-else-return, protected-access, too-few-public-methods, too-many-arguments - -import sys -from typing import Iterable, Union, Tuple, List - -import pyparsing # type: ignore -from pyparsing import (CharsNotIn, Forward, Group, Keyword, Literal, OneOrMore, - Optional, Or, ParseException, ParseResults, ParserElement, Suppress, - Word, ZeroOrMore, alphanums, alphas, cppStyleComment, - delimitedList, empty, nums, stringEnd) - -# Fix deepcopy issue with pyparsing -# Can remove once https://github.com/pyparsing/pyparsing/issues/208 is resolved. -if sys.version_info >= (3, 8): - def fixed_get_attr(self, item): - """ - Fix for monkey-patching issue with deepcopy in pyparsing.ParseResults - """ - if item == '__deepcopy__': - raise AttributeError(item) - try: - return self[item] - except KeyError: - return "" - - # apply the monkey-patch - pyparsing.ParseResults.__getattr__ = fixed_get_attr - - -ParserElement.enablePackrat() - -# rule for identifiers (e.g. variable names) -IDENT = Word(alphas + '_', alphanums + '_') ^ Word(nums) - -RAW_POINTER, SHARED_POINTER, REF = map(Literal, "@*&") - -LPAREN, RPAREN, LBRACE, RBRACE, COLON, SEMI_COLON = map(Suppress, "(){}:;") -LOPBRACK, ROPBRACK, COMMA, EQUAL = map(Suppress, "<>,=") -CONST, VIRTUAL, CLASS, STATIC, PAIR, TEMPLATE, TYPEDEF, INCLUDE = map( - Keyword, - [ - "const", - "virtual", - "class", - "static", - "pair", - "template", - "typedef", - "#include", - ], -) -NAMESPACE = Keyword("namespace") -BASIS_TYPES = map( - Keyword, - [ - "void", - "bool", - "unsigned char", - "char", - "int", - "size_t", - "double", - "float", - ], -) - - -class Typename: - """ - Generic type which can be either a basic type or a class type, - similar to C++'s `typename` aka a qualified dependent type. - Contains type name with full namespace and template arguments. - - E.g. - ``` - gtsam::PinholeCamera - ``` - - will give the name as `PinholeCamera`, namespace as `gtsam`, - and template instantiations as `[gtsam::Cal3S2]`. - - Args: - namespaces_and_name: A list representing the namespaces of the type - with the type being the last element. - instantiations: Template parameters to the type. - """ - - namespaces_name_rule = delimitedList(IDENT, "::") - instantiation_name_rule = delimitedList(IDENT, "::") - rule = Forward() - rule << ( - namespaces_name_rule("namespaces_and_name") # - + Optional( - (LOPBRACK + delimitedList(rule, ",")("instantiations") + ROPBRACK)) - ).setParseAction(lambda t: Typename(t.namespaces_and_name, t.instantiations)) - - def __init__(self, - namespaces_and_name: ParseResults, - instantiations: Union[tuple, list, str, ParseResults] = ()): - self.name = namespaces_and_name[-1] # the name is the last element in this list - self.namespaces = namespaces_and_name[:-1] - - if instantiations: - if isinstance(instantiations, Iterable): - self.instantiations = instantiations # type: ignore - else: - self.instantiations = instantiations.asList() - else: - self.instantiations = [] - - if self.name in ["Matrix", "Vector"] and not self.namespaces: - self.namespaces = ["gtsam"] - - @staticmethod - def from_parse_result(parse_result: Union[str, list]): - """Unpack the parsed result to get the Typename instance.""" - return parse_result[0] - - def __repr__(self) -> str: - return self.to_cpp() - - def instantiated_name(self) -> str: - """Get the instantiated name of the type.""" - res = self.name - for instantiation in self.instantiations: - res += instantiation.instantiated_name() - return res - - def to_cpp(self) -> str: - """Generate the C++ code for wrapping.""" - idx = 1 if self.namespaces and not self.namespaces[0] else 0 - if self.instantiations: - cpp_name = self.name + "<{}>".format( - ", ".join([inst.to_cpp() for inst in self.instantiations]) - ) - else: - cpp_name = self.name - return '{}{}{}'.format( - "::".join(self.namespaces[idx:]), - "::" if self.namespaces[idx:] else "", - cpp_name, - ) - - def __eq__(self, other) -> bool: - if isinstance(other, Typename): - return str(self) == str(other) - else: - return False - - def __ne__(self, other) -> bool: - res = self.__eq__(other) - return not res - - -class QualifiedType: - """Type with qualifiers, such as `const`.""" - - rule = ( - Typename.rule("typename") # - + Optional(SHARED_POINTER("is_shared_ptr") | RAW_POINTER("is_ptr") | REF("is_ref")) - ).setParseAction( - lambda t: QualifiedType(t) - ) - - def __init__(self, t: ParseResults): - self.typename = Typename.from_parse_result(t.typename) - self.is_shared_ptr = t.is_shared_ptr - self.is_ptr = t.is_ptr - self.is_ref = t.is_ref - -class BasisType: - """ - Basis types are the built-in types in C++ such as double, int, char, etc. - - When using templates, the basis type will take on the same form as the template. - - E.g. - ``` - template - void func(const T& x); - ``` - - will give - - ``` - m_.def("CoolFunctionDoubleDouble",[](const double& s) { - return wrap_example::CoolFunction(s); - }, py::arg("s")); - ``` - """ - - rule = ( - Or(BASIS_TYPES)("typename") # - + Optional(SHARED_POINTER("is_shared_ptr") | RAW_POINTER("is_ptr") | REF("is_ref")) # - ).setParseAction(lambda t: BasisType(t)) - - def __init__(self, t: ParseResults): - self.typename = Typename([t.typename]) - self.is_ptr = t.is_ptr - self.is_shared_ptr = t.is_shared_ptr - self.is_ref = t.is_ref - -class Type: - """The type value that is parsed, e.g. void, string, size_t.""" - rule = ( - Optional(CONST("is_const")) # - + (BasisType.rule("basis") | QualifiedType.rule("qualified")) # BR - ).setParseAction(lambda t: Type.from_parse_result(t)) - - def __init__(self, typename: Typename, is_const: str, is_shared_ptr: str, - is_ptr: str, is_ref: str, is_basis: bool): - self.typename = typename - self.is_const = is_const - self.is_shared_ptr = is_shared_ptr - self.is_ptr = is_ptr - self.is_ref = is_ref - self.is_basis = is_basis - - @staticmethod - def from_parse_result(t: ParseResults): - """Return the resulting Type from parsing the source.""" - if t.basis: - return Type( - typename=t.basis.typename, - is_const=t.is_const, - is_shared_ptr=t.basis.is_shared_ptr, - is_ptr=t.basis.is_ptr, - is_ref=t.basis.is_ref, - is_basis=True, - ) - elif t.qualified: - return Type( - typename=t.qualified.typename, - is_const=t.is_const, - is_shared_ptr=t.qualified.is_shared_ptr, - is_ptr=t.qualified.is_ptr, - is_ref=t.qualified.is_ref, - is_basis=False, - ) - else: - raise ValueError("Parse result is not a Type") - - def __repr__(self) -> str: - return "{self.typename} " \ - "{self.is_const}{self.is_shared_ptr}{self.is_ptr}{self.is_ref}".format( - self=self) - - def to_cpp(self, use_boost: bool) -> str: - """ - Generate the C++ code for wrapping. - - Treat all pointers as "const shared_ptr&" - Treat Matrix and Vector as "const Matrix&" and "const Vector&" resp. - """ - shared_ptr_ns = "boost" if use_boost else "std" - - if self.is_shared_ptr: - # always pass by reference: https://stackoverflow.com/a/8741626/1236990 - typename = "{ns}::shared_ptr<{typename}>&".format( - ns=shared_ptr_ns, typename=self.typename.to_cpp()) - elif self.is_ptr: - typename = "{typename}*".format(typename=self.typename.to_cpp()) - elif self.is_ref or self.typename.name in ["Matrix", "Vector"]: - typename = typename = "{typename}&".format( - typename=self.typename.to_cpp()) - else: - typename = self.typename.to_cpp() - - return ("{const}{typename}".format( - const="const " if - (self.is_const - or self.typename.name in ["Matrix", "Vector"]) else "", - typename=typename)) - - -class Argument: - """ - The type and name of a function/method argument. - - E.g. - ``` - void sayHello(/*`s` is the method argument with type `const string&`*/ const string& s); - ``` - """ - rule = (Type.rule("ctype") + - IDENT("name")).setParseAction(lambda t: Argument(t.ctype, t.name)) - - def __init__(self, ctype: Type, name: str): - self.ctype = ctype - self.name = name - self.parent: Union[ArgumentList, None] = None - - def __repr__(self) -> str: - return '{} {}'.format(self.ctype.__repr__(), self.name) - - -class ArgumentList: - """ - List of Argument objects for all arguments in a function. - """ - rule = Optional(delimitedList(Argument.rule)("args_list")).setParseAction( - lambda t: ArgumentList.from_parse_result(t.args_list) - ) - - def __init__(self, args_list: List[Argument]): - self.args_list = args_list - for arg in args_list: - arg.parent = self - self.parent: Union[Method, StaticMethod, Template, Constructor, - GlobalFunction, None] = None - - @staticmethod - def from_parse_result(parse_result: ParseResults): - """Return the result of parsing.""" - if parse_result: - return ArgumentList(parse_result.asList()) - else: - return ArgumentList([]) - - def __repr__(self) -> str: - return self.args_list.__repr__() - - def __len__(self) -> int: - return len(self.args_list) - - def args_names(self) -> List[str]: - """Return a list of the names of all the arguments.""" - return [arg.name for arg in self.args_list] - - def to_cpp(self, use_boost: bool) -> List[str]: - """Generate the C++ code for wrapping.""" - return [arg.ctype.to_cpp(use_boost) for arg in self.args_list] - - -class ReturnType: - """ - Rule to parse the return type. - - The return type can either be a single type or a pair such as . - """ - _pair = ( - PAIR.suppress() # - + LOPBRACK # - + Type.rule("type1") # - + COMMA # - + Type.rule("type2") # - + ROPBRACK # - ) - rule = (_pair ^ Type.rule("type1")).setParseAction( # BR - lambda t: ReturnType(t.type1, t.type2)) - - def __init__(self, type1: Type, type2: Type): - self.type1 = type1 - self.type2 = type2 - self.parent: Union[Method, StaticMethod, GlobalFunction, None] = None - - def is_void(self) -> bool: - """ - Check if the return type is void. - """ - return self.type1.typename.name == "void" and not self.type2 - - def __repr__(self) -> str: - return "{}{}".format( - self.type1, (', ' + self.type2.__repr__()) if self.type2 else '') - - def to_cpp(self, use_boost: bool) -> str: - """ - Generate the C++ code for wrapping. - - If there are two return types, we return a pair<>, - otherwise we return the regular return type. - """ - if self.type2: - return "std::pair<{type1},{type2}>".format( - type1=self.type1.to_cpp(use_boost), - type2=self.type2.to_cpp(use_boost)) - else: - return self.type1.to_cpp(use_boost) - - -class Template: - """ - Rule to parse templated values in the interface file. - - E.g. - template // this is the Template. - class Camera { ... }; - """ - class TypenameAndInstantiations: - """ - Rule to parse the template parameters. - - template // POSE is the Instantiation. - """ - rule = ( - IDENT("typename") # - + Optional( # - EQUAL # - + LBRACE # - + ((delimitedList(Typename.rule)("instantiations"))) # - + RBRACE # - )).setParseAction(lambda t: Template.TypenameAndInstantiations( - t.typename, t.instantiations)) - - def __init__(self, typename: str, instantiations: ParseResults): - self.typename = typename - - if instantiations: - self.instantiations = instantiations.asList() - else: - self.instantiations = [] - - rule = ( # BR - TEMPLATE # - + LOPBRACK # - + delimitedList(TypenameAndInstantiations.rule)( - "typename_and_instantiations_list") # - + ROPBRACK # BR - ).setParseAction( - lambda t: Template(t.typename_and_instantiations_list.asList())) - - def __init__(self, typename_and_instantiations_list: List[TypenameAndInstantiations]): - ti_list = typename_and_instantiations_list - self.typenames = [ti.typename for ti in ti_list] - self.instantiations = [ti.instantiations for ti in ti_list] - - def __repr__(self) -> str: - return "<{0}>".format(", ".join(self.typenames)) - - -class Method: - """ - Rule to parse a method in a class. - - E.g. - ``` - class Hello { - void sayHello() const; - }; - ``` - """ - rule = ( - Optional(Template.rule("template")) # - + ReturnType.rule("return_type") # - + IDENT("name") # - + LPAREN # - + ArgumentList.rule("args_list") # - + RPAREN # - + Optional(CONST("is_const")) # - + SEMI_COLON # BR - ).setParseAction(lambda t: Method(t.template, t.name, t.return_type, t. - args_list, t.is_const)) - - def __init__(self, - template: str, - name: str, - return_type: ReturnType, - args: ArgumentList, - is_const: str, - parent: Union[str, "Class"] = ''): - self.template = template - self.name = name - self.return_type = return_type - self.args = args - self.is_const = is_const - - self.parent = parent - - def __repr__(self) -> str: - return "Method: {} {} {}({}){}".format( - self.template, - self.return_type, - self.name, - self.args, - self.is_const, - ) - - -class StaticMethod: - """ - Rule to parse all the static methods in a class. - - E.g. - ``` - class Hello { - static void changeGreeting(); - }; - ``` - """ - rule = ( - STATIC # - + ReturnType.rule("return_type") # - + IDENT("name") # - + LPAREN # - + ArgumentList.rule("args_list") # - + RPAREN # - + SEMI_COLON # BR - ).setParseAction( - lambda t: StaticMethod(t.name, t.return_type, t.args_list)) - - def __init__(self, - name: str, - return_type: ReturnType, - args: ArgumentList, - parent: Union[str, "Class"] = ''): - self.name = name - self.return_type = return_type - self.args = args - - self.parent = parent - - def __repr__(self) -> str: - return "static {} {}{}".format(self.return_type, self.name, self.args) - - def to_cpp(self) -> str: - """Generate the C++ code for wrapping.""" - return self.name - - -class Constructor: - """ - Rule to parse the class constructor. - Can have 0 or more arguments. - """ - rule = ( - IDENT("name") # - + LPAREN # - + ArgumentList.rule("args_list") # - + RPAREN # - + SEMI_COLON # BR - ).setParseAction(lambda t: Constructor(t.name, t.args_list)) - - def __init__(self, name: str, args: ArgumentList, parent: Union["Class", str] =''): - self.name = name - self.args = args - - self.parent = parent - - def __repr__(self) -> str: - return "Constructor: {}".format(self.name) - - -class Property: - """ - Rule to parse the variable members of a class. - - E.g. - ``` - class Hello { - string name; // This is a property. - }; - ```` - """ - rule = ( - Type.rule("ctype") # - + IDENT("name") # - + SEMI_COLON # - ).setParseAction(lambda t: Property(t.ctype, t.name)) - - def __init__(self, ctype: Type, name: str, parent=''): - self.ctype = ctype - self.name = name - self.parent = parent - - def __repr__(self) -> str: - return '{} {}'.format(self.ctype.__repr__(), self.name) - - -def collect_namespaces(obj): - """ - Get the chain of namespaces from the lowest to highest for the given object. - - Args: - obj: Object of type Namespace, Class or InstantiatedClass. - """ - namespaces = [] - ancestor = obj.parent - while ancestor and ancestor.name: - namespaces = [ancestor.name] + namespaces - ancestor = ancestor.parent - return [''] + namespaces - - -class Class: - """ - Rule to parse a class defined in the interface file. - - E.g. - ``` - class Hello { - ... - }; - ``` - """ - class MethodsAndProperties: - """ - Rule for all the methods and properties within a class. - """ - rule = ZeroOrMore( - Constructor.rule ^ StaticMethod.rule ^ Method.rule ^ Property.rule - ).setParseAction(lambda t: Class.MethodsAndProperties(t.asList())) - - def __init__(self, methods_props: List[Union[Constructor, Method, - StaticMethod, Property]]): - self.ctors = [] - self.methods = [] - self.static_methods = [] - self.properties = [] - for m in methods_props: - if isinstance(m, Constructor): - self.ctors.append(m) - elif isinstance(m, Method): - self.methods.append(m) - elif isinstance(m, StaticMethod): - self.static_methods.append(m) - elif isinstance(m, Property): - self.properties.append(m) - - _parent = COLON + Typename.rule("parent_class") - rule = ( - Optional(Template.rule("template")) # - + Optional(VIRTUAL("is_virtual")) # - + CLASS # - + IDENT("name") # - + Optional(_parent) # - + LBRACE # - + MethodsAndProperties.rule("methods_props") # - + RBRACE # - + SEMI_COLON # BR - ).setParseAction(lambda t: Class( - t.template, - t.is_virtual, - t.name, - t.parent_class, - t.methods_props.ctors, - t.methods_props.methods, - t.methods_props.static_methods, - t.methods_props.properties, - )) - - def __init__( - self, - template: Template, - is_virtual: str, - name: str, - parent_class: list, - ctors: List[Constructor], - methods: List[Method], - static_methods: List[StaticMethod], - properties: List[Property], - parent: str = '', - ): - self.template = template - self.is_virtual = is_virtual - self.name = name - if parent_class: - self.parent_class = Typename.from_parse_result(parent_class) - else: - self.parent_class = '' - - self.ctors = ctors - self.methods = methods - self.static_methods = static_methods - self.properties = properties - self.parent = parent - # Make sure ctors' names and class name are the same. - for ctor in self.ctors: - if ctor.name != self.name: - raise ValueError( - "Error in constructor name! {} != {}".format( - ctor.name, self.name - ) - ) - - for ctor in self.ctors: - ctor.parent = self - for method in self.methods: - method.parent = self - for static_method in self.static_methods: - static_method.parent = self - for _property in self.properties: - _property.parent = self - - def namespaces(self) -> list: - """Get the namespaces which this class is nested under as a list.""" - return collect_namespaces(self) - - def __repr__(self): - return "Class: {self.name}".format(self=self) - - -class TypedefTemplateInstantiation: - """ - Rule for parsing typedefs (with templates) within the interface file. - - E.g. - ``` - typedef SuperComplexName EasierName; - ``` - """ - rule = ( - TYPEDEF + Typename.rule("typename") + IDENT("new_name") + SEMI_COLON - ).setParseAction( - lambda t: TypedefTemplateInstantiation( - Typename.from_parse_result(t.typename), t.new_name - ) - ) - - def __init__(self, typename: Typename, new_name: str, parent: str=''): - self.typename = typename - self.new_name = new_name - self.parent = parent - - -class Include: - """ - Rule to parse #include directives. - """ - rule = ( - INCLUDE + LOPBRACK + CharsNotIn('>')("header") + ROPBRACK - ).setParseAction(lambda t: Include(t.header)) - - def __init__(self, header: CharsNotIn, parent: str = ''): - self.header = header - self.parent = parent - - def __repr__(self) -> str: - return "#include <{}>".format(self.header) - - -class ForwardDeclaration: - """ - Rule to parse forward declarations in the interface file. - """ - rule = ( - Optional(VIRTUAL("is_virtual")) - + CLASS - + Typename.rule("name") - + Optional(COLON + Typename.rule("parent_type")) - + SEMI_COLON - ).setParseAction( - lambda t: ForwardDeclaration(t.name, t.parent_type, t.is_virtual) - ) - - def __init__(self, - name: Typename, - parent_type: str, - is_virtual: str, - parent: str = ''): - self.name = name - if parent_type: - self.parent_type = Typename.from_parse_result(parent_type) - else: - self.parent_type = '' - - self.is_virtual = is_virtual - self.parent = parent - - def __repr__(self) -> str: - return "ForwardDeclaration: {} {}({})".format(self.is_virtual, - self.name, self.parent) - - -class GlobalFunction: - """ - Rule to parse functions defined in the global scope. - """ - rule = ( - Optional(Template.rule("template")) - + ReturnType.rule("return_type") # - + IDENT("name") # - + LPAREN # - + ArgumentList.rule("args_list") # - + RPAREN # - + SEMI_COLON # - ).setParseAction(lambda t: GlobalFunction(t.name, t.return_type, t. - args_list, t.template)) - - def __init__(self, - name: str, - return_type: ReturnType, - args_list: ArgumentList, - template: Template, - parent: str = ''): - self.name = name - self.return_type = return_type - self.args = args_list - self.template = template - - self.parent = parent - self.return_type.parent = self - self.args.parent = self - - def __repr__(self) -> str: - return "GlobalFunction: {}{}({})".format( - self.return_type, self.name, self.args - ) - - def to_cpp(self) -> str: - """Generate the C++ code for wrapping.""" - return self.name - - -def find_sub_namespace(namespace: "Namespace", - str_namespaces: List["Namespace"]) -> list: - """ - Get the namespaces nested under `namespace`, filtered by a list of namespace strings. - - Args: - namespace: The top-level namespace under which to find sub-namespaces. - str_namespaces: The list of namespace strings to filter against. - """ - if not str_namespaces: - return [namespace] - - sub_namespaces = ( - ns for ns in namespace.content if isinstance(ns, Namespace) - ) - - found_namespaces = [ - ns for ns in sub_namespaces if ns.name == str_namespaces[0] - ] - if not found_namespaces: - return [] - - res = [] - for found_namespace in found_namespaces: - ns = find_sub_namespace(found_namespace, str_namespaces[1:]) - if ns: - res += ns - return res - - -class Namespace: - """Rule for parsing a namespace in the interface file.""" - - rule = Forward() - rule << ( - NAMESPACE # - + IDENT("name") # - + LBRACE # - + ZeroOrMore( # BR - ForwardDeclaration.rule # - ^ Include.rule # - ^ Class.rule # - ^ TypedefTemplateInstantiation.rule # - ^ GlobalFunction.rule # - ^ rule # - )("content") # BR - + RBRACE # - ).setParseAction(lambda t: Namespace.from_parse_result(t)) - - def __init__(self, name: str, content: ZeroOrMore, parent=''): - self.name = name - self.content = content - self.parent = parent - for child in self.content: - child.parent = self - - @staticmethod - def from_parse_result(t: ParseResults): - """Return the result of parsing.""" - if t.content: - content = t.content.asList() - else: - content = [] - return Namespace(t.name, content) - - def find_class_or_function( - self, typename: Typename) -> Union[Class, GlobalFunction]: - """ - Find the Class or GlobalFunction object given its typename. - We have to traverse the tree of namespaces. - """ - found_namespaces = find_sub_namespace(self, typename.namespaces) - res = [] - for namespace in found_namespaces: - classes_and_funcs = (c for c in namespace.content - if isinstance(c, (Class, GlobalFunction))) - res += [c for c in classes_and_funcs if c.name == typename.name] - if not res: - raise ValueError( - "Cannot find class {} in module!".format(typename.name) - ) - elif len(res) > 1: - raise ValueError( - "Found more than one classes {} in module!".format( - typename.name - ) - ) - else: - return res[0] - - def top_level(self) -> "Namespace": - """Return the top leve namespace.""" - if self.name == '' or self.parent == '': - return self - else: - return self.parent.top_level() - - def __repr__(self) -> str: - return "Namespace: {}\n\t{}".format(self.name, self.content) - - def full_namespaces(self) -> List["Namespace"]: - """Get the full namespace list.""" - ancestors = collect_namespaces(self) - if self.name: - ancestors.append(self.name) - return ancestors - - -class Module: - """ - Module is just a global namespace. - - E.g. - ``` - namespace gtsam { - ... - } - ``` - """ - - rule = ( - ZeroOrMore(ForwardDeclaration.rule # - ^ Include.rule # - ^ Class.rule # - ^ TypedefTemplateInstantiation.rule # - ^ GlobalFunction.rule # - ^ Namespace.rule # - ).setParseAction(lambda t: Namespace('', t.asList())) + - stringEnd) - - rule.ignore(cppStyleComment) - - @staticmethod - def parseString(s: str) -> ParseResults: - """Parse the source string and apply the rules.""" - return Module.rule.parseString(s)[0] diff --git a/gtwrap/interface_parser/__init__.py b/gtwrap/interface_parser/__init__.py new file mode 100644 index 000000000..8bb1fc7dd --- /dev/null +++ b/gtwrap/interface_parser/__init__.py @@ -0,0 +1,43 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Parser to get the interface of a C++ source file + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +import sys +import pyparsing + +from .classes import * +from .declaration import * +from .function import * +from .module import * +from .namespace import * +from .template import * +from .tokens import * +from .type import * + +# Fix deepcopy issue with pyparsing +# Can remove once https://github.com/pyparsing/pyparsing/issues/208 is resolved. +if sys.version_info >= (3, 8): + + def fixed_get_attr(self, item): + """ + Fix for monkey-patching issue with deepcopy in pyparsing.ParseResults + """ + if item == '__deepcopy__': + raise AttributeError(item) + try: + return self[item] + except KeyError: + return "" + + # apply the monkey-patch + pyparsing.ParseResults.__getattr__ = fixed_get_attr + +pyparsing.ParserElement.enablePackrat() diff --git a/gtwrap/interface_parser/classes.py b/gtwrap/interface_parser/classes.py new file mode 100644 index 000000000..7332e0bfe --- /dev/null +++ b/gtwrap/interface_parser/classes.py @@ -0,0 +1,282 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Parser classes and rules for parsing C++ classes. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +from typing import List, Union + +from pyparsing import Optional, ZeroOrMore + +from .function import ArgumentList, ReturnType +from .template import Template +from .tokens import (CLASS, COLON, CONST, IDENT, LBRACE, LPAREN, RBRACE, + RPAREN, SEMI_COLON, STATIC, VIRTUAL) +from .type import Type, Typename + + +class Method: + """ + Rule to parse a method in a class. + + E.g. + ``` + class Hello { + void sayHello() const; + }; + ``` + """ + rule = ( + Optional(Template.rule("template")) # + + ReturnType.rule("return_type") # + + IDENT("name") # + + LPAREN # + + ArgumentList.rule("args_list") # + + RPAREN # + + Optional(CONST("is_const")) # + + SEMI_COLON # BR + ).setParseAction(lambda t: Method(t.template, t.name, t.return_type, t. + args_list, t.is_const)) + + def __init__(self, + template: str, + name: str, + return_type: ReturnType, + args: ArgumentList, + is_const: str, + parent: Union[str, "Class"] = ''): + self.template = template + self.name = name + self.return_type = return_type + self.args = args + self.is_const = is_const + + self.parent = parent + + def __repr__(self) -> str: + return "Method: {} {} {}({}){}".format( + self.template, + self.return_type, + self.name, + self.args, + self.is_const, + ) + + +class StaticMethod: + """ + Rule to parse all the static methods in a class. + + E.g. + ``` + class Hello { + static void changeGreeting(); + }; + ``` + """ + rule = ( + STATIC # + + ReturnType.rule("return_type") # + + IDENT("name") # + + LPAREN # + + ArgumentList.rule("args_list") # + + RPAREN # + + SEMI_COLON # BR + ).setParseAction( + lambda t: StaticMethod(t.name, t.return_type, t.args_list)) + + def __init__(self, + name: str, + return_type: ReturnType, + args: ArgumentList, + parent: Union[str, "Class"] = ''): + self.name = name + self.return_type = return_type + self.args = args + + self.parent = parent + + def __repr__(self) -> str: + return "static {} {}{}".format(self.return_type, self.name, self.args) + + def to_cpp(self) -> str: + """Generate the C++ code for wrapping.""" + return self.name + + +class Constructor: + """ + Rule to parse the class constructor. + Can have 0 or more arguments. + """ + rule = ( + IDENT("name") # + + LPAREN # + + ArgumentList.rule("args_list") # + + RPAREN # + + SEMI_COLON # BR + ).setParseAction(lambda t: Constructor(t.name, t.args_list)) + + def __init__(self, + name: str, + args: ArgumentList, + parent: Union["Class", str] = ''): + self.name = name + self.args = args + + self.parent = parent + + def __repr__(self) -> str: + return "Constructor: {}".format(self.name) + + +class Property: + """ + Rule to parse the variable members of a class. + + E.g. + ``` + class Hello { + string name; // This is a property. + }; + ```` + """ + rule = ( + Type.rule("ctype") # + + IDENT("name") # + + SEMI_COLON # + ).setParseAction(lambda t: Property(t.ctype, t.name)) + + def __init__(self, ctype: Type, name: str, parent=''): + self.ctype = ctype + self.name = name + self.parent = parent + + def __repr__(self) -> str: + return '{} {}'.format(self.ctype.__repr__(), self.name) + + +def collect_namespaces(obj): + """ + Get the chain of namespaces from the lowest to highest for the given object. + + Args: + obj: Object of type Namespace, Class or InstantiatedClass. + """ + namespaces = [] + ancestor = obj.parent + while ancestor and ancestor.name: + namespaces = [ancestor.name] + namespaces + ancestor = ancestor.parent + return [''] + namespaces + + +class Class: + """ + Rule to parse a class defined in the interface file. + + E.g. + ``` + class Hello { + ... + }; + ``` + """ + class MethodsAndProperties: + """ + Rule for all the methods and properties within a class. + """ + rule = ZeroOrMore(Constructor.rule ^ StaticMethod.rule ^ Method.rule + ^ Property.rule).setParseAction( + lambda t: Class.MethodsAndProperties(t.asList())) + + def __init__(self, methods_props: List[Union[Constructor, Method, + StaticMethod, Property]]): + self.ctors = [] + self.methods = [] + self.static_methods = [] + self.properties = [] + for m in methods_props: + if isinstance(m, Constructor): + self.ctors.append(m) + elif isinstance(m, Method): + self.methods.append(m) + elif isinstance(m, StaticMethod): + self.static_methods.append(m) + elif isinstance(m, Property): + self.properties.append(m) + + _parent = COLON + Typename.rule("parent_class") + rule = ( + Optional(Template.rule("template")) # + + Optional(VIRTUAL("is_virtual")) # + + CLASS # + + IDENT("name") # + + Optional(_parent) # + + LBRACE # + + MethodsAndProperties.rule("methods_props") # + + RBRACE # + + SEMI_COLON # BR + ).setParseAction(lambda t: Class( + t.template, + t.is_virtual, + t.name, + t.parent_class, + t.methods_props.ctors, + t.methods_props.methods, + t.methods_props.static_methods, + t.methods_props.properties, + )) + + def __init__( + self, + template: Template, + is_virtual: str, + name: str, + parent_class: list, + ctors: List[Constructor], + methods: List[Method], + static_methods: List[StaticMethod], + properties: List[Property], + parent: str = '', + ): + self.template = template + self.is_virtual = is_virtual + self.name = name + if parent_class: + self.parent_class = Typename.from_parse_result(parent_class) + else: + self.parent_class = '' + + self.ctors = ctors + self.methods = methods + self.static_methods = static_methods + self.properties = properties + self.parent = parent + # Make sure ctors' names and class name are the same. + for ctor in self.ctors: + if ctor.name != self.name: + raise ValueError("Error in constructor name! {} != {}".format( + ctor.name, self.name)) + + for ctor in self.ctors: + ctor.parent = self + for method in self.methods: + method.parent = self + for static_method in self.static_methods: + static_method.parent = self + for _property in self.properties: + _property.parent = self + + def namespaces(self) -> list: + """Get the namespaces which this class is nested under as a list.""" + return collect_namespaces(self) + + def __repr__(self): + return "Class: {self.name}".format(self=self) diff --git a/gtwrap/interface_parser/declaration.py b/gtwrap/interface_parser/declaration.py new file mode 100644 index 000000000..ad0b9d8d9 --- /dev/null +++ b/gtwrap/interface_parser/declaration.py @@ -0,0 +1,60 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Classes and rules for declarations such as includes and forward declarations. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +from pyparsing import CharsNotIn, Optional + +from .tokens import (CLASS, COLON, INCLUDE, LOPBRACK, ROPBRACK, SEMI_COLON, + VIRTUAL) +from .type import Typename + + +class Include: + """ + Rule to parse #include directives. + """ + rule = (INCLUDE + LOPBRACK + CharsNotIn('>')("header") + + ROPBRACK).setParseAction(lambda t: Include(t.header)) + + def __init__(self, header: CharsNotIn, parent: str = ''): + self.header = header + self.parent = parent + + def __repr__(self) -> str: + return "#include <{}>".format(self.header) + + +class ForwardDeclaration: + """ + Rule to parse forward declarations in the interface file. + """ + rule = (Optional(VIRTUAL("is_virtual")) + CLASS + Typename.rule("name") + + Optional(COLON + Typename.rule("parent_type")) + + SEMI_COLON).setParseAction(lambda t: ForwardDeclaration( + t.name, t.parent_type, t.is_virtual)) + + def __init__(self, + name: Typename, + parent_type: str, + is_virtual: str, + parent: str = ''): + self.name = name + if parent_type: + self.parent_type = Typename.from_parse_result(parent_type) + else: + self.parent_type = '' + + self.is_virtual = is_virtual + self.parent = parent + + def __repr__(self) -> str: + return "ForwardDeclaration: {} {}({})".format(self.is_virtual, + self.name, self.parent) diff --git a/gtwrap/interface_parser/function.py b/gtwrap/interface_parser/function.py new file mode 100644 index 000000000..453577e58 --- /dev/null +++ b/gtwrap/interface_parser/function.py @@ -0,0 +1,166 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Parser classes and rules for parsing C++ functions. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +from typing import List, Union + +from pyparsing import Optional, ParseResults, delimitedList + +from .template import Template +from .tokens import (COMMA, IDENT, LOPBRACK, LPAREN, PAIR, ROPBRACK, RPAREN, + SEMI_COLON) +from .type import Type + + +class Argument: + """ + The type and name of a function/method argument. + + E.g. + ``` + void sayHello(/*`s` is the method argument with type `const string&`*/ const string& s); + ``` + """ + rule = (Type.rule("ctype") + + IDENT("name")).setParseAction(lambda t: Argument(t.ctype, t.name)) + + def __init__(self, ctype: Type, name: str): + self.ctype = ctype + self.name = name + self.parent: Union[ArgumentList, None] = None + + def __repr__(self) -> str: + return '{} {}'.format(self.ctype.__repr__(), self.name) + + +class ArgumentList: + """ + List of Argument objects for all arguments in a function. + """ + rule = Optional(delimitedList(Argument.rule)("args_list")).setParseAction( + lambda t: ArgumentList.from_parse_result(t.args_list)) + + def __init__(self, args_list: List[Argument]): + self.args_list = args_list + for arg in args_list: + arg.parent = self + # The parent object which contains the argument list + # E.g. Method, StaticMethod, Template, Constructor, GlobalFunction + self.parent = None + + @staticmethod + def from_parse_result(parse_result: ParseResults): + """Return the result of parsing.""" + if parse_result: + return ArgumentList(parse_result.asList()) + else: + return ArgumentList([]) + + def __repr__(self) -> str: + return self.args_list.__repr__() + + def __len__(self) -> int: + return len(self.args_list) + + def args_names(self) -> List[str]: + """Return a list of the names of all the arguments.""" + return [arg.name for arg in self.args_list] + + def to_cpp(self, use_boost: bool) -> List[str]: + """Generate the C++ code for wrapping.""" + return [arg.ctype.to_cpp(use_boost) for arg in self.args_list] + + +class ReturnType: + """ + Rule to parse the return type. + + The return type can either be a single type or a pair such as . + """ + _pair = ( + PAIR.suppress() # + + LOPBRACK # + + Type.rule("type1") # + + COMMA # + + Type.rule("type2") # + + ROPBRACK # + ) + rule = (_pair ^ Type.rule("type1")).setParseAction( # BR + lambda t: ReturnType(t.type1, t.type2)) + + def __init__(self, type1: Type, type2: Type): + self.type1 = type1 + self.type2 = type2 + # The parent object which contains the return type + # E.g. Method, StaticMethod, Template, Constructor, GlobalFunction + self.parent = None + + def is_void(self) -> bool: + """ + Check if the return type is void. + """ + return self.type1.typename.name == "void" and not self.type2 + + def __repr__(self) -> str: + return "{}{}".format( + self.type1, (', ' + self.type2.__repr__()) if self.type2 else '') + + def to_cpp(self, use_boost: bool) -> str: + """ + Generate the C++ code for wrapping. + + If there are two return types, we return a pair<>, + otherwise we return the regular return type. + """ + if self.type2: + return "std::pair<{type1},{type2}>".format( + type1=self.type1.to_cpp(use_boost), + type2=self.type2.to_cpp(use_boost)) + else: + return self.type1.to_cpp(use_boost) + + +class GlobalFunction: + """ + Rule to parse functions defined in the global scope. + """ + rule = ( + Optional(Template.rule("template")) + ReturnType.rule("return_type") # + + IDENT("name") # + + LPAREN # + + ArgumentList.rule("args_list") # + + RPAREN # + + SEMI_COLON # + ).setParseAction(lambda t: GlobalFunction(t.name, t.return_type, t. + args_list, t.template)) + + def __init__(self, + name: str, + return_type: ReturnType, + args_list: ArgumentList, + template: Template, + parent: str = ''): + self.name = name + self.return_type = return_type + self.args = args_list + self.template = template + + self.parent = parent + self.return_type.parent = self + self.args.parent = self + + def __repr__(self) -> str: + return "GlobalFunction: {}{}({})".format(self.return_type, self.name, + self.args) + + def to_cpp(self) -> str: + """Generate the C++ code for wrapping.""" + return self.name diff --git a/gtwrap/interface_parser/module.py b/gtwrap/interface_parser/module.py new file mode 100644 index 000000000..5619c1f56 --- /dev/null +++ b/gtwrap/interface_parser/module.py @@ -0,0 +1,55 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Rules and classes for parsing a module. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +# pylint: disable=unnecessary-lambda, unused-import, expression-not-assigned, no-else-return, protected-access, too-few-public-methods, too-many-arguments + +import sys + +import pyparsing # type: ignore +from pyparsing import (ParserElement, ParseResults, ZeroOrMore, + cppStyleComment, stringEnd) + +from .classes import Class +from .declaration import ForwardDeclaration, Include +from .function import GlobalFunction +from .namespace import Namespace +from .template import TypedefTemplateInstantiation + + +class Module: + """ + Module is just a global namespace. + + E.g. + ``` + namespace gtsam { + ... + } + ``` + """ + + rule = ( + ZeroOrMore(ForwardDeclaration.rule # + ^ Include.rule # + ^ Class.rule # + ^ TypedefTemplateInstantiation.rule # + ^ GlobalFunction.rule # + ^ Namespace.rule # + ).setParseAction(lambda t: Namespace('', t.asList())) + + stringEnd) + + rule.ignore(cppStyleComment) + + @staticmethod + def parseString(s: str) -> ParseResults: + """Parse the source string and apply the rules.""" + return Module.rule.parseString(s)[0] diff --git a/gtwrap/interface_parser/namespace.py b/gtwrap/interface_parser/namespace.py new file mode 100644 index 000000000..da505d5f9 --- /dev/null +++ b/gtwrap/interface_parser/namespace.py @@ -0,0 +1,128 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Classes and rules to parse a namespace. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +# pylint: disable=unnecessary-lambda, expression-not-assigned + +from typing import List, Union + +from pyparsing import Forward, ParseResults, ZeroOrMore + +from .classes import Class, collect_namespaces +from .declaration import ForwardDeclaration, Include +from .function import GlobalFunction +from .template import TypedefTemplateInstantiation +from .tokens import IDENT, LBRACE, NAMESPACE, RBRACE +from .type import Typename + + +def find_sub_namespace(namespace: "Namespace", + str_namespaces: List["Namespace"]) -> list: + """ + Get the namespaces nested under `namespace`, filtered by a list of namespace strings. + + Args: + namespace: The top-level namespace under which to find sub-namespaces. + str_namespaces: The list of namespace strings to filter against. + """ + if not str_namespaces: + return [namespace] + + sub_namespaces = (ns for ns in namespace.content + if isinstance(ns, Namespace)) + + found_namespaces = [ + ns for ns in sub_namespaces if ns.name == str_namespaces[0] + ] + if not found_namespaces: + return [] + + res = [] + for found_namespace in found_namespaces: + ns = find_sub_namespace(found_namespace, str_namespaces[1:]) + if ns: + res += ns + return res + + +class Namespace: + """Rule for parsing a namespace in the interface file.""" + + rule = Forward() + rule << ( + NAMESPACE # + + IDENT("name") # + + LBRACE # + + ZeroOrMore( # BR + ForwardDeclaration.rule # + ^ Include.rule # + ^ Class.rule # + ^ TypedefTemplateInstantiation.rule # + ^ GlobalFunction.rule # + ^ rule # + )("content") # BR + + RBRACE # + ).setParseAction(lambda t: Namespace.from_parse_result(t)) + + def __init__(self, name: str, content: ZeroOrMore, parent=''): + self.name = name + self.content = content + self.parent = parent + for child in self.content: + child.parent = self + + @staticmethod + def from_parse_result(t: ParseResults): + """Return the result of parsing.""" + if t.content: + content = t.content.asList() + else: + content = [] + return Namespace(t.name, content) + + def find_class_or_function( + self, typename: Typename) -> Union[Class, GlobalFunction]: + """ + Find the Class or GlobalFunction object given its typename. + We have to traverse the tree of namespaces. + """ + found_namespaces = find_sub_namespace(self, typename.namespaces) + res = [] + for namespace in found_namespaces: + classes_and_funcs = (c for c in namespace.content + if isinstance(c, (Class, GlobalFunction))) + res += [c for c in classes_and_funcs if c.name == typename.name] + if not res: + raise ValueError("Cannot find class {} in module!".format( + typename.name)) + elif len(res) > 1: + raise ValueError( + "Found more than one classes {} in module!".format( + typename.name)) + else: + return res[0] + + def top_level(self) -> "Namespace": + """Return the top leve namespace.""" + if self.name == '' or self.parent == '': + return self + else: + return self.parent.top_level() + + def __repr__(self) -> str: + return "Namespace: {}\n\t{}".format(self.name, self.content) + + def full_namespaces(self) -> List["Namespace"]: + """Get the full namespace list.""" + ancestors = collect_namespaces(self) + if self.name: + ancestors.append(self.name) + return ancestors diff --git a/gtwrap/interface_parser/template.py b/gtwrap/interface_parser/template.py new file mode 100644 index 000000000..99e929d39 --- /dev/null +++ b/gtwrap/interface_parser/template.py @@ -0,0 +1,90 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Classes and rules for parsing C++ templates and typedefs for template instantiations. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +from typing import List + +from pyparsing import Optional, ParseResults, delimitedList + +from .tokens import (EQUAL, IDENT, LBRACE, LOPBRACK, RBRACE, ROPBRACK, + SEMI_COLON, TEMPLATE, TYPEDEF) +from .type import Typename + + +class Template: + """ + Rule to parse templated values in the interface file. + + E.g. + template // this is the Template. + class Camera { ... }; + """ + class TypenameAndInstantiations: + """ + Rule to parse the template parameters. + + template // POSE is the Instantiation. + """ + rule = ( + IDENT("typename") # + + Optional( # + EQUAL # + + LBRACE # + + ((delimitedList(Typename.rule)("instantiations"))) # + + RBRACE # + )).setParseAction(lambda t: Template.TypenameAndInstantiations( + t.typename, t.instantiations)) + + def __init__(self, typename: str, instantiations: ParseResults): + self.typename = typename + + if instantiations: + self.instantiations = instantiations.asList() + else: + self.instantiations = [] + + rule = ( # BR + TEMPLATE # + + LOPBRACK # + + delimitedList(TypenameAndInstantiations.rule)( + "typename_and_instantiations_list") # + + ROPBRACK # BR + ).setParseAction( + lambda t: Template(t.typename_and_instantiations_list.asList())) + + def __init__( + self, + typename_and_instantiations_list: List[TypenameAndInstantiations]): + ti_list = typename_and_instantiations_list + self.typenames = [ti.typename for ti in ti_list] + self.instantiations = [ti.instantiations for ti in ti_list] + + def __repr__(self) -> str: + return "<{0}>".format(", ".join(self.typenames)) + + +class TypedefTemplateInstantiation: + """ + Rule for parsing typedefs (with templates) within the interface file. + + E.g. + ``` + typedef SuperComplexName EasierName; + ``` + """ + rule = (TYPEDEF + Typename.rule("typename") + IDENT("new_name") + + SEMI_COLON).setParseAction(lambda t: TypedefTemplateInstantiation( + Typename.from_parse_result(t.typename), t.new_name)) + + def __init__(self, typename: Typename, new_name: str, parent: str = ''): + self.typename = typename + self.new_name = new_name + self.parent = parent diff --git a/gtwrap/interface_parser/tokens.py b/gtwrap/interface_parser/tokens.py new file mode 100644 index 000000000..6653812c4 --- /dev/null +++ b/gtwrap/interface_parser/tokens.py @@ -0,0 +1,48 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +All the token definitions. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +from pyparsing import Keyword, Literal, Suppress, Word, alphanums, alphas, nums + +# rule for identifiers (e.g. variable names) +IDENT = Word(alphas + '_', alphanums + '_') ^ Word(nums) + +RAW_POINTER, SHARED_POINTER, REF = map(Literal, "@*&") + +LPAREN, RPAREN, LBRACE, RBRACE, COLON, SEMI_COLON = map(Suppress, "(){}:;") +LOPBRACK, ROPBRACK, COMMA, EQUAL = map(Suppress, "<>,=") +CONST, VIRTUAL, CLASS, STATIC, PAIR, TEMPLATE, TYPEDEF, INCLUDE = map( + Keyword, + [ + "const", + "virtual", + "class", + "static", + "pair", + "template", + "typedef", + "#include", + ], +) +NAMESPACE = Keyword("namespace") +BASIS_TYPES = map( + Keyword, + [ + "void", + "bool", + "unsigned char", + "char", + "int", + "size_t", + "double", + "float", + ], +) diff --git a/gtwrap/interface_parser/type.py b/gtwrap/interface_parser/type.py new file mode 100644 index 000000000..4578b9f37 --- /dev/null +++ b/gtwrap/interface_parser/type.py @@ -0,0 +1,232 @@ +""" +GTSAM Copyright 2010-2020, Georgia Tech Research Corporation, +Atlanta, Georgia 30332-0415 +All Rights Reserved + +See LICENSE for the license information + +Define the parser rules and classes for various C++ types. + +Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert +""" + +# pylint: disable=unnecessary-lambda, expression-not-assigned + +from typing import Iterable, Union + +from pyparsing import Forward, Optional, Or, ParseResults, delimitedList + +from .tokens import (BASIS_TYPES, CONST, IDENT, LOPBRACK, RAW_POINTER, REF, + ROPBRACK, SHARED_POINTER) + + +class Typename: + """ + Generic type which can be either a basic type or a class type, + similar to C++'s `typename` aka a qualified dependent type. + Contains type name with full namespace and template arguments. + + E.g. + ``` + gtsam::PinholeCamera + ``` + + will give the name as `PinholeCamera`, namespace as `gtsam`, + and template instantiations as `[gtsam::Cal3S2]`. + + Args: + namespaces_and_name: A list representing the namespaces of the type + with the type being the last element. + instantiations: Template parameters to the type. + """ + + namespaces_name_rule = delimitedList(IDENT, "::") + instantiation_name_rule = delimitedList(IDENT, "::") + rule = Forward() + rule << ( + namespaces_name_rule("namespaces_and_name") # + + Optional( + (LOPBRACK + delimitedList(rule, ",") + ("instantiations") + ROPBRACK))).setParseAction( + lambda t: Typename(t.namespaces_and_name, t.instantiations)) + + def __init__(self, + namespaces_and_name: ParseResults, + instantiations: Union[tuple, list, str, ParseResults] = ()): + self.name = namespaces_and_name[ + -1] # the name is the last element in this list + self.namespaces = namespaces_and_name[:-1] + + if instantiations: + if isinstance(instantiations, Iterable): + self.instantiations = instantiations # type: ignore + else: + self.instantiations = instantiations.asList() + else: + self.instantiations = [] + + if self.name in ["Matrix", "Vector"] and not self.namespaces: + self.namespaces = ["gtsam"] + + @staticmethod + def from_parse_result(parse_result: Union[str, list]): + """Unpack the parsed result to get the Typename instance.""" + return parse_result[0] + + def __repr__(self) -> str: + return self.to_cpp() + + def instantiated_name(self) -> str: + """Get the instantiated name of the type.""" + res = self.name + for instantiation in self.instantiations: + res += instantiation.instantiated_name() + return res + + def to_cpp(self) -> str: + """Generate the C++ code for wrapping.""" + idx = 1 if self.namespaces and not self.namespaces[0] else 0 + if self.instantiations: + cpp_name = self.name + "<{}>".format(", ".join( + [inst.to_cpp() for inst in self.instantiations])) + else: + cpp_name = self.name + return '{}{}{}'.format( + "::".join(self.namespaces[idx:]), + "::" if self.namespaces[idx:] else "", + cpp_name, + ) + + def __eq__(self, other) -> bool: + if isinstance(other, Typename): + return str(self) == str(other) + else: + return False + + def __ne__(self, other) -> bool: + res = self.__eq__(other) + return not res + + +class QualifiedType: + """Type with qualifiers, such as `const`.""" + + rule = ( + Typename.rule("typename") # + + Optional( + SHARED_POINTER("is_shared_ptr") | RAW_POINTER("is_ptr") + | REF("is_ref"))).setParseAction(lambda t: QualifiedType(t)) + + def __init__(self, t: ParseResults): + self.typename = Typename.from_parse_result(t.typename) + self.is_shared_ptr = t.is_shared_ptr + self.is_ptr = t.is_ptr + self.is_ref = t.is_ref + + +class BasisType: + """ + Basis types are the built-in types in C++ such as double, int, char, etc. + + When using templates, the basis type will take on the same form as the template. + + E.g. + ``` + template + void func(const T& x); + ``` + + will give + + ``` + m_.def("CoolFunctionDoubleDouble",[](const double& s) { + return wrap_example::CoolFunction(s); + }, py::arg("s")); + ``` + """ + + rule = ( + Or(BASIS_TYPES)("typename") # + + Optional( + SHARED_POINTER("is_shared_ptr") | RAW_POINTER("is_ptr") + | REF("is_ref")) # + ).setParseAction(lambda t: BasisType(t)) + + def __init__(self, t: ParseResults): + self.typename = Typename([t.typename]) + self.is_ptr = t.is_ptr + self.is_shared_ptr = t.is_shared_ptr + self.is_ref = t.is_ref + + +class Type: + """The type value that is parsed, e.g. void, string, size_t.""" + rule = ( + Optional(CONST("is_const")) # + + (BasisType.rule("basis") | QualifiedType.rule("qualified")) # BR + ).setParseAction(lambda t: Type.from_parse_result(t)) + + def __init__(self, typename: Typename, is_const: str, is_shared_ptr: str, + is_ptr: str, is_ref: str, is_basis: bool): + self.typename = typename + self.is_const = is_const + self.is_shared_ptr = is_shared_ptr + self.is_ptr = is_ptr + self.is_ref = is_ref + self.is_basis = is_basis + + @staticmethod + def from_parse_result(t: ParseResults): + """Return the resulting Type from parsing the source.""" + if t.basis: + return Type( + typename=t.basis.typename, + is_const=t.is_const, + is_shared_ptr=t.basis.is_shared_ptr, + is_ptr=t.basis.is_ptr, + is_ref=t.basis.is_ref, + is_basis=True, + ) + elif t.qualified: + return Type( + typename=t.qualified.typename, + is_const=t.is_const, + is_shared_ptr=t.qualified.is_shared_ptr, + is_ptr=t.qualified.is_ptr, + is_ref=t.qualified.is_ref, + is_basis=False, + ) + else: + raise ValueError("Parse result is not a Type") + + def __repr__(self) -> str: + return "{self.typename} " \ + "{self.is_const}{self.is_shared_ptr}{self.is_ptr}{self.is_ref}".format( + self=self) + + def to_cpp(self, use_boost: bool) -> str: + """ + Generate the C++ code for wrapping. + + Treat all pointers as "const shared_ptr&" + Treat Matrix and Vector as "const Matrix&" and "const Vector&" resp. + """ + shared_ptr_ns = "boost" if use_boost else "std" + + if self.is_shared_ptr: + # always pass by reference: https://stackoverflow.com/a/8741626/1236990 + typename = "{ns}::shared_ptr<{typename}>&".format( + ns=shared_ptr_ns, typename=self.typename.to_cpp()) + elif self.is_ptr: + typename = "{typename}*".format(typename=self.typename.to_cpp()) + elif self.is_ref or self.typename.name in ["Matrix", "Vector"]: + typename = typename = "{typename}&".format( + typename=self.typename.to_cpp()) + else: + typename = self.typename.to_cpp() + + return ("{const}{typename}".format( + const="const " if + (self.is_const + or self.typename.name in ["Matrix", "Vector"]) else "", + typename=typename)) diff --git a/gtwrap/matlab_wrapper.py b/gtwrap/matlab_wrapper.py index 439a61fb4..520b76284 100755 --- a/gtwrap/matlab_wrapper.py +++ b/gtwrap/matlab_wrapper.py @@ -76,6 +76,10 @@ class MatlabWrapper(object): # Files and their content content: List[str] = [] + # Ensure the template file is always picked up from the correct directory. + dir_path = osp.dirname(osp.realpath(__file__)) + wrapper_file_template = osp.join(dir_path, "matlab_wrapper.tpl") + def __init__(self, module, module_name, @@ -664,10 +668,8 @@ class MatlabWrapper(object): """Generate the C++ file for the wrapper.""" file_name = self._wrapper_name() + '.cpp' - wrapper_file = textwrap.dedent('''\ - # include - # include - ''') + with open(self.wrapper_file_template) as f: + wrapper_file = f.read() return file_name, wrapper_file diff --git a/gtwrap/pybind_wrapper.py b/gtwrap/pybind_wrapper.py index 3d82298da..c275e575d 100755 --- a/gtwrap/pybind_wrapper.py +++ b/gtwrap/pybind_wrapper.py @@ -13,7 +13,6 @@ Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellae # pylint: disable=too-many-arguments, too-many-instance-attributes, no-self-use, no-else-return, too-many-arguments, unused-format-string-argument, line-too-long import re -import textwrap import gtwrap.interface_parser as parser import gtwrap.template_instantiator as instantiator diff --git a/templates/matlab_wrapper.tpl.in b/templates/matlab_wrapper.tpl.in new file mode 100644 index 000000000..2885e9244 --- /dev/null +++ b/templates/matlab_wrapper.tpl.in @@ -0,0 +1,2 @@ +# include <${GTWRAP_INCLUDE_NAME}/matlab.h> +# include diff --git a/tests/test_interface_parser.py b/tests/test_interface_parser.py index c43802b2a..fad846365 100644 --- a/tests/test_interface_parser.py +++ b/tests/test_interface_parser.py @@ -16,16 +16,13 @@ import os import sys import unittest -from pyparsing import ParseException - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gtwrap.interface_parser import (ArgumentList, Class, Constructor, ForwardDeclaration, GlobalFunction, Include, Method, Module, Namespace, ReturnType, StaticMethod, Type, - TypedefTemplateInstantiation, Typename, - find_sub_namespace) + TypedefTemplateInstantiation, Typename) class TestInterfaceParser(unittest.TestCase): @@ -35,7 +32,8 @@ class TestInterfaceParser(unittest.TestCase): typename = Typename.rule.parseString("size_t")[0] self.assertEqual("size_t", typename.name) - typename = Typename.rule.parseString("gtsam::PinholeCamera")[0] + typename = Typename.rule.parseString( + "gtsam::PinholeCamera")[0] self.assertEqual("PinholeCamera", typename.name) self.assertEqual(["gtsam"], typename.namespaces) self.assertEqual("Cal3S2", typename.instantiations[0].name) diff --git a/tests/test_pybind_wrapper.py b/tests/test_pybind_wrapper.py index 7896ab28b..e2fdbe3bb 100644 --- a/tests/test_pybind_wrapper.py +++ b/tests/test_pybind_wrapper.py @@ -13,7 +13,9 @@ import sys import unittest sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.append(os.path.normpath(os.path.abspath(os.path.join(__file__, '../../../build/wrap')))) +sys.path.append( + os.path.normpath( + os.path.abspath(os.path.join(__file__, '../../../build/wrap')))) import gtwrap.interface_parser as parser import gtwrap.template_instantiator as instantiator @@ -38,14 +40,12 @@ class TestWrap(unittest.TestCase): module_template = template_file.read() # Create Pybind wrapper instance - wrapper = PybindWrapper( - module=module, - module_name=module_name, - use_boost=False, - top_module_namespaces=[''], - ignore_classes=[''], - module_template=module_template - ) + wrapper = PybindWrapper(module=module, + module_name=module_name, + use_boost=False, + top_module_namespaces=[''], + ignore_classes=[''], + module_template=module_template) cc_content = wrapper.wrap() @@ -70,7 +70,8 @@ class TestWrap(unittest.TestCase): output = self.wrap_content(content, 'geometry_py', 'actual-python') - expected = path.join(self.TEST_DIR, 'expected-python/geometry_pybind.cpp') + expected = path.join(self.TEST_DIR, + 'expected-python/geometry_pybind.cpp') success = filecmp.cmp(output, expected) if not success: @@ -86,14 +87,17 @@ class TestWrap(unittest.TestCase): with open(os.path.join(self.TEST_DIR, 'testNamespaces.h'), 'r') as f: content = f.read() - output = self.wrap_content(content, 'testNamespaces_py', 'actual-python') + output = self.wrap_content(content, 'testNamespaces_py', + 'actual-python') - expected = path.join(self.TEST_DIR, 'expected-python/testNamespaces_py.cpp') + expected = path.join(self.TEST_DIR, + 'expected-python/testNamespaces_py.cpp') success = filecmp.cmp(output, expected) if not success: os.system("diff {} {}".format(output, expected)) self.assertTrue(success) + if __name__ == '__main__': unittest.main()