diff --git a/wrap/.github/workflows/linux-ci.yml b/wrap/.github/workflows/linux-ci.yml index 6c7ef1285..1c9ece3a5 100644 --- a/wrap/.github/workflows/linux-ci.yml +++ b/wrap/.github/workflows/linux-ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Checkout diff --git a/wrap/.github/workflows/macos-ci.yml b/wrap/.github/workflows/macos-ci.yml index adba486c5..96bb5fd52 100644 --- a/wrap/.github/workflows/macos-ci.yml +++ b/wrap/.github/workflows/macos-ci.yml @@ -5,12 +5,12 @@ on: [pull_request] jobs: build: name: Tests for 🐍 ${{ matrix.python-version }} - runs-on: macos-12 + runs-on: macos-14 strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Checkout diff --git a/wrap/CMakeLists.txt b/wrap/CMakeLists.txt index 2a11a760d..294c333bd 100644 --- a/wrap/CMakeLists.txt +++ b/wrap/CMakeLists.txt @@ -20,6 +20,8 @@ set(INSTALL_LIB_DIR lib/${PROJECT_NAME}) set(INSTALL_BIN_DIR bin/${PROJECT_NAME}) set(INSTALL_INCLUDE_DIR include/${PROJECT_NAME}) +option(GTWRAP_ADD_DOCSTRINGS "Whether to add docstrings to the Python bindings from Doxygen-generated XML located at {project_root}/xml" OFF) + # ############################################################################## # Package Configuration diff --git a/wrap/cmake/PybindWrap.cmake b/wrap/cmake/PybindWrap.cmake index dd579bd5e..2c98b690d 100644 --- a/wrap/cmake/PybindWrap.cmake +++ b/wrap/cmake/PybindWrap.cmake @@ -13,6 +13,14 @@ gtwrap_get_python_version(${WRAP_PYTHON_VERSION}) message(STATUS "Setting Python version for wrapper") set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION}) +if(GTWRAP_ADD_DOCSTRINGS) + set(GTWRAP_PYTHON_DOCS_SOURCE "${CMAKE_SOURCE_DIR}/xml") + message(STATUS "Python docstring generation is on. XML source: '${GTWRAP_PYTHON_DOCS_SOURCE}'") +else() + message(STATUS "Python docstring generation is off.") + set(GTWRAP_PYTHON_DOCS_SOURCE "") +endif() + # User-friendly Pybind11 wrapping and installing function. Builds a Pybind11 # module from the provided interface_headers. For example, for the interface # header gtsam.h, this will build the wrap module 'gtsam_py.cc'. @@ -82,6 +90,7 @@ function( --out "${cpp_file}" --module_name ${module_name} --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} --template ${module_template} --is_submodule ${_WRAP_BOOST_ARG} + --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" DEPENDS "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h" VERBATIM) @@ -100,6 +109,7 @@ function( --out "${generated_cpp}" --module_name ${module_name} --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes} --template ${module_template} ${_WRAP_BOOST_ARG} + --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}" DEPENDS "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h" VERBATIM) diff --git a/wrap/gtwrap/pybind_wrapper.py b/wrap/gtwrap/pybind_wrapper.py index 479c2d67d..b8d3d8070 100755 --- a/wrap/gtwrap/pybind_wrapper.py +++ b/wrap/gtwrap/pybind_wrapper.py @@ -19,6 +19,7 @@ from typing import List import gtwrap.interface_parser as parser import gtwrap.template_instantiator as instantiator +from gtwrap.xml_parser.xml_parser import XMLDocParser class PybindWrapper: """ @@ -29,8 +30,9 @@ class PybindWrapper: module_name, top_module_namespaces='', use_boost_serialization=False, - ignore_classes=(), - module_template=""): + ignore_classes=(), + module_template="", + xml_source=""): self.module_name = module_name self.top_module_namespaces = top_module_namespaces self.use_boost_serialization = use_boost_serialization @@ -44,6 +46,8 @@ class PybindWrapper: 'nonlocal', 'yield', 'break', 'for', 'not', 'class', 'from', 'or', 'continue', 'global', 'pass' ] + self.xml_source = xml_source + self.xml_parser = XMLDocParser() self.dunder_methods = ('len', 'contains', 'iter') @@ -260,7 +264,7 @@ class PybindWrapper: '[]({opt_self}{opt_comma}{args_signature_with_names}){{' '{function_call}' '}}' - '{py_args_names}){suffix}'.format( + '{py_args_names}{docstring}){suffix}'.format( prefix=prefix, cdef="def_static" if is_static else "def", py_method=py_method, @@ -271,6 +275,12 @@ class PybindWrapper: function_call=function_call, py_args_names=py_args_names, suffix=suffix, + # Try to get the function's docstring from the Doxygen XML. + # If extract_docstring errors or fails to find a docstring, it just prints a warning. + # The incantation repr(...)[1:-1].replace('"', r'\"') replaces newlines with \n + # and " with \" so that the docstring can be put into a C++ string on a single line. + docstring=', "' + repr(self.xml_parser.extract_docstring(self.xml_source, cpp_class, cpp_method, method.args.names()))[1:-1].replace('"', r'\"') + '"' + if self.xml_source != "" else "", )) # Create __repr__ override diff --git a/wrap/gtwrap/xml_parser/__init__.py b/wrap/gtwrap/xml_parser/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/wrap/gtwrap/xml_parser/xml_parser.py b/wrap/gtwrap/xml_parser/xml_parser.py new file mode 100644 index 000000000..9b3d4d804 --- /dev/null +++ b/wrap/gtwrap/xml_parser/xml_parser.py @@ -0,0 +1,304 @@ +import os +import sys +from pathlib import Path +import xml.etree.ElementTree as ET + + +class XMLDocParser: + """ + Parses and extracts docs from Doxygen-generated XML. + """ + + def __init__(self): + # Memory for overloaded functions with identical parameter name sets + self._memory = {} + # This is useful for investigating functions that cause problems for extract_docstring. + # Set this to true to have useful information for debugging this class, as in the CLI + # function at the bottom of this class. + self._verbose = False + + def parse_xml(self, xml_file: str): + """ + Get the ElementTree of an XML file given the file name. + If an error occurs, prints a warning and returns None. + """ + try: + return ET.parse(xml_file) + except FileNotFoundError: + print(f"Warning: XML file '{xml_file}' not found.") + return None + except ET.ParseError: + print(f"Warning: Failed to parse XML file '{xml_file}'.") + return None + + def extract_docstring(self, xml_folder: str, cpp_class: str, + cpp_method: str, method_args_names: 'list[str]'): + """ + Extract the docstrings for a C++ class's method from the Doxygen-generated XML. + + Args: + xml_folder (str): The path to the folder that contains all of the Doxygen-generated XML. + cpp_class (str): The name of the C++ class that contains the function whose docstring is to be extracted. + cpp_method (str): The name of the C++ method whose docstring is to be extracted. + method_args_names (list): A list of the names of the cpp_method's parameters. + """ + self.print_if_verbose(f"Extracting docs for {cpp_class}.{cpp_method}") + + # Get all of the member definitions in cpp_class with name cpp_method + maybe_member_defs = self.get_member_defs(xml_folder, cpp_class, + cpp_method) + + # Filter member definitions which don't match the given argument names + member_defs, ignored_params = self.filter_member_defs( + maybe_member_defs, method_args_names) + + # Find which member to get docs from, if there are multiple that match in name and args + documenting_index = self.determine_documenting_index( + cpp_class, cpp_method, method_args_names) + + # Extract the docs for the function that matches cpp_class.cpp_method(*method_args_names). + return self.get_formatted_docstring(member_defs[documenting_index], + ignored_params) + + def get_member_defs(self, xml_folder: str, cpp_class: str, + cpp_method: str): + """Get all of the member definitions in cpp_class with name cpp_method. + + Args: + xml_folder (str): The folder containing the Doxygen XML documentation. + cpp_class (str): The name of the C++ class that contains the function whose docstring is to be extracted. + cpp_method (str): The name of the C++ method whose docstring is to be extracted. + + Returns: + list: All of the member definitions in cpp_class with name cpp_method. + """ + xml_folder_path = Path(xml_folder) + + # Create the path to the Doxygen XML index file. + xml_index_file = xml_folder_path / "index.xml" + + # Parse the index file + index_tree = self.parse_xml(xml_index_file) + if not index_tree: + self.print_if_verbose(f"Index file {xml_index_file} was empty.") + return "" + + index_root = index_tree.getroot() + + # Find the compound with name == cpp_class + class_index = index_root.find(f"./*[name='{cpp_class}']") + + if class_index is None: + self.print_if_verbose( + f"Could not extract docs for {cpp_class}.{cpp_method}; class not found in index file." + ) + return "" + + # Create the path to the file with the documentation for cpp_class. + xml_class_file = xml_folder_path / class_index.attrib['refid'] + '.xml' + + # Parse the class file + class_tree = self.parse_xml(xml_class_file) + if not class_tree: + self.print_if_verbose(f"Class file {xml_class_file} was empty.") + return "" + + class_root = class_tree.getroot() + + # Find the member(s) in cpp_class with name == cpp_method + maybe_member_defs = class_root.findall( + f"compounddef/sectiondef//*[name='{cpp_method}']") + + return maybe_member_defs + + def filter_member_defs(self, maybe_member_defs: list, + method_args_names: list): + """ + Remove member definitions which do not match the supplied argument names list. + + Args: + maybe_member_defs (list): The list of all member definitions in the class which share the same name. + method_args_names (list): The list of argument names in the definition of the function whose documentation is desired. + Supplying the argument names allows for the filtering of overloaded functions with the same name but different arguments. + + Returns: + tuple[list, list]: (the filtered member definitions, parameters which should be ignored because they are optional) + """ + member_defs = [] + + # Optional parameters we should ignore if we encounter them in the docstring + ignored_params = [] + + # Filter out the members which don't match the method_args_names + for maybe_member_def in maybe_member_defs: + self.print_if_verbose( + f"Investigating member_def with argstring {maybe_member_def.find('argsstring').text}" + ) + # Find the number of required parameters and the number of total parameters from the + # Doxygen XML for this member_def + params = maybe_member_def.findall("param") + num_tot_params = len(params) + # Calculate required params by subtracting the number of optional params (params where defval is + # set--defval means default value) from the number of total params + num_req_params = num_tot_params - sum([ + 1 if param.find("defval") is not None else 0 + for param in params + ]) + + # If the number of parameters in method_args_names matches neither number, eliminate this member_def + # This is done because wrap generates a python wrapper function twice for every function with + # optional parameters: one with none of the optional parameters, and one with all of the optional + # parameters, required. + if len(method_args_names) != num_req_params and len( + method_args_names) != num_tot_params: + self.print_if_verbose( + f"Wrong number of parameters: got {len(method_args_names)}, expected required {num_req_params} or total {num_tot_params}." + ) + continue + + # If the parameter names don't match, eliminate this member_def + eliminate = False + for i, arg_name in enumerate(method_args_names): + # Try to find the name of the parameter in the XML + param_name = params[i].find( + "declname" + ) # declname is the tag that usually contains the param name + # If we couldn't find the declname, try the defname (used uncommonly) + if param_name is None: + param_name = params[i].find("defname") + if param_name is None: + # Can't find the name for this parameter. This may be an unreachable statement but Doxygen is + # not well-documented enough to rely on a or a always being defined inside a . + eliminate = True + continue + # Eliminate if any param name doesn't match the expected name + if arg_name != param_name.text: + eliminate = True + if eliminate: + self.print_if_verbose("Names didn't match.") + continue + + # At this point, this member_def can be assumed to be the desired function (or is indistinguishable + # from it based on all of the reliable information we have--if this is the case, we need to rely on + # the _memory to give the correct docs for each.) + member_defs.append(maybe_member_def) + self.print_if_verbose("Confirmed as correct function.") + + # Remember which parameters to ignore, if any + for i in range(len(method_args_names), num_tot_params): + ignored_params.append(params[i].find("declname").text) + + return member_defs, ignored_params + + def determine_documenting_index(self, cpp_class: str, cpp_method: str, + method_args_names: list, + member_defs: list): + """ + Determine which member definition to retrieve documentation from, if there are multiple. + + Args: + cpp_class (str): The name of the C++ class that contains the function whose docstring is to be extracted. + cpp_method (str): The name of the C++ method whose docstring is to be extracted. + method_args_names (list): A list of the names of the cpp_method's parameters. + member_defs (list): All of the member definitions of cpp_class which match cpp_method in name + and whose arguments have the same names as method_args_names. + + Returns: + int: The index indicating which member definition to document. + """ + # If there are multiple member defs that match the method args names, + # remember how many we've encountered already so that we can return + # the docs for the first one we haven't yet extracted. + # This is only relevant if there are overloaded functions where the + # parameter types are different but the parameter names are the same, + # e.g. foo(int bar) and foo(string bar). The parameter types cannot be + # relied on because they cannot be assumed to be the same between GTSAM + # implementation and pybind11 generated wrapper, e.g. OptionalJacobian + # in GTSAM becomes Eigen::Matrix in the pybind11 code. + documenting_index = 0 + if len(member_defs) > 1: + function_key = f"{cpp_class}.{cpp_method}({','.join(method_args_names) if method_args_names else ''})" + if function_key in self._memory: + self._memory[function_key] += 1 + documenting_index = self._memory[function_key] + else: + self._memory[function_key] = 0 + + return documenting_index + + def get_formatted_docstring(self, + member_def: 'xml.etree.ElementTree.Element', + ignored_params: list): + """Gets the formatted docstring for the supplied XML element representing a member definition. + + Args: + member_def (xml.etree.ElementTree.Element): The member definition to document. + ignored_params (list): The optional parameters which should be ignored, if any. + + Returns: + str: The formatted docstring. + """ + docstring = "" + + brief_description = member_def.find(".//briefdescription") + detailed_description = member_def.find(".//detaileddescription") + + # Add the brief description first, if it exists. + if brief_description is not None: + for para in brief_description.findall("para"): + docstring += "".join(t for t in para.itertext() if t.strip()) + + # Add the detailed description. This includes the parameter list and the return value. + if detailed_description is not None: + docstring += "\n" + # Add non-parameter detailed description + for element in list(detailed_description): + if element.tag == "para" and "parameterlist" not in [ + e.tag for e in element + ]: + docstring += "".join( + t for t in element.itertext() if t.strip()) + " " + + # Add parameter docs + parameter_list = detailed_description.find(".//parameterlist") + if parameter_list is not None: + for i, parameter_item in enumerate( + parameter_list.findall(".//parameteritem")): + name = parameter_item.find(".//parametername").text + desc = parameter_item.find( + ".//parameterdescription/para").text + if name not in ignored_params: + docstring += f"{name.strip() if name else f'[Parameter {i}]'}: {desc.strip() if desc else 'No description provided'}\n" + + # Add return value docs + return_sect = detailed_description.find(".//simplesect") + if return_sect is not None and return_sect.attrib[ + "kind"] == "return" and return_sect.find( + "para").text is not None: + docstring += f"Returns: {return_sect.find('para').text.strip()}" + + return docstring.strip() + + def print_if_verbose(self, text: str): + """ + Print text if the parser is in verbose mode. + """ + if self._verbose: + print(text) + + +if __name__ == "__main__": + if len(sys.argv) != 5: + print( + "Usage: python xml_parser.py " + ) + else: + parser = XMLDocParser() + parser._verbose = True + xml_file = sys.argv[1] + extracted_doc = parser.extract_docstring(xml_file, sys.argv[2], + sys.argv[3], + sys.argv[4].split(",")) + + print() + print(extracted_doc.strip()) diff --git a/wrap/pybind11/.clang-tidy b/wrap/pybind11/.clang-tidy index 23018386c..96cb6f582 100644 --- a/wrap/pybind11/.clang-tidy +++ b/wrap/pybind11/.clang-tidy @@ -57,10 +57,12 @@ Checks: | readability-string-compare, readability-suspicious-call-argument, readability-uniqueptr-delete-release, + -bugprone-chained-comparison, -bugprone-easily-swappable-parameters, -bugprone-exception-escape, -bugprone-reserved-identifier, -bugprone-unused-raii, + -performance-enum-size, CheckOptions: - key: modernize-use-equals-default.IgnoreMacros diff --git a/wrap/pybind11/.codespell-ignore-lines b/wrap/pybind11/.codespell-ignore-lines index 2a01d63eb..e8cbf3144 100644 --- a/wrap/pybind11/.codespell-ignore-lines +++ b/wrap/pybind11/.codespell-ignore-lines @@ -12,6 +12,17 @@ template template class_ &def(const detail::op_ &op, const Extra &...extra) { class_ &def_cast(const detail::op_ &op, const Extra &...extra) { + int valu; + explicit movable_int(int v) : valu{v} {} + movable_int(movable_int &&other) noexcept : valu(other.valu) { other.valu = 91; } + explicit indestructible_int(int v) : valu{v} {} + REQUIRE(hld.as_raw_ptr_unowned()->valu == 19); + REQUIRE(othr.valu == 19); + REQUIRE(orig.valu == 91); + (m.pass_valu, "Valu", "pass_valu:Valu(_MvCtor)*_CpCtor"), +atyp_valu rtrn_valu() { atyp_valu obj{"Valu"}; return obj; } + assert m.atyp_valu().get_mtxt() == "Valu" +// valu(e), ref(erence), ptr or p (pointer), r = rvalue, m = mutable, c = const, @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) struct IntStruct { explicit IntStruct(int v) : value(v){}; diff --git a/wrap/pybind11/.github/CONTRIBUTING.md b/wrap/pybind11/.github/CONTRIBUTING.md index f5a08e2d7..f2d8007df 100644 --- a/wrap/pybind11/.github/CONTRIBUTING.md +++ b/wrap/pybind11/.github/CONTRIBUTING.md @@ -81,7 +81,7 @@ nox -s build ### Full setup To setup an ideal development environment, run the following commands on a -system with CMake 3.14+: +system with CMake 3.15+: ```bash python3 -m venv venv @@ -96,8 +96,8 @@ Tips: * You can use `virtualenv` (faster, from PyPI) instead of `venv`. * You can select any name for your environment folder; if it contains "env" it will be ignored by git. -* If you don't have CMake 3.14+, just add "cmake" to the pip install command. -* You can use `-DPYBIND11_FINDPYTHON=ON` to use FindPython on CMake 3.12+ +* If you don't have CMake 3.15+, just add "cmake" to the pip install command. +* You can use `-DPYBIND11_FINDPYTHON=ON` to use FindPython. * In classic mode, you may need to set `-DPYTHON_EXECUTABLE=/path/to/python`. FindPython uses `-DPython_ROOT_DIR=/path/to` or `-DPython_EXECUTABLE=/path/to/python`. @@ -149,8 +149,8 @@ To run the tests, you can "build" the check target: cmake --build build --target check ``` -`--target` can be spelled `-t` in CMake 3.15+. You can also run individual -tests with these targets: +`--target` can be spelled `-t`. You can also run individual tests with these +targets: * `pytest`: Python tests only, using the [pytest](https://docs.pytest.org/en/stable/) framework diff --git a/wrap/pybind11/.github/workflows/ci.yml b/wrap/pybind11/.github/workflows/ci.yml index 3054d842a..80f1f0d74 100644 --- a/wrap/pybind11/.github/workflows/ci.yml +++ b/wrap/pybind11/.github/workflows/ci.yml @@ -39,6 +39,8 @@ jobs: - 'pypy-3.8' - 'pypy-3.9' - 'pypy-3.10' + - 'pypy-3.11' + - 'graalpy-24.1' # Items in here will either be added to the build matrix (if not # present), or add new keys to an existing matrix element if all the @@ -64,9 +66,45 @@ jobs: # Inject a couple Windows 2019 runs - runs-on: windows-2019 python: '3.9' + # Inject a few runs with different runtime libraries + - runs-on: windows-2022 + python: '3.9' + args: > + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded + - runs-on: windows-2022 + python: '3.10' + args: > + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL + # This needs a python built with MTd + # - runs-on: windows-2022 + # python: '3.11' + # args: > + # -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug + - runs-on: windows-2022 + python: '3.12' + args: > + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebugDLL # Extra ubuntu latest job - runs-on: ubuntu-latest python: '3.11' + # Run tests with py::smart_holder as the default holder + # with recent (or ideally latest) released Python version. + - runs-on: ubuntu-latest + python: '3.12' + args: > + -DCMAKE_CXX_FLAGS="-DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE" + - runs-on: macos-13 + python: '3.12' + args: > + -DCMAKE_CXX_FLAGS="-DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE" + - runs-on: windows-2022 + python: '3.12' + args: > + -DCMAKE_CXX_FLAGS="/DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE /GR /EHsc" + exclude: + # The setup-python action currently doesn't have graalpy for windows + - python: 'graalpy-24.1' + runs-on: 'windows-2022' name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}" @@ -122,6 +160,7 @@ jobs: -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DPYBIND11_NUMPY_1_ONLY=ON + -DPYBIND11_PYTEST_ARGS=-v -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 @@ -151,6 +190,7 @@ jobs: -DPYBIND11_WERROR=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DPYBIND11_NUMPY_1_ONLY=ON + -DPYBIND11_PYTEST_ARGS=-v -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 @@ -170,6 +210,7 @@ jobs: run: > cmake -S . -B build3 -DPYBIND11_WERROR=ON + -DPYBIND11_PYTEST_ARGS=-v -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 @@ -243,7 +284,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v3.1.0 + uses: deadsnakes/action@v3.2.0 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} @@ -310,22 +351,11 @@ jobs: strategy: fail-fast: false matrix: - clang: - - 3.6 - - 3.7 - - 3.9 - - 7 - - 9 - - dev - std: - - 11 container_suffix: - "" include: - clang: 5 std: 14 - - clang: 10 - std: 17 - clang: 11 std: 20 - clang: 12 @@ -340,6 +370,12 @@ jobs: - clang: 16 std: 20 container_suffix: "-bullseye" + - clang: 17 + std: 20 + container_suffix: "-bookworm" + - clang: 18 + std: 20 + container_suffix: "-bookworm" name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64" container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" @@ -497,11 +533,9 @@ jobs: fail-fast: false matrix: include: - - { gcc: 7, std: 11 } - - { gcc: 7, std: 17 } - - { gcc: 8, std: 14 } - - { gcc: 8, std: 17 } + - { gcc: 9, std: 20 } - { gcc: 10, std: 17 } + - { gcc: 10, std: 20 } - { gcc: 11, std: 20 } - { gcc: 12, std: 20 } - { gcc: 13, std: 20 } @@ -719,9 +753,9 @@ jobs: # This tests an "install" with the CMake tools install-classic: - name: "🐍 3.7 • Debian • x86 • Install" + name: "🐍 3.9 • Debian • x86 • Install" runs-on: ubuntu-latest - container: i386/debian:buster + container: i386/debian:bullseye steps: - uses: actions/checkout@v1 # v1 is required to run inside docker @@ -801,7 +835,6 @@ jobs: fail-fast: false matrix: python: - - '3.7' - '3.8' - '3.9' - '3.10' @@ -819,8 +852,6 @@ jobs: args: -DCMAKE_CXX_STANDARD=20 - python: '3.8' args: -DCMAKE_CXX_STANDARD=17 - - python: '3.7' - args: -DCMAKE_CXX_STANDARD=14 name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}" @@ -999,7 +1030,6 @@ jobs: git mingw-w64-${{matrix.env}}-gcc mingw-w64-${{matrix.env}}-python-pip - mingw-w64-${{matrix.env}}-python-numpy mingw-w64-${{matrix.env}}-cmake mingw-w64-${{matrix.env}}-make mingw-w64-${{matrix.env}}-python-pytest @@ -1011,7 +1041,7 @@ jobs: with: msystem: ${{matrix.sys}} install: >- - git + mingw-w64-${{matrix.env}}-python-numpy mingw-w64-${{matrix.env}}-python-scipy mingw-w64-${{matrix.env}}-eigen3 @@ -1109,7 +1139,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v2.0 - name: Install ninja-build tool - uses: seanmiddleditch/gha-setup-ninja@v5 + uses: seanmiddleditch/gha-setup-ninja@v6 - name: Run pip installs run: | diff --git a/wrap/pybind11/.github/workflows/configure.yml b/wrap/pybind11/.github/workflows/configure.yml index 0e55a0795..2031ec823 100644 --- a/wrap/pybind11/.github/workflows/configure.yml +++ b/wrap/pybind11/.github/workflows/configure.yml @@ -31,7 +31,7 @@ jobs: include: - runs-on: ubuntu-20.04 arch: x64 - cmake: "3.5" + cmake: "3.15" - runs-on: ubuntu-20.04 arch: x64 @@ -39,22 +39,22 @@ jobs: - runs-on: macos-13 arch: x64 - cmake: "3.7" + cmake: "3.15" - runs-on: windows-2019 arch: x64 # x86 compilers seem to be missing on 2019 image cmake: "3.18" - name: 🐍 3.7 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} + name: 🐍 3.8 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 - - name: Setup Python 3.7 + - name: Setup Python 3.8 uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.8 architecture: ${{ matrix.arch }} - name: Prepare env diff --git a/wrap/pybind11/.github/workflows/emscripten.yaml b/wrap/pybind11/.github/workflows/emscripten.yaml new file mode 100644 index 000000000..aaaae1a19 --- /dev/null +++ b/wrap/pybind11/.github/workflows/emscripten.yaml @@ -0,0 +1,30 @@ +name: WASM + +on: + workflow_dispatch: + pull_request: + branches: + - master + - stable + - v* + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-wasm-emscripten: + name: Pyodide wheel + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + + - uses: pypa/cibuildwheel@v2.23 + env: + PYODIDE_BUILD_EXPORTS: whole_archive + with: + package-dir: tests + only: cp312-pyodide_wasm32 diff --git a/wrap/pybind11/.github/workflows/format.yml b/wrap/pybind11/.github/workflows/format.yml index 1eaa56e1c..e50dc0bb7 100644 --- a/wrap/pybind11/.github/workflows/format.yml +++ b/wrap/pybind11/.github/workflows/format.yml @@ -41,7 +41,7 @@ jobs: # in .github/CONTRIBUTING.md and update as needed. name: Clang-Tidy runs-on: ubuntu-latest - container: silkeh/clang:15-bullseye + container: silkeh/clang:18-bookworm steps: - uses: actions/checkout@v4 diff --git a/wrap/pybind11/.github/workflows/pip.yml b/wrap/pybind11/.github/workflows/pip.yml index a054ce695..497870c99 100644 --- a/wrap/pybind11/.github/workflows/pip.yml +++ b/wrap/pybind11/.github/workflows/pip.yml @@ -91,18 +91,19 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' needs: [packaging] - environment: pypi + environment: + name: pypi + url: https://pypi.org/p/pybind11 permissions: id-token: write attestations: write - contents: read steps: # Downloads all to directories matching the artifact names - uses: actions/download-artifact@v4 - name: Generate artifact attestation for sdist and wheel - uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2 + uses: actions/attest-build-provenance@bd77c077858b8d561b7a36cbe48ef4cc642ca39d # v2.2.2 with: subject-path: "*/pybind11*" @@ -110,8 +111,10 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: standard/ + attestations: true - name: Publish global package uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: global/ + attestations: true diff --git a/wrap/pybind11/.pre-commit-config.yaml b/wrap/pybind11/.pre-commit-config.yaml index 3cec1ebe0..e5002e8fd 100644 --- a/wrap/pybind11/.pre-commit-config.yaml +++ b/wrap/pybind11/.pre-commit-config.yaml @@ -25,14 +25,14 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v18.1.5" + rev: "v19.1.7" hooks: - id: clang-format types_or: [c++, c, cuda] # Ruff, the Python auto-correcting linter/formatter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.9.9 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -40,7 +40,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.10.0" + rev: "v1.15.0" hooks: - id: mypy args: [] @@ -62,7 +62,7 @@ repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v4.6.0" + rev: "v5.0.0" hooks: - id: check-added-large-files - id: check-case-conflict @@ -76,10 +76,11 @@ repos: - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace + exclude: \.patch?$ # Also code format the docs - repo: https://github.com/adamchainz/blacken-docs - rev: "1.16.0" + rev: "1.19.1" hooks: - id: blacken-docs additional_dependencies: @@ -90,10 +91,11 @@ repos: rev: "v1.5.5" hooks: - id: remove-tabs + exclude: (^docs/.*|\.patch)?$ # Avoid directional quotes - repo: https://github.com/sirosen/texthooks - rev: "0.6.6" + rev: "0.6.8" hooks: - id: fix-ligatures - id: fix-smartquotes @@ -108,7 +110,7 @@ repos: # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.49" + rev: "0.50" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed @@ -119,7 +121,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.3.0" + rev: "v2.4.1" hooks: - id: codespell exclude: ".supp$" @@ -142,14 +144,14 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v3.2.2" + rev: "v3.3.4" hooks: - id: pylint files: ^pybind11 # Check schemas on some of our YAML files - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.4 + rev: 0.31.2 hooks: - id: check-readthedocs - id: check-github-workflows diff --git a/wrap/pybind11/CMakeLists.txt b/wrap/pybind11/CMakeLists.txt index 3526a1a66..8783dba6e 100644 --- a/wrap/pybind11/CMakeLists.txt +++ b/wrap/pybind11/CMakeLists.txt @@ -10,16 +10,7 @@ if(NOT CMAKE_VERSION VERSION_LESS "3.27") cmake_policy(GET CMP0148 _pybind11_cmp0148) endif() -cmake_minimum_required(VERSION 3.5) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) if(_pybind11_cmp0148) cmake_policy(SET CMP0148 ${_pybind11_cmp0148}) @@ -27,9 +18,7 @@ if(_pybind11_cmp0148) endif() # Avoid infinite recursion if tests include this as a subdirectory -if(DEFINED PYBIND11_MASTER_PROJECT) - return() -endif() +include_guard(GLOBAL) # Extract project version from source file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/pybind11/detail/common.h" @@ -74,14 +63,6 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) set(PYBIND11_MASTER_PROJECT ON) - if(OSX AND CMAKE_VERSION VERSION_LESS 3.7) - # Bug in macOS CMake < 3.7 is unable to download catch - message(WARNING "CMAKE 3.7+ needed on macOS to download catch, and newer HIGHLY recommended") - elseif(WINDOWS AND CMAKE_VERSION VERSION_LESS 3.8) - # Only tested with 3.8+ in CI. - message(WARNING "CMAKE 3.8+ tested on Windows, previous versions untested") - endif() - message(STATUS "CMake ${CMAKE_VERSION}") if(CMAKE_CXX_STANDARD) @@ -133,8 +114,7 @@ cmake_dependent_option( "Install pybind11 headers in Python include directory instead of default installation prefix" OFF "PYBIND11_INSTALL" OFF) -cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" ${_pybind11_findpython_default} - "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) +option(PYBIND11_FINDPYTHON "Force new FindPython" ${_pybind11_findpython_default}) # Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests # (makes transition easier while we support both modes). @@ -149,17 +129,26 @@ endif() set(PYBIND11_HEADERS include/pybind11/detail/class.h include/pybind11/detail/common.h + include/pybind11/detail/cpp_conduit.h include/pybind11/detail/descr.h + include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h include/pybind11/detail/init.h include/pybind11/detail/internals.h + include/pybind11/detail/struct_smart_holder.h include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h + include/pybind11/detail/using_smart_holder.h + include/pybind11/detail/value_and_holder.h + include/pybind11/detail/exception_translation.h include/pybind11/attr.h include/pybind11/buffer_info.h include/pybind11/cast.h include/pybind11/chrono.h include/pybind11/common.h include/pybind11/complex.h + include/pybind11/conduit/pybind11_conduit_v1.h + include/pybind11/conduit/pybind11_platform_abi_id.h + include/pybind11/conduit/wrap_include_python_h.h include/pybind11/options.h include/pybind11/eigen.h include/pybind11/eigen/common.h @@ -178,11 +167,13 @@ set(PYBIND11_HEADERS include/pybind11/stl.h include/pybind11/stl_bind.h include/pybind11/stl/filesystem.h + include/pybind11/trampoline_self_life_support.h include/pybind11/type_caster_pyobject_ptr.h - include/pybind11/typing.h) + include/pybind11/typing.h + include/pybind11/warnings.h) # Compare with grep and warn if mismatched -if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) +if(PYBIND11_MASTER_PROJECT) file( GLOB_RECURSE _pybind11_header_check LIST_DIRECTORIES false @@ -200,10 +191,7 @@ if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) endif() endif() -# CMake 3.12 added list(TRANSFORM PREPEND -# But we can't use it yet -string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" PYBIND11_HEADERS - "${PYBIND11_HEADERS}") +list(TRANSFORM PYBIND11_HEADERS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") # Cache variable so this can be used in parent projects set(pybind11_INCLUDE_DIR @@ -273,25 +261,11 @@ if(PYBIND11_INSTALL) tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) - if(CMAKE_VERSION VERSION_LESS 3.14) - # Remove CMAKE_SIZEOF_VOID_P from ConfigVersion.cmake since the library does - # not depend on architecture specific settings or libraries. - set(_PYBIND11_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) - unset(CMAKE_SIZEOF_VOID_P) - - write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - VERSION ${PROJECT_VERSION} - COMPATIBILITY AnyNewerVersion) - - set(CMAKE_SIZEOF_VOID_P ${_PYBIND11_CMAKE_SIZEOF_VOID_P}) - else() - # CMake 3.14+ natively supports header-only libraries - write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - VERSION ${PROJECT_VERSION} - COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT) - endif() + # CMake natively supports header-only libraries + write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake diff --git a/wrap/pybind11/README.rst b/wrap/pybind11/README.rst index 0d1e1d291..eaea399ab 100644 --- a/wrap/pybind11/README.rst +++ b/wrap/pybind11/README.rst @@ -1,7 +1,7 @@ .. figure:: https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png :alt: pybind11 logo -**pybind11 — Seamless operability between C++11 and Python** +**pybind11 (v3) — Seamless interoperability between C++ and Python** |Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| |CI| |Build status| @@ -34,7 +34,7 @@ dependency. Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. Without comments, the core header files only require ~4K -lines of code and depend on Python (3.7+, or PyPy) and the C++ +lines of code and depend on Python (3.8+, or PyPy) and the C++ standard library. This compact implementation was possible thanks to some C++11 language features (specifically: tuples, lambda functions and variadic templates). Since its creation, this library has grown beyond @@ -79,7 +79,7 @@ Goodies In addition to the core functionality, pybind11 provides some extra goodies: -- Python 3.7+, and PyPy3 7.3 are supported with an implementation-agnostic +- Python 3.8+, and PyPy3 7.3 are supported with an implementation-agnostic interface (pybind11 2.9 was the last version to support Python 2 and 3.5). - It is possible to bind C++11 lambda functions with captured @@ -134,11 +134,34 @@ About This project was created by `Wenzel Jakob `_. Significant features and/or -improvements to the code were contributed by Jonas Adler, Lori A. Burns, -Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel -Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov, Johan Mabille, Tomasz Miąsko, -Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim -Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. +improvements to the code were contributed by +Jonas Adler, +Lori A. Burns, +Sylvain Corlay, +Eric Cousineau, +Aaron Gokaslan, +Ralf Grosse-Kunstleve, +Trent Houliston, +Axel Huebl, +@hulucc, +Yannick Jadoul, +Sergey Lyskov, +Johan Mabille, +Tomasz Miąsko, +Dean Moldovan, +Ben Pritchard, +Jason Rhinelander, +Boris Schäling, +Pim Schellart, +Henry Schreiner, +Ivan Smirnov, +Dustin Spicuzza, +Boris Staletic, +Ethan Steinberg, +Patrick Stewart, +Ivor Wanders, +and +Xiaofei Wang. We thank Google for a generous financial contribution to the continuous integration infrastructure used by this project. diff --git a/wrap/pybind11/docs/advanced/cast/custom.rst b/wrap/pybind11/docs/advanced/cast/custom.rst index 8138cac61..5a626f3ba 100644 --- a/wrap/pybind11/docs/advanced/cast/custom.rst +++ b/wrap/pybind11/docs/advanced/cast/custom.rst @@ -1,35 +1,53 @@ Custom type casters =================== -In very rare cases, applications may require custom type casters that cannot be -expressed using the abstractions provided by pybind11, thus requiring raw -Python C API calls. This is fairly advanced usage and should only be pursued by -experts who are familiar with the intricacies of Python reference counting. +Some applications may prefer custom type casters that convert between existing +Python types and C++ types, similar to the ``list`` ↔ ``std::vector`` +and ``dict`` ↔ ``std::map`` conversions which are built into pybind11. +Implementing custom type casters is fairly advanced usage. +While it is recommended to use the pybind11 API as much as possible, more complex examples may +require familiarity with the intricacies of the Python C API. +You can refer to the `Python/C API Reference Manual `_ +for more information. -The following snippets demonstrate how this works for a very simple ``inty`` -type that that should be convertible from Python types that provide a -``__int__(self)`` method. +The following snippets demonstrate how this works for a very simple ``Point2D`` type. +We want this type to be convertible to C++ from Python types implementing the +``Sequence`` protocol and having two elements of type ``float``. +When returned from C++ to Python, it should be converted to a Python ``tuple[float, float]``. +For this type we could provide Python bindings for different arithmetic functions implemented +in C++ (here demonstrated by a simple ``negate`` function). + +.. + PLEASE KEEP THE CODE BLOCKS IN SYNC WITH + tests/test_docs_advanced_cast_custom.cpp + tests/test_docs_advanced_cast_custom.py + Ideally, change the test, run pre-commit (incl. clang-format), + then copy the changed code back here. + Also use TEST_SUBMODULE in tests, but PYBIND11_MODULE in docs. .. code-block:: cpp - struct inty { long long_value; }; + namespace user_space { - void print(inty s) { - std::cout << s.long_value << std::endl; - } + struct Point2D { + double x; + double y; + }; -The following Python snippet demonstrates the intended usage from the Python side: + Point2D negate(const Point2D &point) { return Point2D{-point.x, -point.y}; } + + } // namespace user_space + + +The following Python snippet demonstrates the intended usage of ``negate`` from the Python side: .. code-block:: python - class A: - def __int__(self): - return 123 + from my_math_module import docs_advanced_cast_custom as m - - from example import print - - print(A()) + point1 = [1.0, -1.0] + point2 = m.negate(point1) + assert point2 == (-1.0, 1.0) To register the necessary conversion routines, it is necessary to add an instantiation of the ``pybind11::detail::type_caster`` template. @@ -38,47 +56,57 @@ type is explicitly allowed. .. code-block:: cpp - namespace PYBIND11_NAMESPACE { namespace detail { - template <> struct type_caster { - public: - /** - * This macro establishes the name 'inty' in - * function signatures and declares a local variable - * 'value' of type inty - */ - PYBIND11_TYPE_CASTER(inty, const_name("inty")); + namespace pybind11 { + namespace detail { - /** - * Conversion part 1 (Python->C++): convert a PyObject into a inty - * instance or return false upon failure. The second argument - * indicates whether implicit conversions should be applied. - */ - bool load(handle src, bool) { - /* Extract PyObject from handle */ - PyObject *source = src.ptr(); - /* Try converting into a Python integer value */ - PyObject *tmp = PyNumber_Long(source); - if (!tmp) + template <> + struct type_caster { + // This macro inserts a lot of boilerplate code and sets the type hint. + // `io_name` is used to specify different type hints for arguments and return values. + // The signature of our negate function would then look like: + // `negate(Sequence[float]) -> tuple[float, float]` + PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]")); + + // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments + // are used to indicate the return value policy and parent object (for + // return_value_policy::reference_internal) and are often ignored by custom casters. + // The return value should reflect the type hint specified by the second argument of `io_name`. + static handle + cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) { + return py::make_tuple(number.x, number.y).release(); + } + + // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The + // second argument indicates whether implicit conversions should be allowed. + // The accepted types should reflect the type hint specified by the first argument of + // `io_name`. + bool load(handle src, bool /*convert*/) { + // Check if handle is a Sequence + if (!py::isinstance(src)) { + return false; + } + auto seq = py::reinterpret_borrow(src); + // Check if exactly two values are in the Sequence + if (seq.size() != 2) { + return false; + } + // Check if each element is either a float or an int + for (auto item : seq) { + if (!py::isinstance(item) && !py::isinstance(item)) { return false; - /* Now try to convert into a C++ int */ - value.long_value = PyLong_AsLong(tmp); - Py_DECREF(tmp); - /* Ensure return code was OK (to avoid out-of-range errors etc) */ - return !(value.long_value == -1 && !PyErr_Occurred()); + } } + value.x = seq[0].cast(); + value.y = seq[1].cast(); + return true; + } + }; - /** - * Conversion part 2 (C++ -> Python): convert an inty instance into - * a Python object. The second and third arguments are used to - * indicate the return value policy and parent object (for - * ``return_value_policy::reference_internal``) and are generally - * ignored by implicit casters. - */ - static handle cast(inty src, return_value_policy /* policy */, handle /* parent */) { - return PyLong_FromLong(src.long_value); - } - }; - }} // namespace PYBIND11_NAMESPACE::detail + } // namespace detail + } // namespace pybind11 + + // Bind the negate function + PYBIND11_MODULE(docs_advanced_cast_custom, m) { m.def("negate", user_space::negate); } .. note:: @@ -86,8 +114,22 @@ type is explicitly allowed. that ``T`` is default-constructible (``value`` is first default constructed and then ``load()`` assigns to it). +.. note:: + For further information on the ``return_value_policy`` argument of ``cast`` refer to :ref:`return_value_policies`. + To learn about the ``convert`` argument of ``load`` see :ref:`nonconverting_arguments`. + .. warning:: When using custom type casters, it's important to declare them consistently - in every compilation unit of the Python extension module. Otherwise, + in every compilation unit of the Python extension module to satisfy the C++ One Definition Rule + (`ODR `_). Otherwise, undefined behavior can ensue. + +.. note:: + + Using the type hint ``Sequence[float]`` signals to static type checkers, that not only tuples may be + passed, but any type implementing the Sequence protocol, e.g., ``list[float]``. + Unfortunately, that loses the length information ``tuple[float, float]`` provides. + One way of still providing some length information in type hints is using ``typing.Annotated``, e.g., + ``Annotated[Sequence[float], 2]``, or further add libraries like + `annotated-types `_. diff --git a/wrap/pybind11/docs/advanced/cast/eigen.rst b/wrap/pybind11/docs/advanced/cast/eigen.rst index a5c11a3f1..894ce97f3 100644 --- a/wrap/pybind11/docs/advanced/cast/eigen.rst +++ b/wrap/pybind11/docs/advanced/cast/eigen.rst @@ -259,7 +259,7 @@ copying to take place: "small"_a // <- This one can be copied if needed ); -With the above binding code, attempting to call the the ``some_method(m)`` +With the above binding code, attempting to call the ``some_method(m)`` method on a ``MyClass`` object, or attempting to call ``some_function(m, m2)`` will raise a ``RuntimeError`` rather than making a temporary copy of the array. It will, however, allow the ``m2`` argument to be copied into a temporary if diff --git a/wrap/pybind11/docs/advanced/cast/overview.rst b/wrap/pybind11/docs/advanced/cast/overview.rst index 011bd4c7a..d5a34ef94 100644 --- a/wrap/pybind11/docs/advanced/cast/overview.rst +++ b/wrap/pybind11/docs/advanced/cast/overview.rst @@ -151,7 +151,7 @@ as arguments and return values, refer to the section on binding :ref:`classes`. +------------------------------------+---------------------------+-----------------------------------+ | ``std::variant<...>`` | Type-safe union (C++17) | :file:`pybind11/stl.h` | +------------------------------------+---------------------------+-----------------------------------+ -| ``std::filesystem::path`` | STL path (C++17) [#]_ | :file:`pybind11/stl/filesystem.h` | +| ``std::filesystem::path`` | STL path (C++17) [#]_ | :file:`pybind11/stl/filesystem.h` | +------------------------------------+---------------------------+-----------------------------------+ | ``std::function<...>`` | STL polymorphic function | :file:`pybind11/functional.h` | +------------------------------------+---------------------------+-----------------------------------+ @@ -167,4 +167,4 @@ as arguments and return values, refer to the section on binding :ref:`classes`. +------------------------------------+---------------------------+-----------------------------------+ .. [#] ``std::filesystem::path`` is converted to ``pathlib.Path`` and - ``os.PathLike`` is converted to ``std::filesystem::path``. + can be loaded from ``os.PathLike``, ``str``, and ``bytes``. diff --git a/wrap/pybind11/docs/advanced/cast/stl.rst b/wrap/pybind11/docs/advanced/cast/stl.rst index 03d49b295..1e17bc389 100644 --- a/wrap/pybind11/docs/advanced/cast/stl.rst +++ b/wrap/pybind11/docs/advanced/cast/stl.rst @@ -162,15 +162,15 @@ the declaration .. code-block:: cpp - PYBIND11_MAKE_OPAQUE(std::vector); + PYBIND11_MAKE_OPAQUE(std::vector) before any binding code (e.g. invocations to ``class_::def()``, etc.). This macro must be specified at the top level (and outside of any namespaces), since it adds a template instantiation of ``type_caster``. If your binding code consists of multiple compilation units, it must be present in every file (typically via a common header) preceding any usage of ``std::vector``. Opaque types must -also have a corresponding ``class_`` declaration to associate them with a name -in Python, and to define a set of available operations, e.g.: +also have a corresponding ``py::class_`` declaration to associate them with a +name in Python, and to define a set of available operations, e.g.: .. code-block:: cpp @@ -207,8 +207,8 @@ The following example showcases usage of :file:`pybind11/stl_bind.h`: // Don't forget this #include - PYBIND11_MAKE_OPAQUE(std::vector); - PYBIND11_MAKE_OPAQUE(std::map); + PYBIND11_MAKE_OPAQUE(std::vector) + PYBIND11_MAKE_OPAQUE(std::map) // ... diff --git a/wrap/pybind11/docs/advanced/classes.rst b/wrap/pybind11/docs/advanced/classes.rst index 01a490b72..0ada0cb60 100644 --- a/wrap/pybind11/docs/advanced/classes.rst +++ b/wrap/pybind11/docs/advanced/classes.rst @@ -64,7 +64,7 @@ helper class that is defined as follows: .. code-block:: cpp - class PyAnimal : public Animal { + class PyAnimal : public Animal, py::trampoline_self_life_support { public: /* Inherit the constructors */ using Animal::Animal; @@ -80,6 +80,18 @@ helper class that is defined as follows: } }; +The ``py::trampoline_self_life_support`` base class is needed to ensure +that a ``std::unique_ptr`` can safely be passed between Python and C++. To +steer clear of notorious pitfalls (e.g. inheritance slicing), it is best +practice to always use the base class, in combination with +``py::smart_holder``. + +.. note:: + For completeness, the base class has no effect if a holder other than + ``py::smart_holder`` used, including the default ``std::unique_ptr``. + Please think twice, though, the pitfalls are very real, and the overhead + for using the safer ``py::smart_holder`` is very likely to be in the noise. + The macro :c:macro:`PYBIND11_OVERRIDE_PURE` should be used for pure virtual functions, and :c:macro:`PYBIND11_OVERRIDE` should be used for functions which have a default implementation. There are also two alternate macros @@ -95,18 +107,18 @@ The binding code also needs a few minor adaptations (highlighted): :emphasize-lines: 2,3 PYBIND11_MODULE(example, m) { - py::class_(m, "Animal") + py::class_(m, "Animal") .def(py::init<>()) .def("go", &Animal::go); - py::class_(m, "Dog") + py::class_(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); } Importantly, pybind11 is made aware of the trampoline helper class by -specifying it as an extra template argument to :class:`class_`. (This can also +specifying it as an extra template argument to ``py::class_``. (This can also be combined with other template arguments such as a custom holder type; the order of template types does not matter). Following this, we are able to define a constructor as usual. @@ -116,9 +128,9 @@ Bindings should be made against the actual class, not the trampoline helper clas .. code-block:: cpp :emphasize-lines: 3 - py::class_(m, "Animal"); + py::class_(m, "Animal"); .def(py::init<>()) - .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ + .def("go", &Animal::go); /* <--- DO NOT USE &PyAnimal::go HERE */ Note, however, that the above is sufficient for allowing python classes to extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the @@ -244,13 +256,13 @@ override the ``name()`` method): .. code-block:: cpp - class PyAnimal : public Animal { + class PyAnimal : public Animal, py::trampoline_self_life_support { public: using Animal::Animal; // Inherit constructors std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Animal, go, n_times); } std::string name() override { PYBIND11_OVERRIDE(std::string, Animal, name, ); } }; - class PyDog : public Dog { + class PyDog : public Dog, py::trampoline_self_life_support { public: using Dog::Dog; // Inherit constructors std::string go(int n_times) override { PYBIND11_OVERRIDE(std::string, Dog, go, n_times); } @@ -272,7 +284,7 @@ declare or override any virtual methods itself: .. code-block:: cpp class Husky : public Dog {}; - class PyHusky : public Husky { + class PyHusky : public Husky, py::trampoline_self_life_support { public: using Husky::Husky; // Inherit constructors std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Husky, go, n_times); } @@ -287,13 +299,15 @@ follows: .. code-block:: cpp - template class PyAnimal : public AnimalBase { + template + class PyAnimal : public AnimalBase, py::trampoline_self_life_support { public: using AnimalBase::AnimalBase; // Inherit constructors std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, AnimalBase, go, n_times); } std::string name() override { PYBIND11_OVERRIDE(std::string, AnimalBase, name, ); } }; - template class PyDog : public PyAnimal { + template + class PyDog : public PyAnimal, py::trampoline_self_life_support { public: using PyAnimal::PyAnimal; // Inherit constructors // Override PyAnimal's pure virtual go() with a non-pure one: @@ -311,9 +325,9 @@ The classes are then registered with pybind11 using: .. code-block:: cpp - py::class_> animal(m, "Animal"); - py::class_> dog(m, "Dog"); - py::class_> husky(m, "Husky"); + py::class_, py::smart_holder> animal(m, "Animal"); + py::class_, py::smart_holder> dog(m, "Dog"); + py::class_, py::smart_holder> husky(m, "Husky"); // ... add animal, dog, husky definitions Note that ``Husky`` did not require a dedicated trampoline template class at @@ -499,12 +513,12 @@ an alias: // ... virtual ~Example() = default; }; - class PyExample : public Example { + class PyExample : public Example, py::trampoline_self_life_support { public: using Example::Example; PyExample(Example &&base) : Example(std::move(base)) {} }; - py::class_(m, "Example") + py::class_(m, "Example") // Returns an Example pointer. If a PyExample is needed, the Example // instance will be moved via the extra constructor in PyExample, above. .def(py::init([]() { return new Example(); })) @@ -550,9 +564,10 @@ pybind11. The underlying issue is that the ``std::unique_ptr`` holder type that is responsible for managing the lifetime of instances will reference the destructor even if no deallocations ever take place. In order to expose classes with private or protected destructors, it is possible to override the holder -type via a holder type argument to ``class_``. Pybind11 provides a helper class -``py::nodelete`` that disables any destructor invocations. In this case, it is -crucial that instances are deallocated on the C++ side to avoid memory leaks. +type via a holder type argument to ``py::class_``. Pybind11 provides a helper +class ``py::nodelete`` that disables any destructor invocations. In this case, +it is crucial that instances are deallocated on the C++ side to avoid memory +leaks. .. code-block:: cpp @@ -826,8 +841,7 @@ An instance can now be pickled as follows: always use the latest available version. Beware: failure to follow these instructions will cause important pybind11 memory allocation routines to be skipped during unpickling, which will likely lead to memory corruption - and/or segmentation faults. Python defaults to version 3 (Python 3-3.7) and - version 4 for Python 3.8+. + and/or segmentation faults. .. seealso:: @@ -872,7 +886,7 @@ Multiple Inheritance pybind11 can create bindings for types that derive from multiple base types (aka. *multiple inheritance*). To do so, specify all bases in the template -arguments of the ``class_`` declaration: +arguments of the ``py::class_`` declaration: .. code-block:: cpp @@ -947,11 +961,11 @@ because of conflicting definitions on the external type: // dogs.cpp // Binding for external library class: - py::class(m, "Pet") + py::class_(m, "Pet") .def("name", &pets::Pet::name); // Binding for local extension class: - py::class(m, "Dog") + py::class_(m, "Dog") .def(py::init()); .. code-block:: cpp @@ -959,11 +973,11 @@ because of conflicting definitions on the external type: // cats.cpp, in a completely separate project from the above dogs.cpp. // Binding for external library class: - py::class(m, "Pet") + py::class_(m, "Pet") .def("get_name", &pets::Pet::name); // Binding for local extending class: - py::class(m, "Cat") + py::class_(m, "Cat") .def(py::init()); .. code-block:: pycon @@ -981,13 +995,13 @@ the ``py::class_`` constructor: .. code-block:: cpp // Pet binding in dogs.cpp: - py::class(m, "Pet", py::module_local()) + py::class_(m, "Pet", py::module_local()) .def("name", &pets::Pet::name); .. code-block:: cpp // Pet binding in cats.cpp: - py::class(m, "Pet", py::module_local()) + py::class_(m, "Pet", py::module_local()) .def("get_name", &pets::Pet::name); This makes the Python-side ``dogs.Pet`` and ``cats.Pet`` into distinct classes, @@ -1105,7 +1119,7 @@ described trampoline: virtual int foo() const { return 42; } }; - class Trampoline : public A { + class Trampoline : public A, py::trampoline_self_life_support { public: int foo() const override { PYBIND11_OVERRIDE(int, A, foo, ); } }; @@ -1115,7 +1129,7 @@ described trampoline: using A::foo; }; - py::class_(m, "A") // <-- `Trampoline` here + py::class_(m, "A") // <-- `Trampoline` here .def("foo", &Publicist::foo); // <-- `Publicist` here, not `Trampoline`! Binding final classes @@ -1196,7 +1210,7 @@ but once again each instantiation must be explicitly specified: T fn(V v); }; - py::class>(m, "MyClassT") + py::class_>(m, "MyClassT") .def("fn", &MyClass::fn); Custom automatic downcasters diff --git a/wrap/pybind11/docs/advanced/deadlock.md b/wrap/pybind11/docs/advanced/deadlock.md new file mode 100644 index 000000000..f1bab5bdb --- /dev/null +++ b/wrap/pybind11/docs/advanced/deadlock.md @@ -0,0 +1,391 @@ +# Double locking, deadlocking, GIL + +[TOC] + +## Introduction + +### Overview + +In concurrent programming with locks, *deadlocks* can arise when more than one +mutex is locked at the same time, and careful attention has to be paid to lock +ordering to avoid this. Here we will look at a common situation that occurs in +native extensions for CPython written in C++. + +### Deadlocks + +A deadlock can occur when more than one thread attempts to lock more than one +mutex, and two of the threads lock two of the mutexes in different orders. For +example, consider mutexes `mu1` and `mu2`, and threads T1 and T2, executing: + +| | T1 | T2 | +|--- | ------------------- | -------------------| +|1 | `mu1.lock()`{.good} | `mu2.lock()`{.good}| +|2 | `mu2.lock()`{.bad} | `mu1.lock()`{.bad} | +|3 | `/* work */` | `/* work */` | +|4 | `mu2.unlock()` | `mu1.unlock()` | +|5 | `mu1.unlock()` | `mu2.unlock()` | + +Now if T1 manages to lock `mu1` and T2 manages to lock `mu2` (as indicated in +green), then both threads will block while trying to lock the respective other +mutex (as indicated in red), but they are also unable to release the mutex that +they have locked (step 5). + +**The problem** is that it is possible for one thread to attempt to lock `mu1` +and then `mu2`, and for another thread to attempt to lock `mu2` and then `mu1`. +Note that it does not matter if either mutex is unlocked at any intermediate +point; what matters is only the order of any attempt to *lock* the mutexes. For +example, the following, more complex series of operations is just as prone to +deadlock: + +| | T1 | T2 | +|--- | ------------------- | -------------------| +|1 | `mu1.lock()`{.good} | `mu1.lock()`{.good}| +|2 | waiting for T2 | `mu2.lock()`{.good}| +|3 | waiting for T2 | `/* work */` | +|3 | waiting for T2 | `mu1.unlock()` | +|3 | `mu2.lock()`{.bad} | `/* work */` | +|3 | `/* work */` | `mu1.lock()`{.bad} | +|3 | `/* work */` | `/* work */` | +|4 | `mu2.unlock()` | `mu1.unlock()` | +|5 | `mu1.unlock()` | `mu2.unlock()` | + +When the mutexes involved in a locking sequence are known at compile-time, then +avoiding deadlocks is “merely” a matter of arranging the lock +operations carefully so as to only occur in one single, fixed order. However, it +is also possible for mutexes to only be determined at runtime. A typical example +of this is a database where each row has its own mutex. An operation that +modifies two rows in a single transaction (e.g. “transferring an amount +from one account to another”) must lock two row mutexes, but the locking +order cannot be established at compile time. In this case, a dynamic +“deadlock avoidance algorithm” is needed. (In C++, `std::lock` +provides such an algorithm. An algorithm might use a non-blocking `try_lock` +operation on a mutex, which can either succeed or fail to lock the mutex, but +returns without blocking.) + +Conceptually, one could also consider it a deadlock if _the same_ thread +attempts to lock a mutex that it has already locked (e.g. when some locked +operation accidentally recurses into itself): `mu.lock();`{.good} +`mu.lock();`{.bad} However, this is a slightly separate issue: Typical mutexes +are either of _recursive_ or _non-recursive_ kind. A recursive mutex allows +repeated locking and requires balanced unlocking. A non-recursive mutex can be +implemented more efficiently, and/but for efficiency reasons does not actually +guarantee a deadlock on second lock. Instead, the API simply forbids such use, +making it a precondition that the thread not already hold the mutex, with +undefined behaviour on violation. + +### “Once” initialization + +A common programming problem is to have an operation happen precisely once, even +if requested concurrently. While it is clear that we need to track in some +shared state somewhere whether the operation has already happened, it is worth +noting that this state only ever transitions, once, from `false` to `true`. This +is considerably simpler than a general shared state that can change values +arbitrarily. Next, we also need a mechanism for all but one thread to block +until the initialization has completed, which we can provide with a mutex. The +simplest solution just always locks the mutex: + +```c++ +// The "once" mechanism: +constinit absl::Mutex mu(absl::kConstInit); +constinit bool init_done = false; + +// The operation of interest: +void f(); + +void InitOnceNaive() { + absl::MutexLock lock(&mu); + if (!init_done) { + f(); + init_done = true; + } +} +``` + +This works, but the efficiency-minded reader will observe that once the +operation has completed, all future lock contention on the mutex is +unnecessary. This leads to the (in)famous “double-locking” +algorithm, which was historically hard to write correctly. The idea is to check +the boolean *before* locking the mutex, and avoid locking if the operation has +already completed. However, accessing shared state concurrently when at least +one access is a write is prone to causing a data race and needs to be done +according to an appropriate concurrent programming model. In C++ we use atomic +variables: + +```c++ +// The "once" mechanism: +constinit absl::Mutex mu(absl::kConstInit); +constinit std::atomic init_done = false; + +// The operation of interest: +void f(); + +void InitOnceWithFastPath() { + if (!init_done.load(std::memory_order_acquire)) { + absl::MutexLock lock(&mu); + if (!init_done.load(std::memory_order_relaxed)) { + f(); + init_done.store(true, std::memory_order_release); + } + } +} +``` + +Checking the flag now happens without holding the mutex lock, and if the +operation has already completed, we return immediately. After locking the mutex, +we need to check the flag again, since multiple threads can reach this point. + +*Atomic details.* Since the atomic flag variable is accessed concurrently, we +have to think about the memory order of the accesses. There are two separate +cases: The first, outer check outside the mutex lock, and the second, inner +check under the lock. The outer check and the flag update form an +acquire/release pair: *if* the load sees the value `true` (which must have been +written by the store operation), then it also sees everything that happened +before the store, namely the operation `f()`. By contrast, the inner check can +use relaxed memory ordering, since in that case the mutex operations provide the +necessary ordering: if the inner load sees the value `true`, it happened after +the `lock()`, which happened after the `unlock()`, which happened after the +store. + +The C++ standard library, and Abseil, provide a ready-made solution of this +algorithm called `std::call_once`/`absl::call_once`. (The interface is the same, +but the Abseil implementation is possibly better.) + +```c++ +// The "once" mechanism: +constinit absl::once_flag init_flag; + +// The operation of interest: +void f(); + +void InitOnceWithCallOnce() { + absl::call_once(once_flag, f); +} +``` + +Even though conceptually this is performing the same algorithm, this +implementation has some considerable advantages: The `once_flag` type is a small +and trivial, integer-like type and is trivially destructible. Not only does it +take up less space than a mutex, it also generates less code since it does not +have to run a destructor, which would need to be added to the program's global +destructor list. + +The final clou comes with the C++ semantics of a `static` variable declared at +block scope: According to [[stmt.dcl]](https://eel.is/c++draft/stmt.dcl#3): + +> Dynamic initialization of a block variable with static storage duration or +> thread storage duration is performed the first time control passes through its +> declaration; such a variable is considered initialized upon the completion of +> its initialization. [...] If control enters the declaration concurrently while +> the variable is being initialized, the concurrent execution shall wait for +> completion of the initialization. + +This is saying that the initialization of a local, `static` variable precisely +has the “once” semantics that we have been discussing. We can +therefore write the above example as follows: + +```c++ +// The operation of interest: +void f(); + +void InitOnceWithStatic() { + static int unused = (f(), 0); +} +``` + +This approach is by far the simplest and easiest, but the big difference is that +the mutex (or mutex-like object) in this implementation is no longer visible or +in the user’s control. This is perfectly fine if the initializer is +simple, but if the initializer itself attempts to lock any other mutex +(including by initializing another static variable!), then we have no control +over the lock ordering! + +Finally, you may have noticed the `constinit`s around the earlier code. Both +`constinit` and `constexpr` specifiers on a declaration mean that the variable +is *constant-initialized*, which means that no initialization is performed at +runtime (the initial value is already known at compile time). This in turn means +that a static variable guard mutex may not be needed, and static initialization +never blocks. The difference between the two is that a `constexpr`-specified +variable is also `const`, and a variable cannot be `constexpr` if it has a +non-trivial destructor. Such a destructor also means that the guard mutex is +needed after all, since the destructor must be registered to run at exit, +conditionally on initialization having happened. + +## Python, CPython, GIL + +With CPython, a Python program can call into native code. To this end, the +native code registers callback functions with the Python runtime via the CPython +API. In order to ensure that the internal state of the Python runtime remains +consistent, there is a single, shared mutex called the “global interpreter +lock”, or GIL for short. Upon entry of one of the user-provided callback +functions, the GIL is locked (or “held”), so that no other mutations +of the Python runtime state can occur until the native callback returns. + +Many native extensions do not interact with the Python runtime for at least some +part of them, and so it is common for native extensions to _release_ the GIL, do +some work, and then reacquire the GIL before returning. Similarly, when code is +generally not holding the GIL but needs to interact with the runtime briefly, it +will first reacquire the GIL. The GIL is reentrant, and constructions to acquire +and subsequently release the GIL are common, and often don't worry about whether +the GIL is already held. + +If the native code is written in C++ and contains local, `static` variables, +then we are now dealing with at least _two_ mutexes: the static variable guard +mutex, and the GIL from CPython. + +A common problem in such code is an operation with “only once” +semantics that also ends up requiring the GIL to be held at some point. As per +the above description of “once”-style techniques, one might find a +static variable: + +```c++ +// CPython callback, assumes that the GIL is held on entry. +PyObject* InvokeWidget(PyObject* self) { + static PyObject* impl = CreateWidget(); + return PyObject_CallOneArg(impl, self); +} +``` + +This seems reasonable, but bear in mind that there are two mutexes (the "guard +mutex" and "the GIL"), and we must think about the lock order. Otherwise, if the +callback is called from multiple threads, a deadlock may ensue. + +Let us consider what we can see here: On entry, the GIL is already locked, and +we are locking the guard mutex. This is one lock order. Inside the initializer +`CreateWidget`, with both mutexes already locked, the function can freely access +the Python runtime. + +However, it is entirely possible that `CreateWidget` will want to release the +GIL at one point and reacquire it later: + +```c++ +// Assumes that the GIL is held on entry. +// Ensures that the GIL is held on exit. +PyObject* CreateWidget() { + // ... + Py_BEGIN_ALLOW_THREADS // releases GIL + // expensive work, not accessing the Python runtime + Py_END_ALLOW_THREADS // acquires GIL, #! + // ... + return result; +} +``` + +Now we have a second lock order: the guard mutex is locked, and then the GIL is +locked (at `#!`). To see how this deadlocks, consider threads T1 and T2 both +having the runtime attempt to call `InvokeWidget`. T1 locks the GIL and +proceeds, locking the guard mutex and calling `CreateWidget`; T2 is blocked +waiting for the GIL. Then T1 releases the GIL to do “expensive +work”, and T2 awakes and locks the GIL. Now T2 is blocked trying to +acquire the guard mutex, but T1 is blocked reacquiring the GIL (at `#!`). + +In other words: if we want to support “once-called” functions that +can arbitrarily release and reacquire the GIL, as is very common, then the only +lock order that we can ensure is: guard mutex first, GIL second. + +To implement this, we must rewrite our code. Naively, we could always release +the GIL before a `static` variable with blocking initializer: + +```c++ +// CPython callback, assumes that the GIL is held on entry. +PyObject* InvokeWidget(PyObject* self) { + Py_BEGIN_ALLOW_THREADS // releases GIL + static PyObject* impl = CreateWidget(); + Py_END_ALLOW_THREADS // acquires GIL + + return PyObject_CallOneArg(impl, self); +} +``` + +But similar to the `InitOnceNaive` example above, this code cycles the GIL +(possibly descheduling the thread) even when the static variable has already +been initialized. If we want to avoid this, we need to abandon the use of a +static variable, since we do not control the guard mutex well enough. Instead, +we use an operation whose mutex locking is under our control, such as +`call_once`. For example: + +```c++ +// CPython callback, assumes that the GIL is held on entry. +PyObject* InvokeWidget(PyObject* self) { + static constinit PyObject* impl = nullptr; + static constinit std::atomic init_done = false; + static constinit absl::once_flag init_flag; + + if (!init_done.load(std::memory_order_acquire)) { + Py_BEGIN_ALLOW_THREADS // releases GIL + absl::call_once(init_flag, [&]() { + PyGILState_STATE s = PyGILState_Ensure(); // acquires GIL + impl = CreateWidget(); + PyGILState_Release(s); // releases GIL + init_done.store(true, std::memory_order_release); + }); + Py_END_ALLOW_THREADS // acquires GIL + } + + return PyObject_CallOneArg(impl, self); +} +``` + +The lock order is now always guard mutex first, GIL second. Unfortunately we +have to duplicate the “double-checked done flag”, effectively +leading to triple checking, because the flag state inside the `absl::once_flag` +is not accessible to the user. In other words, we cannot ask `init_flag` whether +it has been used yet. + +However, we can perform one last, minor optimisation: since we assume that the +GIL is held on entry, and again when the initializing operation returns, the GIL +actually serializes access to our done flag variable, which therefore does not +need to be atomic. (The difference to the previous, atomic code may be small, +depending on the architecture. For example, on x86-64, acquire/release on a bool +is nearly free ([demo](https://godbolt.org/z/P9vYWf4fE)).) + +```c++ +// CPython callback, assumes that the GIL is held on entry, and indeed anywhere +// directly in this function (i.e. the GIL can be released inside CreateWidget, +// but must be reaqcuired when that call returns). +PyObject* InvokeWidget(PyObject* self) { + static constinit PyObject* impl = nullptr; + static constinit bool init_done = false; // guarded by GIL + static constinit absl::once_flag init_flag; + + if (!init_done) { + Py_BEGIN_ALLOW_THREADS // releases GIL + // (multiple threads may enter here) + absl::call_once(init_flag, [&]() { + // (only one thread enters here) + PyGILState_STATE s = PyGILState_Ensure(); // acquires GIL + impl = CreateWidget(); + init_done = true; // (GIL is held) + PyGILState_Release(s); // releases GIL + }); + + Py_END_ALLOW_THREADS // acquires GIL + } + + return PyObject_CallOneArg(impl, self); +} +``` + +## Debugging tips + +* Build with symbols. +* Ctrl-C sends `SIGINT`, Ctrl-\\ + sends `SIGQUIT`. Both have their uses. +* Useful `gdb` commands: + * `py-bt` prints a Python backtrace if you are in a Python frame. + * `thread apply all bt 10` prints the top-10 frames for each thread. A + full backtrace can be prohibitively expensive, and the top few frames + are often good enough. + * `p PyGILState_Check()` shows whether a thread is holding the GIL. For + all threads, run `thread apply all p PyGILState_Check()` to find out + which thread is holding the GIL. + * The `static` variable guard mutex is accessed with functions like + `cxa_guard_acquire` (though this depends on ABI details and can vary). + The guard mutex itself contains information about which thread is + currently holding it. + +## Links + +* Article on + [double-checked locking](https://preshing.com/20130930/double-checked-locking-is-fixed-in-cpp11/) +* [The Deadlock Empire](https://deadlockempire.github.io/), hands-on exercises + to construct deadlocks diff --git a/wrap/pybind11/docs/advanced/embedding.rst b/wrap/pybind11/docs/advanced/embedding.rst index cbed82158..e78a8f4ce 100644 --- a/wrap/pybind11/docs/advanced/embedding.rst +++ b/wrap/pybind11/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.5...3.29) + cmake_minimum_required(VERSION 3.15...3.30) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` diff --git a/wrap/pybind11/docs/advanced/exceptions.rst b/wrap/pybind11/docs/advanced/exceptions.rst index e20f42b5f..8f0e9c93a 100644 --- a/wrap/pybind11/docs/advanced/exceptions.rst +++ b/wrap/pybind11/docs/advanced/exceptions.rst @@ -368,8 +368,7 @@ Should they throw or fail to catch any exceptions in their call graph, the C++ runtime calls ``std::terminate()`` to abort immediately. Similarly, Python exceptions raised in a class's ``__del__`` method do not -propagate, but are logged by Python as an unraisable error. In Python 3.8+, a -`system hook is triggered +propagate, but ``sys.unraisablehook()`` `is triggered `_ and an auditing event is logged. diff --git a/wrap/pybind11/docs/advanced/functions.rst b/wrap/pybind11/docs/advanced/functions.rst index 372934b09..ff00c9c8a 100644 --- a/wrap/pybind11/docs/advanced/functions.rst +++ b/wrap/pybind11/docs/advanced/functions.rst @@ -81,9 +81,11 @@ The following table provides an overview of available policies: | | it is no longer used. Warning: undefined behavior will ensue when the C++ | | | side deletes an object that is still referenced and used by Python. | +--------------------------------------------------+----------------------------------------------------------------------------+ -| :enum:`return_value_policy::reference_internal` | Indicates that the lifetime of the return value is tied to the lifetime | -| | of a parent object, namely the implicit ``this``, or ``self`` argument of | -| | the called method or property. Internally, this policy works just like | +| :enum:`return_value_policy::reference_internal` | If the return value is an lvalue reference or a pointer, the parent object | +| | (the implicit ``this``, or ``self`` argument of the called method or | +| | property) is kept alive for at least the lifespan of the return value. | +| | **Otherwise this policy falls back to :enum:`return_value_policy::move` | +| | (see #5528).** Internally, this policy works just like | | | :enum:`return_value_policy::reference` but additionally applies a | | | ``keep_alive<0, 1>`` *call policy* (described in the next section) that | | | prevents the parent object from being garbage collected as long as the | diff --git a/wrap/pybind11/docs/advanced/misc.rst b/wrap/pybind11/docs/advanced/misc.rst index ddd7f3937..3aeb8ab9d 100644 --- a/wrap/pybind11/docs/advanced/misc.rst +++ b/wrap/pybind11/docs/advanced/misc.rst @@ -62,7 +62,11 @@ will acquire the GIL before calling the Python callback. Similarly, the back into Python. When writing C++ code that is called from other C++ code, if that code accesses -Python state, it must explicitly acquire and release the GIL. +Python state, it must explicitly acquire and release the GIL. A separate +document on deadlocks [#f8]_ elaborates on a particularly subtle interaction +with C++'s block-scope static variable initializer guard mutexes. + +.. [#f8] See docs/advanced/deadlock.md The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to acquire and release the global interpreter lock in the body of a C++ @@ -76,7 +80,7 @@ could be realized as follows (important changes highlighted): .. code-block:: cpp :emphasize-lines: 8,30,31 - class PyAnimal : public Animal { + class PyAnimal : public Animal, py::trampoline_self_life_support { public: /* Inherit the constructors */ using Animal::Animal; @@ -94,12 +98,12 @@ could be realized as follows (important changes highlighted): }; PYBIND11_MODULE(example, m) { - py::class_ animal(m, "Animal"); + py::class_ animal(m, "Animal"); animal .def(py::init<>()) .def("go", &Animal::go); - py::class_(m, "Dog", animal) + py::class_(m, "Dog", animal) .def(py::init<>()); m.def("call_go", [](Animal *animal) -> std::string { @@ -142,6 +146,9 @@ following checklist. destructors can sometimes get invoked in weird and unexpected circumstances as a result of exceptions. +- C++ static block-scope variable initialization that calls back into Python can + cause deadlocks; see [#f8]_ for a detailed discussion. + - You should try running your code in a debug build. That will enable additional assertions within pybind11 that will throw exceptions on certain GIL handling errors (reference counting operations). @@ -181,7 +188,7 @@ from Section :ref:`inheritance`. Suppose now that ``Pet`` bindings are defined in a module named ``basic``, whereas the ``Dog`` bindings are defined somewhere else. The challenge is of course that the variable ``pet`` is not available anymore though it is needed -to indicate the inheritance relationship to the constructor of ``class_``. +to indicate the inheritance relationship to the constructor of ``py::class_``. However, it can be acquired as follows: .. code-block:: cpp @@ -193,7 +200,7 @@ However, it can be acquired as follows: .def("bark", &Dog::bark); Alternatively, you can specify the base class as a template parameter option to -``class_``, which performs an automated lookup of the corresponding Python +``py::class_``, which performs an automated lookup of the corresponding Python type. Like the above code, however, this also requires invoking the ``import`` function once to ensure that the pybind11 binding code of the module ``basic`` has been executed: diff --git a/wrap/pybind11/docs/advanced/smart_ptrs.rst b/wrap/pybind11/docs/advanced/smart_ptrs.rst index 3c40ce123..599f384db 100644 --- a/wrap/pybind11/docs/advanced/smart_ptrs.rst +++ b/wrap/pybind11/docs/advanced/smart_ptrs.rst @@ -1,11 +1,70 @@ -Smart pointers -############## +Smart pointers & ``py::class_`` +############################### -std::unique_ptr -=============== +The binding generator for classes, ``py::class_``, can be passed a template +type that denotes a special *holder* type that is used to manage references to +the object. If no such holder type template argument is given, the default for +a type ``T`` is ``std::unique_ptr``. -Given a class ``Example`` with Python bindings, it's possible to return -instances wrapped in C++11 unique pointers, like so +.. note:: + + A ``py::class_`` for a given C++ type ``T`` — and all its derived types — + can only use a single holder type. + + +.. _smart_holder: + +``py::smart_holder`` +==================== + +Starting with pybind11v3, ``py::smart_holder`` is built into pybind11. It is +the recommended ``py::class_`` holder for most situations. However, for +backward compatibility it is **not** the default holder, and there are no +plans to make it the default holder in the future. + +It is extremely easy to use the safer and more versatile ``py::smart_holder``: +simply add ``py::smart_holder`` to ``py::class_``: + +* ``py::class_`` to + +* ``py::class_``. + +.. note:: + + A shorthand, ``py::classh``, is provided for + ``py::class_``. The ``h`` in ``py::classh`` stands + for **smart_holder** but is shortened for brevity, ensuring it has the + same number of characters as ``py::class_``. This design choice facilitates + easy experimentation with ``py::smart_holder`` without introducing + distracting whitespace noise in diffs. + +The ``py::smart_holder`` functionality includes the following: + +* Support for **two-way** Python/C++ conversions for both + ``std::unique_ptr`` and ``std::shared_ptr`` **simultaneously**. + +* Passing a Python object back to C++ via ``std::unique_ptr``, safely + **disowning** the Python object. + +* Safely passing "trampoline" objects (objects with C++ virtual function + overrides implemented in Python, see :ref:`overriding_virtuals`) via + ``std::unique_ptr`` or ``std::shared_ptr`` back to C++: + associated Python objects are automatically kept alive for the lifetime + of the smart-pointer. + +* Full support for ``std::enable_shared_from_this`` (`cppreference + `_). + + +``std::unique_ptr`` +=================== + +This is the default ``py::class_`` holder and works as expected in +most situations. However, handling base-and-derived classes involves a +``reinterpret_cast``, which is, strictly speaking, undefined behavior. +Also note that the ``std::unique_ptr`` holder only supports passing a +``std::unique_ptr`` from C++ to Python, but not the other way around. +For example, the following code works as expected with ``py::class_``: .. code-block:: cpp @@ -15,116 +74,54 @@ instances wrapped in C++11 unique pointers, like so m.def("create_example", &create_example); -In other words, there is nothing special that needs to be done. While returning -unique pointers in this way is allowed, it is *illegal* to use them as function -arguments. For instance, the following function signature cannot be processed -by pybind11. +However, this will fail with ``py::class_`` (but works with +``py::class_``): .. code-block:: cpp void do_something_with_example(std::unique_ptr ex) { ... } -The above signature would imply that Python needs to give up ownership of an -object that is passed to this function, which is generally not possible (for -instance, the object might be referenced elsewhere). +.. note:: -std::shared_ptr -=============== + The ``reinterpret_cast`` mentioned above is `here + `_. + For completeness: The same cast is also applied to ``py::smart_holder``, + but that is safe, because ``py::smart_holder`` is not templated. -The binding generator for classes, :class:`class_`, can be passed a template -type that denotes a special *holder* type that is used to manage references to -the object. If no such holder type template argument is given, the default for -a type named ``Type`` is ``std::unique_ptr``, which means that the object -is deallocated when Python's reference count goes to zero. -It is possible to switch to other types of reference counting wrappers or smart -pointers, which is useful in codebases that rely on them. For instance, the -following snippet causes ``std::shared_ptr`` to be used instead. +``std::shared_ptr`` +=================== + +It is possible to use ``std::shared_ptr`` as the holder, for example: .. code-block:: cpp - py::class_ /* <- holder type */> obj(m, "Example"); + py::class_ /* <- holder type */>(m, "Example"); -Note that any particular class can only be associated with a single holder type. +Compared to using ``py::class_``, there are two noteworthy disadvantages: -One potential stumbling block when using holder types is that they need to be -applied consistently. Can you guess what's broken about the following binding -code? +* Because a ``py::class_`` for a given C++ type ``T`` can only use a + single holder type, ``std::unique_ptr`` cannot even be passed from C++ + to Python. This will become apparent only at runtime, often through a + segmentation fault. -.. code-block:: cpp +* Similar to the ``std::unique_ptr`` holder, the handling of base-and-derived + classes involves a ``reinterpret_cast`` that has strictly speaking undefined + behavior, although it works as expected in most situations. - class Child { }; - - class Parent { - public: - Parent() : child(std::make_shared()) { } - Child *get_child() { return child.get(); } /* Hint: ** DON'T DO THIS ** */ - private: - std::shared_ptr child; - }; - - PYBIND11_MODULE(example, m) { - py::class_>(m, "Child"); - - py::class_>(m, "Parent") - .def(py::init<>()) - .def("get_child", &Parent::get_child); - } - -The following Python code will cause undefined behavior (and likely a -segmentation fault). - -.. code-block:: python - - from example import Parent - - print(Parent().get_child()) - -The problem is that ``Parent::get_child()`` returns a pointer to an instance of -``Child``, but the fact that this instance is already managed by -``std::shared_ptr<...>`` is lost when passing raw pointers. In this case, -pybind11 will create a second independent ``std::shared_ptr<...>`` that also -claims ownership of the pointer. In the end, the object will be freed **twice** -since these shared pointers have no way of knowing about each other. - -There are two ways to resolve this issue: - -1. For types that are managed by a smart pointer class, never use raw pointers - in function arguments or return values. In other words: always consistently - wrap pointers into their designated holder types (such as - ``std::shared_ptr<...>``). In this case, the signature of ``get_child()`` - should be modified as follows: - -.. code-block:: cpp - - std::shared_ptr get_child() { return child; } - -2. Adjust the definition of ``Child`` by specifying - ``std::enable_shared_from_this`` (see cppreference_ for details) as a - base class. This adds a small bit of information to ``Child`` that allows - pybind11 to realize that there is already an existing - ``std::shared_ptr<...>`` and communicate with it. In this case, the - declaration of ``Child`` should look as follows: - -.. _cppreference: http://en.cppreference.com/w/cpp/memory/enable_shared_from_this - -.. code-block:: cpp - - class Child : public std::enable_shared_from_this { }; .. _smart_pointers: Custom smart pointers ===================== -pybind11 supports ``std::unique_ptr`` and ``std::shared_ptr`` right out of the -box. For any other custom smart pointer, transparent conversions can be enabled -using a macro invocation similar to the following. It must be declared at the -top namespace level before any binding code: +For custom smart pointers (e.g. ``c10::intrusive_ptr`` in pytorch), transparent +conversions can be enabled using a macro invocation similar to the following. +It must be declared at the top namespace level before any binding code: .. code-block:: cpp - PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr); + PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr) The first argument of :func:`PYBIND11_DECLARE_HOLDER_TYPE` should be a placeholder name that is used as a template parameter of the second argument. @@ -136,7 +133,7 @@ by default. Specify .. code-block:: cpp - PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr, true); + PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr, true) if ``SmartPtr`` can always be initialized from a ``T*`` pointer without the risk of inconsistencies (such as multiple independent ``SmartPtr`` instances @@ -154,7 +151,7 @@ specialized: .. code-block:: cpp // Always needed for custom holder types - PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr); + PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr) // Only needed if the type's `.get()` goes by another name namespace PYBIND11_NAMESPACE { namespace detail { @@ -167,8 +164,70 @@ specialized: The above specialization informs pybind11 that the custom ``SmartPtr`` class provides ``.get()`` functionality via ``.getPointer()``. +.. note:: + + The two noteworthy disadvantages mentioned under the ``std::shared_ptr`` + section apply similarly to custom smart pointer holders, but there is no + established safe alternative in this case. + .. seealso:: The file :file:`tests/test_smart_ptr.cpp` contains a complete example that demonstrates how to work with custom reference-counting holder types in more detail. + + +Be careful not to accidentally undermine automatic lifetime management +====================================================================== + +``py::class_``-wrapped objects automatically manage the lifetime of the +wrapped C++ object, in collaboration with the chosen holder type. +When wrapping C++ functions involving raw pointers, care needs to be taken +to not inadvertently transfer ownership, resulting in multiple Python +objects acting as owners, causing heap-use-after-free or double-free errors. +For example: + +.. code-block:: cpp + + class Child { }; + + class Parent { + public: + Parent() : child(std::make_shared()) { } + Child *get_child() { return child.get(); } /* DANGER */ + private: + std::shared_ptr child; + }; + + PYBIND11_MODULE(example, m) { + py::class_>(m, "Child"); + + py::class_>(m, "Parent") + .def(py::init<>()) + .def("get_child", &Parent::get_child); /* PROBLEM */ + } + +The following Python code leads to undefined behavior, likely resulting in +a segmentation fault. + +.. code-block:: python + + from example import Parent + + print(Parent().get_child()) + +Part of the ``/* PROBLEM */`` here is that pybind11 falls back to using +``return_value_policy::take_ownership`` as the default (see +:ref:`return_value_policies`). The fact that the ``Child`` instance is +already managed by ``std::shared_ptr`` is lost. Therefore pybind11 +will create a second independent ``std::shared_ptr`` that also +claims ownership of the pointer, eventually leading to heap-use-after-free +or double-free errors. + +There are various ways to resolve this issue, either by changing +the ``Child`` or ``Parent`` C++ implementations (e.g. using +``std::enable_shared_from_this`` as a base class for +``Child``, or adding a member function to ``Parent`` that returns +``std::shared_ptr``), or if that is not feasible, by using +``return_value_policy::reference_internal``. What is the best approach +depends on the exact situation. diff --git a/wrap/pybind11/docs/basics.rst b/wrap/pybind11/docs/basics.rst index e9b24c7fa..c7a0208c4 100644 --- a/wrap/pybind11/docs/basics.rst +++ b/wrap/pybind11/docs/basics.rst @@ -78,6 +78,13 @@ For brevity, all code examples assume that the following two lines are present: namespace py = pybind11; +.. note:: + + ``pybind11/pybind11.h`` includes ``Python.h``, as such it must be the first file + included in any source file or header for `the same reasons as Python.h`_. + +.. _`the same reasons as Python.h`: https://docs.python.org/3/extending/extending.html#a-simple-example + Some features may require additional headers, but those will be specified as needed. .. _simple_example: @@ -135,7 +142,7 @@ On Linux, the above example can be compiled using the following command: .. code-block:: bash - $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) + $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3 -m pybind11 --extension-suffix) .. note:: diff --git a/wrap/pybind11/docs/benchmark.py b/wrap/pybind11/docs/benchmark.py index a273674f4..26e390eb4 100644 --- a/wrap/pybind11/docs/benchmark.py +++ b/wrap/pybind11/docs/benchmark.py @@ -48,7 +48,7 @@ def generate_dummy_code_boost(nclasses=10): decl += "\n" for cl in range(nclasses): - decl += "class cl%03i {\n" % cl + decl += f"class cl{cl:03} {{\n" decl += "public:\n" bindings += f' py::class_("cl{cl:03}")\n' for fn in range(nfns): @@ -85,5 +85,5 @@ for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]: n2 = dt.datetime.now() elapsed = (n2 - n1).total_seconds() size = os.stat("test.so").st_size - print(" {%i, %f, %i}," % (nclasses * nfns, elapsed, size)) + print(f" {{{nclasses * nfns}, {elapsed:.6f}, {size}}},") print("}") diff --git a/wrap/pybind11/docs/changelog.rst b/wrap/pybind11/docs/changelog.rst index ab6c713c1..a91082113 100644 --- a/wrap/pybind11/docs/changelog.rst +++ b/wrap/pybind11/docs/changelog.rst @@ -15,6 +15,146 @@ IN DEVELOPMENT Changes will be summarized here periodically. +New Features: + +* Support for Python 3.7 was removed. (Official end-of-life: 2023-06-27). + `#5191 `_ + +* stl.h ``list|set|map_caster`` were made more user friendly: it is no longer + necessary to explicitly convert Python iterables to ``tuple()``, ``set()``, + or ``map()`` in many common situations. + `#4686 `_ + +* Support for CMake older than 3.15 removed. CMake 3.15-3.30 supported. + `#5304 `_ + +* The ``array_caster`` in pybind11/stl.h was enhanced to support value types that are not default-constructible. + `#5305 `_ + +* Added ``py::warnings`` namespace with ``py::warnings::warn`` and ``py::warnings::new_warning_type`` that provides the interface for Python warnings. + `#5291 `_ + +Version 2.13.6 (September 13, 2024) +----------------------------------- + +New Features: + +* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all + ``py::class_``-wrapped types, to enable type-safe interoperability between + different independent Python/C++ bindings systems, including pybind11 + versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on + pybind11 2.11.2, 2.12.1, and 2.13.6+. + `#5296 `_ + + +Bug fixes: + +* Using ``__cpp_nontype_template_args`` instead of ``__cpp_nontype_template_parameter_class``. + `#5330 `_ + +* Properly translate C++ exception to Python exception when creating Python buffer from wrapped object. + `#5324 `_ + + +Documentation: + +* Adds an answer (FAQ) for "What is a highly conclusive and simple way to find memory leaks?". + `#5340 `_ + + +Version 2.13.5 (August 22, 2024) +-------------------------------- + +Bug fixes: + +* Fix includes when using Windows long paths (``\\?\`` prefix). + `#5321 `_ + +* Support ``-Wpedantic`` in C++20 mode. + `#5322 `_ + +* Fix and test ```` support for ``py::tuple`` and ``py::list``. + `#5314 `_ + +Version 2.13.4 (August 14, 2024) +-------------------------------- + +Bug fixes: + +* Fix paths with spaces, including on Windows. + (Replaces regression from `#5302 `_) + `#4874 `_ + +Documentation: + +* Remove repetitive words. + `#5308 `_ + + +Version 2.13.3 (August 13, 2024) +-------------------------------- + +Bug fixes: + +* Quote paths from pybind11-config + `#5302 `_ + + +* Fix typo in Emscripten support when in config mode (CMake) + `#5301 `_ + + +Version 2.13.2 (August 13, 2024) +-------------------------------- + +New Features: + +* A ``pybind11::detail::type_caster_std_function_specializations`` feature was added, to support specializations for + ``std::function``'s with return types that require custom to-Python conversion behavior (to primary use case is to catch and + convert exceptions). + `#4597 `_ + + +Changes: + + +* Use ``PyMutex`` instead of ``std::mutex`` for internal locking in the free-threaded build. + `#5219 `_ + +* Add a special type annotation for C++ empty tuple. + `#5214 `_ + +* When compiling for WebAssembly, add the required exception flags (CMake 3.13+). + `#5298 `_ + +Bug fixes: + +* Make ``gil_safe_call_once_and_store`` thread-safe in free-threaded CPython. + `#5246 `_ + +* A missing ``#include `` in pybind11/typing.h was added to fix build errors (in case user code does not already depend + on that include). + `#5208 `_ + +* Fix regression introduced in #5201 for GCC<10.3 in C++20 mode. + `#5205 `_ + + +.. fix(cmake) + +* Remove extra = when assigning flto value in the case for Clang in CMake. + `#5207 `_ + + +Tests: + +* Adding WASM testing to our CI (Pyodide / Emscripten via scikit-build-core). + `#4745 `_ + +* clang-tidy (in GitHub Actions) was updated from clang 15 to clang 18. + `#5272 `_ + + Version 2.13.1 (June 26, 2024) ------------------------------ @@ -129,6 +269,18 @@ Other: * Update docs and noxfile. `#5071 `_ +Version 2.12.1 (September 13, 2024) +----------------------------------- + +New Features: + +* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all + ``py::class_``-wrapped types, to enable type-safe interoperability between + different independent Python/C++ bindings systems, including pybind11 + versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on + pybind11 2.11.2, 2.12.1, and 2.13.6+. + `#5296 `_ + Version 2.12.0 (March 27, 2024) ------------------------------- @@ -304,6 +456,18 @@ Other: * An ``assert()`` was added to help Coverty avoid generating a false positive. `#4817 `_ +Version 2.11.2 (September 13, 2024) +----------------------------------- + +New Features: + +* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all + ``py::class_``-wrapped types, to enable type-safe interoperability between + different independent Python/C++ bindings systems, including pybind11 + versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on + pybind11 2.11.2, 2.12.1, and 2.13.6+. + `#5296 `_ + Version 2.11.1 (July 17, 2023) ------------------------------ diff --git a/wrap/pybind11/docs/classes.rst b/wrap/pybind11/docs/classes.rst index 4f2167dac..5406668f0 100644 --- a/wrap/pybind11/docs/classes.rst +++ b/wrap/pybind11/docs/classes.rst @@ -34,11 +34,18 @@ The binding code for ``Pet`` looks as follows: .def("getName", &Pet::getName); } -:class:`class_` creates bindings for a C++ *class* or *struct*-style data +``py::class_`` creates bindings for a C++ *class* or *struct*-style data structure. :func:`init` is a convenience function that takes the types of a constructor's parameters as template arguments and wraps the corresponding -constructor (see the :ref:`custom_constructors` section for details). An -interactive Python session demonstrating this example is shown below: +constructor (see the :ref:`custom_constructors` section for details). + +.. note:: + + Starting with pybind11v3, it is recommended to include `py::smart_holder` + in most situations for safety, especially if you plan to support conversions + to C++ smart pointers. See :ref:`smart_holder` for more information. + +An interactive Python session demonstrating this example is shown below: .. code-block:: pycon @@ -258,7 +265,7 @@ inheritance relationship: There are two different ways of indicating a hierarchical relationship to pybind11: the first specifies the C++ base class as an extra template -parameter of the :class:`class_`: +parameter of the ``py::class_``: .. code-block:: cpp @@ -272,7 +279,7 @@ parameter of the :class:`class_`: .def("bark", &Dog::bark); Alternatively, we can also assign a name to the previously bound ``Pet`` -:class:`class_` object and reference it when binding the ``Dog`` class: +``py::class_`` object and reference it when binding the ``Dog`` class: .. code-block:: cpp @@ -498,7 +505,7 @@ The binding code for this example looks as follows: To ensure that the nested types ``Kind`` and ``Attributes`` are created within the scope of ``Pet``, the -``pet`` :class:`class_` instance must be supplied to the :class:`enum_` and :class:`class_` +``pet`` ``py::class_`` instance must be supplied to the :class:`enum_` and ``py::class_`` constructor. The :func:`enum_::export_values` function exports the enum entries into the parent scope, which should be skipped for newer C++11-style strongly typed enums. diff --git a/wrap/pybind11/docs/compiling.rst b/wrap/pybind11/docs/compiling.rst index 0b7c178b0..94042c3e5 100644 --- a/wrap/pybind11/docs/compiling.rst +++ b/wrap/pybind11/docs/compiling.rst @@ -18,14 +18,14 @@ A Python extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...3.29) + cmake_minimum_required(VERSION 3.15...3.30) project(example LANGUAGES CXX) set(PYBIND11_FINDPYTHON ON) find_package(pybind11 CONFIG REQUIRED) pybind11_add_module(example example.cpp) - install(TARGET example DESTINATION .) + install(TARGETS example DESTINATION .) (You use the ``add_subdirectory`` instead, see the example in :ref:`cmake`.) In this example, the code is located in a file named :file:`example.cpp`. Either @@ -319,11 +319,11 @@ Building with CMake For C++ codebases that have an existing CMake-based build system, a Python extension module can be created with just a few lines of code, as seen above in -the module section. Pybind11 currently supports a lower minimum if you don't -use the modern FindPython, though be aware that CMake 3.27 removed the old -mechanism, so pybind11 will automatically switch if the old mechanism is not -available. Please opt into the new mechanism if at all possible. Our default -may change in future versions. This is the minimum required: +the module section. Pybind11 currently defaults to the old mechanism, though be +aware that CMake 3.27 removed the old mechanism, so pybind11 will automatically +switch if the old mechanism is not available. Please opt into the new mechanism +if at all possible. Our default may change in future versions. This is the +minimum required: @@ -333,6 +333,9 @@ may change in future versions. This is the minimum required: .. versionchanged:: 2.11 CMake 3.5+ is required. +.. versionchanged:: 2.14 + CMake 3.15+ is required. + Further information can be found at :doc:`cmake/index`. @@ -388,7 +391,7 @@ that will be respected instead of the built-in flag search. The ``OPT_SIZE`` flag enables size-based optimization equivalent to the standard ``/Os`` or ``-Os`` compiler flags and the ``MinSizeRel`` build type, -which avoid optimizations that that can substantially increase the size of the +which avoid optimizations that can substantially increase the size of the resulting binary. This flag is particularly useful in projects that are split into performance-critical parts and associated bindings. In this case, we can compile the project in release mode (and hence, optimize performance globally), @@ -426,7 +429,7 @@ with ``PYTHON_EXECUTABLE``. For example: .. code-block:: bash - cmake -DPYBIND11_PYTHON_VERSION=3.7 .. + cmake -DPYBIND11_PYTHON_VERSION=3.8 .. # Another method: cmake -DPYTHON_EXECUTABLE=/path/to/python .. @@ -444,7 +447,7 @@ See the `Config file`_ docstring for details of relevant CMake variables. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.15...3.30) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) @@ -483,17 +486,16 @@ can refer to the same [cmake_example]_ repository for a full sample project FindPython mode --------------- -CMake 3.12+ (3.15+ recommended, 3.18.2+ ideal) added a new module called -FindPython that had a highly improved search algorithm and modern targets -and tools. If you use FindPython, pybind11 will detect this and use the -existing targets instead: +Modern CMake (3.18.2+ ideal) added a new module called FindPython that had a +highly improved search algorithm and modern targets and tools. If you use +FindPython, pybind11 will detect this and use the existing targets instead: .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...3.22) + cmake_minimum_required(VERSION 3.15...3.30) project(example LANGUAGES CXX) - find_package(Python 3.7 COMPONENTS Interpreter Development REQUIRED) + find_package(Python 3.8 COMPONENTS Interpreter Development REQUIRED) find_package(pybind11 CONFIG REQUIRED) # or add_subdirectory(pybind11) @@ -541,7 +543,7 @@ available in all modes. The targets provided are: Just the "linking" part of pybind11:module ``pybind11::module`` - Everything for extension modules - ``pybind11::pybind11`` + ``Python::Module`` (FindPython CMake 3.15+) or ``pybind11::python_link_helper`` + Everything for extension modules - ``pybind11::pybind11`` + ``Python::Module`` (FindPython) or ``pybind11::python_link_helper`` ``pybind11::embed`` Everything for embedding the Python interpreter - ``pybind11::pybind11`` + ``Python::Python`` (FindPython) or Python libs @@ -568,7 +570,7 @@ You can use these targets to build complex applications. For example, the .. code-block:: cmake - cmake_minimum_required(VERSION 3.5...3.29) + cmake_minimum_required(VERSION 3.15...3.30) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -626,7 +628,7 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.5...3.29) + cmake_minimum_required(VERSION 3.15...3.30) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -719,7 +721,7 @@ customizable pybind11-based wrappers by parsing C++ header files. [litgen]_ is an automatic python bindings generator with a focus on generating documented and discoverable bindings: bindings will nicely reproduce the documentation -found in headers. It is is based on srcML (srcml.org), a highly scalable, multi-language +found in headers. It is based on srcML (srcml.org), a highly scalable, multi-language parsing tool with a developer centric approach. The API that you want to expose to python must be C++14 compatible (but your implementation can use more modern constructs). diff --git a/wrap/pybind11/docs/faq.rst b/wrap/pybind11/docs/faq.rst index 1eb00efad..31e33f8b5 100644 --- a/wrap/pybind11/docs/faq.rst +++ b/wrap/pybind11/docs/faq.rst @@ -247,6 +247,50 @@ been received, you must either explicitly interrupt execution by throwing }); } +What is a highly conclusive and simple way to find memory leaks (e.g. in pybind11 bindings)? +============================================================================================ + +Use ``while True`` & ``top`` (Linux, macOS). + +For example, locally change tests/test_type_caster_pyobject_ptr.py like this: + +.. code-block:: diff + + def test_return_list_pyobject_ptr_reference(): + + while True: + vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + # Commenting out the next `assert` will leak the Python references. + # An easy way to see evidence of the leaks: + # Insert `while True:` as the first line of this function and monitor the + # process RES (Resident Memory Size) with the Unix top command. + - assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + + # assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + +Then run the test as you would normally do, which will go into the infinite loop. + +**In another shell, but on the same machine** run: + +.. code-block:: bash + + top + +This will show: + +.. code-block:: + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1266095 rwgk 20 0 5207496 611372 45696 R 100.0 0.3 0:08.01 test_type_caste + +Look for the number under ``RES`` there. You'll see it going up very quickly. + +**Don't forget to Ctrl-C the test command** before your machine becomes +unresponsive due to swapping. + +This method only takes a couple minutes of effort and is very conclusive. +What you want to see is that the ``RES`` number is stable after a couple +seconds. + CMake doesn't detect the right Python version ============================================= @@ -258,9 +302,9 @@ CMake configure line. (Replace ``$(which python)`` with a path to python if your prefer.) You can alternatively try ``-DPYBIND11_FINDPYTHON=ON``, which will activate the -new CMake FindPython support instead of pybind11's custom search. Requires -CMake 3.12+, and 3.15+ or 3.18.2+ are even better. You can set this in your -``CMakeLists.txt`` before adding or finding pybind11, as well. +new CMake FindPython support instead of pybind11's custom search. Newer CMake, +like, 3.18.2+, is recommended. You can set this in your ``CMakeLists.txt`` +before adding or finding pybind11, as well. Inconsistent detection of Python version in CMake and pybind11 ============================================================== @@ -281,11 +325,11 @@ There are three possible solutions: from CMake and rely on pybind11 in detecting Python version. If this is not possible, the CMake machinery should be called *before* including pybind11. 2. Set ``PYBIND11_FINDPYTHON`` to ``True`` or use ``find_package(Python - COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, - 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead - of the old, deprecated search tools, and these modules are much better at - finding the correct Python. If FindPythonLibs/Interp are not available - (CMake 3.27+), then this will be ignored and FindPython will be used. + COMPONENTS Interpreter Development)`` on modern CMake ( 3.18.2+ best). + Pybind11 in these cases uses the new CMake FindPython instead of the old, + deprecated search tools, and these modules are much better at finding the + correct Python. If FindPythonLibs/Interp are not available (CMake 3.27+), + then this will be ignored and FindPython will be used. 3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. However, you will have to use the target-based system, and do more setup yourself, because it does not know about or include things that depend on diff --git a/wrap/pybind11/docs/limitations.rst b/wrap/pybind11/docs/limitations.rst index def5ad659..1b06ea872 100644 --- a/wrap/pybind11/docs/limitations.rst +++ b/wrap/pybind11/docs/limitations.rst @@ -50,10 +50,6 @@ clean, well written patch would likely be accepted to solve them. One consequence is that containers of ``char *`` are currently not supported. `#2245 `_ -- The ``cpptest`` does not run on Windows with Python 3.8 or newer, due to DLL - loader changes. User code that is correctly installed should not be affected. - `#2560 `_ - Python 3.9.0 warning ^^^^^^^^^^^^^^^^^^^^ diff --git a/wrap/pybind11/docs/reference.rst b/wrap/pybind11/docs/reference.rst index e64a03519..c2757988d 100644 --- a/wrap/pybind11/docs/reference.rst +++ b/wrap/pybind11/docs/reference.rst @@ -68,8 +68,8 @@ Convenience functions converting to Python types .. _extras: -Passing extra arguments to ``def`` or ``class_`` -================================================ +Passing extra arguments to ``def`` or ``py::class_`` +==================================================== .. doxygengroup:: annotations :members: diff --git a/wrap/pybind11/docs/requirements.txt b/wrap/pybind11/docs/requirements.txt index 8dffd36b5..aa2d86867 100644 --- a/wrap/pybind11/docs/requirements.txt +++ b/wrap/pybind11/docs/requirements.txt @@ -16,9 +16,9 @@ breathe==4.35.0 \ --hash=sha256:5165541c3c67b6c7adde8b3ecfe895c6f7844783c4076b6d8d287e4f33d62386 \ --hash=sha256:52c581f42ca4310737f9e435e3851c3d1f15446205a85fbc272f1f97ed74f5be # via -r requirements.in -certifi==2024.2.2 \ - --hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \ - --hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1 +certifi==2024.7.4 \ + --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \ + --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90 # via requests charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -130,9 +130,9 @@ imagesize==1.4.1 \ --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \ --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a # via sphinx -jinja2==3.1.5 \ - --hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \ - --hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via sphinx markupsafe==2.1.5 \ --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ diff --git a/wrap/pybind11/docs/upgrade.rst b/wrap/pybind11/docs/upgrade.rst index 17c26aaa9..5cef2b81a 100644 --- a/wrap/pybind11/docs/upgrade.rst +++ b/wrap/pybind11/docs/upgrade.rst @@ -24,7 +24,8 @@ changes are that: function is not available anymore. Due to NumPy changes, you may experience difficulties updating to NumPy 2. -Please see the [NumPy 2 migration guide](https://numpy.org/devdocs/numpy_2_0_migration_guide.html) for details. +Please see the `NumPy 2 migration guide `_ +for details. For example, a more direct change could be that the default integer ``"int_"`` (and ``"uint"``) is now ``ssize_t`` and not ``long`` (affects 64bit windows). diff --git a/wrap/pybind11/include/pybind11/attr.h b/wrap/pybind11/include/pybind11/attr.h index 1044db94d..ef2ca1709 100644 --- a/wrap/pybind11/include/pybind11/attr.h +++ b/wrap/pybind11/include/pybind11/attr.h @@ -81,6 +81,10 @@ struct dynamic_attr {}; /// Annotation which enables the buffer protocol for a type struct buffer_protocol {}; +/// Annotation which enables releasing the GIL before calling the C++ destructor of wrapped +/// instances (pybind/pybind11#1446). +struct release_gil_before_calling_cpp_dtor {}; + /// Annotation which requests that a special metaclass is created for a type struct metaclass { handle value; @@ -272,7 +276,7 @@ struct function_record { struct type_record { PYBIND11_NOINLINE type_record() : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), - default_holder(true), module_local(false), is_final(false) {} + module_local(false), is_final(false), release_gil_before_calling_cpp_dtor(false) {} /// Handle to the parent scope handle scope; @@ -322,15 +326,17 @@ struct type_record { /// Does the class implement the buffer protocol? bool buffer_protocol : 1; - /// Is the default (unique_ptr) holder type used? - bool default_holder : 1; - /// Is the class definition local to the module shared object? bool module_local : 1; /// Is the class inheritable from python classes? bool is_final : 1; + /// Solves pybind/pybind11#1446 + bool release_gil_before_calling_cpp_dtor : 1; + + holder_enum_t holder_enum_v = holder_enum_t::undefined; + PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) { auto *base_info = detail::get_type_info(base, false); if (!base_info) { @@ -340,18 +346,22 @@ struct type_record { + "\" referenced unknown base type \"" + tname + "\""); } - if (default_holder != base_info->default_holder) { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool this_has_unique_ptr_holder = (holder_enum_v == holder_enum_t::std_unique_ptr); + bool base_has_unique_ptr_holder + = (base_info->holder_enum_v == holder_enum_t::std_unique_ptr); + if (this_has_unique_ptr_holder != base_has_unique_ptr_holder) { std::string tname(base.name()); detail::clean_type_id(tname); pybind11_fail("generic_type: type \"" + std::string(name) + "\" " - + (default_holder ? "does not have" : "has") + + (this_has_unique_ptr_holder ? "does not have" : "has") + " a non-default holder type while its base \"" + tname + "\" " - + (base_info->default_holder ? "does not" : "does")); + + (base_has_unique_ptr_holder ? "does not" : "does")); } bases.append((PyObject *) base_info->type); -#if PY_VERSION_HEX < 0x030B0000 +#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET dynamic_attr |= base_info->type->tp_dictoffset != 0; #else dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; @@ -603,6 +613,14 @@ struct process_attribute : process_attribute_default static void init(const module_local &l, type_record *r) { r->module_local = l.value; } }; +template <> +struct process_attribute + : process_attribute_default { + static void init(const release_gil_before_calling_cpp_dtor &, type_record *r) { + r->release_gil_before_calling_cpp_dtor = true; + } +}; + /// Process a 'prepend' attribute, putting this at the beginning of the overload chain template <> struct process_attribute : process_attribute_default { diff --git a/wrap/pybind11/include/pybind11/cast.h b/wrap/pybind11/include/pybind11/cast.h index 624b8ebac..47575084c 100644 --- a/wrap/pybind11/include/pybind11/cast.h +++ b/wrap/pybind11/include/pybind11/cast.h @@ -158,7 +158,7 @@ public: } else { handle src_or_index = src; // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. -#if PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) +#if defined(PYPY_VERSION) object index; if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) index = reinterpret_steal(PyNumber_Index(src.ptr())); @@ -343,7 +343,7 @@ public: #else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. - else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) { + else if (auto *tp_as_number = Py_TYPE(src.ptr())->tp_as_number) { if (PYBIND11_NB_BOOL(tp_as_number)) { res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr()); } @@ -740,6 +740,13 @@ class type_caster> : public tuple_caster {} template class type_caster> : public tuple_caster {}; +template <> +class type_caster> : public tuple_caster { +public: + // PEP 484 specifies this syntax for an empty tuple + static constexpr auto name = const_name("tuple[()]"); +}; + /// Helper class which abstracts away certain actions. Users can provide specializations for /// custom holders, but it's only necessary if the type has a non-standard interface. template @@ -747,6 +754,7 @@ struct holder_helper { static auto get(const T &p) -> decltype(p.get()) { return p.get(); } }; +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to shared_ptr specialization. /// Type caster for holder types like std::shared_ptr, etc. /// The SFINAE hook is provided to help work around the current lack of support /// for smart-pointer interoperability. Please consider it an implementation @@ -782,16 +790,19 @@ public: protected: friend class type_caster_generic; void check_holder_compat() { - if (typeinfo->default_holder) { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool inst_has_unique_ptr_holder + = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr); + if (inst_has_unique_ptr_holder) { throw cast_error("Unable to load a custom holder type from a default-holder instance"); } } - bool load_value(value_and_holder &&v_h) { + void load_value(value_and_holder &&v_h) { if (v_h.holder_constructed()) { value = v_h.value_ptr(); holder = v_h.template holder(); - return true; + return; } throw cast_error("Unable to cast from non-held to held instance (T& to Holder) " #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) @@ -828,10 +839,144 @@ protected: holder_type holder; }; +template +struct copyable_holder_caster_shared_ptr_with_smart_holder_support_enabled : std::true_type {}; + +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor copyable_holder_caster to reduce code duplication. +template +struct copyable_holder_caster< + type, + std::shared_ptr, + enable_if_t::value>> + : public type_caster_base { +public: + using base = type_caster_base; + static_assert(std::is_base_of>::value, + "Holder classes are only supported for custom types"); + using base::base; + using base::cast; + using base::typeinfo; + using base::value; + + bool load(handle src, bool convert) { + if (base::template load_impl>>( + src, convert)) { + sh_load_helper.maybe_set_python_instance_is_alias(src); + return true; + } + return false; + } + + explicit operator std::shared_ptr *() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + pybind11_fail("Passing `std::shared_ptr *` from Python to C++ is not supported " + "(inherently unsafe)."); + } + return std::addressof(shared_ptr_storage); + } + + explicit operator std::shared_ptr &() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + shared_ptr_storage = sh_load_helper.load_as_shared_ptr(value); + } + return shared_ptr_storage; + } + + static handle + cast(const std::shared_ptr &src, return_value_policy policy, handle parent) { + const auto *ptr = src.get(); + auto st = type_caster_base::src_and_type(ptr); + if (st.second == nullptr) { + return handle(); // no type info: error will be set already + } + if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + return smart_holder_type_caster_support::smart_holder_from_shared_ptr( + src, policy, parent, st); + } + return type_caster_base::cast_holder(ptr, &src); + } + + // This function will succeed even if the `responsible_parent` does not own the + // wrapped C++ object directly. + // It is the responsibility of the caller to ensure that the `responsible_parent` + // has a `keep_alive` relationship with the owner of the wrapped C++ object, or + // that the wrapped C++ object lives for the duration of the process. + static std::shared_ptr shared_ptr_with_responsible_parent(handle responsible_parent) { + copyable_holder_caster loader; + loader.load(responsible_parent, /*convert=*/false); + assert(loader.typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder); + return loader.sh_load_helper.load_as_shared_ptr(loader.value, responsible_parent); + } + +protected: + friend class type_caster_generic; + void check_holder_compat() { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool inst_has_unique_ptr_holder + = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr); + if (inst_has_unique_ptr_holder) { + throw cast_error("Unable to load a custom holder type from a default-holder instance"); + } + } + + void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = v_h; + sh_load_helper.was_populated = true; + value = sh_load_helper.get_void_ptr_or_nullptr(); + return; + } + if (v_h.holder_constructed()) { + value = v_h.value_ptr(); + shared_ptr_storage = v_h.template holder>(); + return; + } + throw cast_error("Unable to cast from non-held to held instance (T& to Holder) " +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "type information)"); +#else + "of type '" + + type_id>() + "''"); +#endif + } + + template , + detail::enable_if_t::value, int> = 0> + bool try_implicit_casts(handle, bool) { + return false; + } + + template , + detail::enable_if_t::value, int> = 0> + bool try_implicit_casts(handle src, bool convert) { + for (auto &cast : typeinfo->implicit_casts) { + copyable_holder_caster sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + } else { + shared_ptr_storage + = std::shared_ptr(sub_caster.shared_ptr_storage, (type *) value); + } + return true; + } + } + return false; + } + + static bool try_direct_conversions(handle) { return false; } + + smart_holder_type_caster_support::load_helper> sh_load_helper; // Const2Mutbl + std::shared_ptr shared_ptr_storage; +}; + /// Specialize for the common std::shared_ptr, so users don't need to template class type_caster> : public copyable_holder_caster> {}; +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to unique_ptr specialization. /// Type caster for holder types like std::unique_ptr. /// Please consider the SFINAE hook an implementation detail, as explained /// in the comment for the copyable_holder_caster. @@ -847,6 +992,143 @@ struct move_only_holder_caster { static constexpr auto name = type_caster_base::name; }; +template +struct move_only_holder_caster_unique_ptr_with_smart_holder_support_enabled : std::true_type {}; + +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor move_only_holder_caster to reduce code duplication. +template +struct move_only_holder_caster< + type, + std::unique_ptr, + enable_if_t::value>> + : public type_caster_base { +public: + using base = type_caster_base; + static_assert(std::is_base_of>::value, + "Holder classes are only supported for custom types"); + using base::base; + using base::cast; + using base::typeinfo; + using base::value; + + static handle + cast(std::unique_ptr &&src, return_value_policy policy, handle parent) { + auto *ptr = src.get(); + auto st = type_caster_base::src_and_type(ptr); + if (st.second == nullptr) { + return handle(); // no type info: error will be set already + } + if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + return smart_holder_type_caster_support::smart_holder_from_unique_ptr( + std::move(src), policy, parent, st); + } + return type_caster_generic::cast(st.first, + return_value_policy::take_ownership, + {}, + st.second, + nullptr, + nullptr, + std::addressof(src)); + } + + static handle + cast(const std::unique_ptr &src, return_value_policy policy, handle parent) { + if (!src) { + return none().release(); + } + if (policy == return_value_policy::automatic) { + policy = return_value_policy::reference_internal; + } + if (policy != return_value_policy::reference_internal) { + throw cast_error("Invalid return_value_policy for const unique_ptr&"); + } + return type_caster_base::cast(src.get(), policy, parent); + } + + bool load(handle src, bool convert) { + if (base::template load_impl< + move_only_holder_caster>>(src, convert)) { + sh_load_helper.maybe_set_python_instance_is_alias(src); + return true; + } + return false; + } + + void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = v_h; + sh_load_helper.loaded_v_h.type = typeinfo; + sh_load_helper.was_populated = true; + value = sh_load_helper.get_void_ptr_or_nullptr(); + return; + } + pybind11_fail("Passing `std::unique_ptr` from Python to C++ requires `py::class_` (with T = " + + clean_type_id(typeinfo->cpptype->name()) + ")"); + } + + template + using cast_op_type + = conditional_t::type, + const std::unique_ptr &>::value + || std::is_same::type, + const std::unique_ptr &>::value, + const std::unique_ptr &, + std::unique_ptr>; + + explicit operator std::unique_ptr() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + return sh_load_helper.template load_as_unique_ptr(value); + } + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); + } + + explicit operator const std::unique_ptr &() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + // Get shared_ptr to ensure that the Python object is not disowned elsewhere. + shared_ptr_storage = sh_load_helper.load_as_shared_ptr(value); + // Build a temporary unique_ptr that is meant to never expire. + unique_ptr_storage = std::shared_ptr>( + new std::unique_ptr{ + sh_load_helper.template load_as_const_unique_ptr( + shared_ptr_storage.get())}, + [](std::unique_ptr *ptr) { + if (!ptr) { + pybind11_fail("FATAL: `const std::unique_ptr &` was disowned " + "(EXPECT UNDEFINED BEHAVIOR)."); + } + (void) ptr->release(); + delete ptr; + }); + return *unique_ptr_storage; + } + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); + } + + bool try_implicit_casts(handle src, bool convert) { + for (auto &cast : typeinfo->implicit_casts) { + move_only_holder_caster sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + } else { + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ + ":" PYBIND11_TOSTRING(__LINE__)); + } + return true; + } + } + return false; + } + + static bool try_direct_conversions(handle) { return false; } + + smart_holder_type_caster_support::load_helper> sh_load_helper; // Const2Mutbl + std::shared_ptr shared_ptr_storage; // Serves as a pseudo lock. + std::shared_ptr> unique_ptr_storage; +}; + template class type_caster> : public move_only_holder_caster> {}; @@ -856,18 +1138,20 @@ using type_caster_holder = conditional_t::val copyable_holder_caster, move_only_holder_caster>; -template -struct always_construct_holder { +template +struct always_construct_holder_value { static constexpr bool value = Value; }; +template +struct always_construct_holder : always_construct_holder_value {}; + /// Create a specialization for custom holder types (silently ignores std::shared_ptr) #define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...) \ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ namespace detail { \ template \ - struct always_construct_holder : always_construct_holder { \ - }; \ + struct always_construct_holder : always_construct_holder_value<__VA_ARGS__> {}; \ template \ class type_caster::value>> \ : public type_caster_holder {}; \ @@ -878,10 +1162,14 @@ struct always_construct_holder { template struct is_holder_type : std::is_base_of, detail::type_caster> {}; -// Specialization for always-supported unique_ptr holders: + +// Specializations for always-supported holders: template struct is_holder_type> : std::true_type {}; +template +struct is_holder_type : std::true_type {}; + #ifdef PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION // See PR #4888 // This leads to compilation errors if a specialization is missing. @@ -1005,10 +1293,18 @@ template <> struct handle_type_name { static constexpr auto name = const_name("*args"); }; +template +struct handle_type_name> { + static constexpr auto name = const_name("*args: ") + make_caster::name; +}; template <> struct handle_type_name { static constexpr auto name = const_name("**kwargs"); }; +template +struct handle_type_name> { + static constexpr auto name = const_name("**kwargs: ") + make_caster::name; +}; template <> struct handle_type_name { static constexpr auto name = const_name(); @@ -1314,6 +1610,31 @@ object object_or_cast(T &&o) { return pybind11::cast(std::forward(o)); } +// Declared in pytypes.h: +// Implemented here so that make_caster can be used. +template +template +str_attr_accessor object_api::attr_with_type_hint(const char *key) const { +#if !defined(__cpp_inline_variables) + static_assert(always_false::value, + "C++17 feature __cpp_inline_variables not available: " + "https://en.cppreference.com/w/cpp/language/static#Static_data_members"); +#endif + object ann = annotations(); + if (ann.contains(key)) { + throw std::runtime_error("__annotations__[\"" + std::string(key) + "\"] was set already."); + } + ann[key] = make_caster::name.text; + return {derived(), key}; +} + +template +template +obj_attr_accessor object_api::attr_with_type_hint(handle key) const { + (void) attr_with_type_hint(key.cast().c_str()); + return {derived(), reinterpret_borrow(key)}; +} + // Placeholder type for the unneeded (and dead code) static variable in the // PYBIND11_OVERRIDE_OVERRIDE macro struct override_unused {}; @@ -1496,7 +1817,7 @@ struct kw_only {}; /// \ingroup annotations /// Annotation indicating that all previous arguments are positional-only; the is the equivalent of -/// an unnamed '/' argument (in Python 3.8) +/// an unnamed '/' argument struct pos_only {}; template @@ -1557,15 +1878,24 @@ struct function_call { handle init_self; }; +// See PR #5396 for the discussion that led to this +template +struct is_same_or_base_of : std::is_same {}; + +// Only evaluate is_base_of if Derived is complete. +// is_base_of raises a compiler error if Derived is incomplete. +template +struct is_same_or_base_of + : any_of, std::is_base_of> {}; + /// Helper class which loads arguments for C++ functions called from Python template class argument_loader { using indices = make_index_sequence; - template - using argument_is_args = std::is_same, args>; + using argument_is_args = is_same_or_base_of>; template - using argument_is_kwargs = std::is_same, kwargs>; + using argument_is_kwargs = is_same_or_base_of>; // Get kwargs argument position, or -1 if not present: static constexpr auto kwargs_pos = constexpr_last(); diff --git a/wrap/pybind11/include/pybind11/conduit/README.txt b/wrap/pybind11/include/pybind11/conduit/README.txt new file mode 100644 index 000000000..9a2c53ba4 --- /dev/null +++ b/wrap/pybind11/include/pybind11/conduit/README.txt @@ -0,0 +1,15 @@ +NOTE +---- + +The C++ code here + +** only depends on ** + +and nothing else. + +DO NOT ADD CODE WITH OTHER EXTERNAL DEPENDENCIES TO THIS DIRECTORY. + +Read on: + +pybind11_conduit_v1.h — Type-safe interoperability between different + independent Python/C++ bindings systems. diff --git a/wrap/pybind11/include/pybind11/conduit/pybind11_conduit_v1.h b/wrap/pybind11/include/pybind11/conduit/pybind11_conduit_v1.h new file mode 100644 index 000000000..e3a453453 --- /dev/null +++ b/wrap/pybind11/include/pybind11/conduit/pybind11_conduit_v1.h @@ -0,0 +1,111 @@ +// Copyright (c) 2024 The pybind Community. + +/* The pybind11_conduit_v1 feature enables type-safe interoperability between + +* different independent Python/C++ bindings systems, + +* including pybind11 versions with different PYBIND11_INTERNALS_VERSION's. + +The naming of the feature is a bit misleading: + +* The feature is in no way tied to pybind11 internals. + +* It just happens to originate from pybind11 and currently still lives there. + +* The only external dependency is . + +The implementation is a VERY light-weight dependency. It is designed to be +compatible with any ISO C++11 (or higher) compiler, and does NOT require +C++ Exception Handling to be enabled. + +Please see https://github.com/pybind/pybind11/pull/5296 for more background. + +The implementation involves a + +def _pybind11_conduit_v1_( + self, + pybind11_platform_abi_id: bytes, + cpp_type_info_capsule: capsule, + pointer_kind: bytes) -> capsule + +method that is meant to be added to Python objects wrapping C++ objects +(e.g. pybind11::class_-wrapped types). + +The design of the _pybind11_conduit_v1_ feature provides two layers of +protection against C++ ABI mismatches: + +* The first and most important layer is that the pybind11_platform_abi_id's + must match between extensions. — This will never be perfect, but is the same + pragmatic approach used in pybind11 since 2017 + (https://github.com/pybind/pybind11/commit/96997a4b9d4ec3d389a570604394af5d5eee2557, + PYBIND11_INTERNALS_ID). + +* The second layer is that the typeid(std::type_info).name()'s must match + between extensions. + +The implementation below (which is shorter than this comment!), serves as a +battle-tested specification. The main API is this one function: + +auto *cpp_pointer = pybind11_conduit_v1::get_type_pointer_ephemeral(py_obj); + +It is meant to be a minimalistic reference implementation, intentionally +without comprehensive error reporting. It is expected that major bindings +systems will roll their own, compatible implementations, potentially with +system-specific error reporting. The essential specifications all bindings +systems need to agree on are merely: + +* PYBIND11_PLATFORM_ABI_ID (const char* literal). + +* The cpp_type_info capsule (see below: a void *ptr and a const char *name). + +* The cpp_conduit capsule (see below: a void *ptr and a const char *name). + +* "raw_pointer_ephemeral" means: the lifetime of the pointer is the lifetime + of the py_obj. + +*/ + +// THIS MUST STAY AT THE TOP! +#include "pybind11_platform_abi_id.h" + +#include +#include + +namespace pybind11_conduit_v1 { + +inline void *get_raw_pointer_ephemeral(PyObject *py_obj, const std::type_info *cpp_type_info) { + PyObject *cpp_type_info_capsule + = PyCapsule_New(const_cast(static_cast(cpp_type_info)), + typeid(std::type_info).name(), + nullptr); + if (cpp_type_info_capsule == nullptr) { + return nullptr; + } + PyObject *cpp_conduit = PyObject_CallMethod(py_obj, + "_pybind11_conduit_v1_", + "yOy", + PYBIND11_PLATFORM_ABI_ID, + cpp_type_info_capsule, + "raw_pointer_ephemeral"); + Py_DECREF(cpp_type_info_capsule); + if (cpp_conduit == nullptr) { + return nullptr; + } + void *raw_ptr = PyCapsule_GetPointer(cpp_conduit, cpp_type_info->name()); + Py_DECREF(cpp_conduit); + if (PyErr_Occurred()) { + return nullptr; + } + return raw_ptr; +} + +template +T *get_type_pointer_ephemeral(PyObject *py_obj) { + void *raw_ptr = get_raw_pointer_ephemeral(py_obj, &typeid(T)); + if (raw_ptr == nullptr) { + return nullptr; + } + return static_cast(raw_ptr); +} + +} // namespace pybind11_conduit_v1 diff --git a/wrap/pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h b/wrap/pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h new file mode 100644 index 000000000..d21fdc56d --- /dev/null +++ b/wrap/pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h @@ -0,0 +1,87 @@ +#pragma once + +// Copyright (c) 2024 The pybind Community. + +// To maximize reusability: +// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING. + +#include "wrap_include_python_h.h" + +// Implementation details. DO NOT USE ELSEWHERE. (Unfortunately we cannot #undef them.) +// This is duplicated here to maximize portability. +#define PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x) #x +#define PYBIND11_PLATFORM_ABI_ID_TOSTRING(x) PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x) + +#ifdef PYBIND11_COMPILER_TYPE +// // To maintain backward compatibility (see PR #5439). +# define PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE "" +#else +# define PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE "_" +# if defined(__MINGW32__) +# define PYBIND11_COMPILER_TYPE "mingw" +# elif defined(__CYGWIN__) +# define PYBIND11_COMPILER_TYPE "gcc_cygwin" +# elif defined(_MSC_VER) +# define PYBIND11_COMPILER_TYPE "msvc" +# elif defined(__clang__) || defined(__GNUC__) +# define PYBIND11_COMPILER_TYPE "system" // Assumed compatible with system compiler. +# else +# error "Unknown PYBIND11_COMPILER_TYPE: PLEASE REVISE THIS CODE." +# endif +#endif + +// PR #5439 made this macro obsolete. However, there are many manipulations of this macro in the +// wild. Therefore, to maintain backward compatibility, it is kept around. +#ifndef PYBIND11_STDLIB +# define PYBIND11_STDLIB "" +#endif + +#ifndef PYBIND11_BUILD_ABI +# if defined(_MSC_VER) // See PR #4953. +# if defined(_MT) && defined(_DLL) // Corresponding to CL command line options /MD or /MDd. +# if (_MSC_VER) / 100 == 19 +# define PYBIND11_BUILD_ABI "_md_mscver19" +# else +# error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE." +# endif +# elif defined(_MT) // Corresponding to CL command line options /MT or /MTd. +# define PYBIND11_BUILD_ABI "_mt_mscver" PYBIND11_PLATFORM_ABI_ID_TOSTRING(_MSC_VER) +# else +# if (_MSC_VER) / 100 == 19 +# define PYBIND11_BUILD_ABI "_none_mscver19" +# else +# error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE." +# endif +# endif +# elif defined(_LIBCPP_ABI_VERSION) // https://libcxx.llvm.org/DesignDocs/ABIVersioning.html +# define PYBIND11_BUILD_ABI \ + "_libcpp_abi" PYBIND11_PLATFORM_ABI_ID_TOSTRING(_LIBCPP_ABI_VERSION) +# elif defined(_GLIBCXX_USE_CXX11_ABI) // See PR #5439. +# if defined(__NVCOMPILER) +// // Assume that NVHPC is in the 1xxx ABI family. +// // THIS ASSUMPTION IS NOT FUTURE PROOF but apparently the best we can do. +// // Please let us know if there is a way to validate the assumption here. +# elif !defined(__GXX_ABI_VERSION) +# error \ + "Unknown platform or compiler (_GLIBCXX_USE_CXX11_ABI): PLEASE REVISE THIS CODE." +# endif +# if defined(__GXX_ABI_VERSION) && __GXX_ABI_VERSION < 1002 || __GXX_ABI_VERSION >= 2000 +# error "Unknown platform or compiler (__GXX_ABI_VERSION): PLEASE REVISE THIS CODE." +# endif +# define PYBIND11_BUILD_ABI \ + "_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_" PYBIND11_PLATFORM_ABI_ID_TOSTRING( \ + _GLIBCXX_USE_CXX11_ABI) +# else +# error "Unknown platform or compiler: PLEASE REVISE THIS CODE." +# endif +#endif + +// On MSVC, debug and release builds are not ABI-compatible! +#if defined(_MSC_VER) && defined(_DEBUG) +# define PYBIND11_BUILD_TYPE "_debug" +#else +# define PYBIND11_BUILD_TYPE "" +#endif + +#define PYBIND11_PLATFORM_ABI_ID \ + PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE diff --git a/wrap/pybind11/include/pybind11/conduit/wrap_include_python_h.h b/wrap/pybind11/include/pybind11/conduit/wrap_include_python_h.h new file mode 100644 index 000000000..316d1afc8 --- /dev/null +++ b/wrap/pybind11/include/pybind11/conduit/wrap_include_python_h.h @@ -0,0 +1,72 @@ +#pragma once + +// Copyright (c) 2024 The pybind Community. + +// STRONG REQUIREMENT: +// This header is a wrapper around `#include `, therefore it +// MUST BE INCLUDED BEFORE ANY STANDARD HEADERS are included. +// See also: +// https://docs.python.org/3/c-api/intro.html#include-files +// Quoting from there: +// Note: Since Python may define some pre-processor definitions which affect +// the standard headers on some systems, you must include Python.h before +// any standard headers are included. + +// To maximize reusability: +// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING. + +// Disable linking to pythonX_d.lib on Windows in debug mode. +#if defined(_MSC_VER) && defined(_DEBUG) && !defined(Py_DEBUG) +// Workaround for a VS 2022 issue. +// See https://github.com/pybind/pybind11/pull/3497 for full context. +// NOTE: This workaround knowingly violates the Python.h include order +// requirement (see above). +# include +# if _MSVC_STL_VERSION >= 143 +# include +# endif +# define PYBIND11_DEBUG_MARKER +# undef _DEBUG +#endif + +// Don't let Python.h #define (v)snprintf as macro because they are implemented +// properly in Visual Studio since 2015. +#if defined(_MSC_VER) +# define HAVE_SNPRINTF 1 +#endif + +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4505) +// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed +#endif + +#include +#include +#include + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif + +#if defined(PYBIND11_DEBUG_MARKER) +# define _DEBUG +# undef PYBIND11_DEBUG_MARKER +#endif + +// Python #defines overrides on all sorts of core functions, which +// tends to wreak havok in C++ codebases that expect these to work +// like regular functions (potentially with several overloads). +#if defined(isalnum) +# undef isalnum +# undef isalpha +# undef islower +# undef isspace +# undef isupper +# undef tolower +# undef toupper +#endif + +#if defined(copysign) +# undef copysign +#endif diff --git a/wrap/pybind11/include/pybind11/detail/class.h b/wrap/pybind11/include/pybind11/detail/class.h index d30621c88..08e23afb5 100644 --- a/wrap/pybind11/include/pybind11/detail/class.h +++ b/wrap/pybind11/include/pybind11/detail/class.h @@ -9,8 +9,10 @@ #pragma once -#include "../attr.h" -#include "../options.h" +#include +#include + +#include "exception_translation.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -310,7 +312,31 @@ inline void traverse_offset_bases(void *valueptr, } } +#ifdef Py_GIL_DISABLED +inline void enable_try_inc_ref(PyObject *obj) { + // TODO: Replace with PyUnstable_Object_EnableTryIncRef when available. + // See https://github.com/python/cpython/issues/128844 + if (_Py_IsImmortal(obj)) { + return; + } + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); + if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { + // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. + return; + } + if (_Py_atomic_compare_exchange_ssize( + &obj->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { + return; + } + } +} +#endif + inline bool register_instance_impl(void *ptr, instance *self) { +#ifdef Py_GIL_DISABLED + enable_try_inc_ref(reinterpret_cast(self)); +#endif with_instance_map(ptr, [&](instance_map &instances) { instances.emplace(ptr, self); }); return true; // unused, but gives the same signature as the deregister func } @@ -431,6 +457,8 @@ inline void clear_instance(PyObject *self) { if (instance->owned || v_h.holder_constructed()) { v_h.type->dealloc(v_h); } + } else if (v_h.holder_constructed()) { + v_h.type->dealloc(v_h); // Disowned instance. } } // Deallocate the value/holder layout internals: @@ -466,19 +494,9 @@ extern "C" inline void pybind11_object_dealloc(PyObject *self) { type->tp_free(self); -#if PY_VERSION_HEX < 0x03080000 - // `type->tp_dealloc != pybind11_object_dealloc` means that we're being called - // as part of a derived type's dealloc, in which case we're not allowed to decref - // the type here. For cross-module compatibility, we shouldn't compare directly - // with `pybind11_object_dealloc`, but with the common one stashed in internals. - auto pybind11_object_type = (PyTypeObject *) get_internals().instance_base; - if (type->tp_dealloc == pybind11_object_type->tp_dealloc) - Py_DECREF(type); -#else // This was not needed before Python 3.8 (Python issue 35810) // https://github.com/pybind/pybind11/issues/1946 Py_DECREF(type); -#endif } std::string error_string(); @@ -558,7 +576,7 @@ extern "C" inline int pybind11_clear(PyObject *self) { inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; -#if PY_VERSION_HEX < 0x030B0000 +#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it #else @@ -591,31 +609,85 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla return -1; } std::memset(view, 0, sizeof(Py_buffer)); - buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + std::unique_ptr info = nullptr; + try { + info.reset(tinfo->get_buffer(obj, tinfo->get_buffer_data)); + } catch (...) { + try_translate_exceptions(); + raise_from(PyExc_BufferError, "Error getting buffer"); + return -1; + } + if (info == nullptr) { + pybind11_fail("FATAL UNEXPECTED SITUATION: tinfo->get_buffer() returned nullptr."); + } + if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { - delete info; // view->obj = nullptr; // Was just memset to 0, so not necessary set_error(PyExc_BufferError, "Writable buffer requested for readonly storage"); return -1; } - view->obj = obj; - view->ndim = 1; - view->internal = info; - view->buf = info->ptr; + + // Fill in all the information, and then downgrade as requested by the caller, or raise an + // error if that's not possible. view->itemsize = info->itemsize; view->len = view->itemsize; for (auto s : info->shape) { view->len *= s; } + view->ndim = static_cast(info->ndim); + view->shape = info->shape.data(); + view->strides = info->strides.data(); view->readonly = static_cast(info->readonly); if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { view->format = const_cast(info->format.c_str()); } - if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { - view->ndim = (int) info->ndim; - view->strides = info->strides.data(); - view->shape = info->shape.data(); + + // Note, all contiguity flags imply PyBUF_STRIDES and lower. + if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'C') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "C-contiguous buffer requested for discontiguous storage"); + return -1; + } + } else if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'F') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "Fortran-contiguous buffer requested for discontiguous storage"); + return -1; + } + } else if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'A') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, "Contiguous buffer requested for discontiguous storage"); + return -1; + } + + } else if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES) { + // If no strides are requested, the buffer must be C-contiguous. + // https://docs.python.org/3/c-api/buffer.html#contiguity-requests + if (PyBuffer_IsContiguous(view, 'C') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "C-contiguous buffer requested for discontiguous storage"); + return -1; + } + + view->strides = nullptr; + + // Since this is a contiguous buffer, it can also pretend to be 1D. + if ((flags & PyBUF_ND) != PyBUF_ND) { + view->shape = nullptr; + view->ndim = 0; + } } + + // Set these after all checks so they don't leak out into the caller, and can be automatically + // cleaned up on error. + view->buf = info->ptr; + view->internal = info.release(); + view->obj = obj; Py_INCREF(view->obj); return 0; } diff --git a/wrap/pybind11/include/pybind11/detail/common.h b/wrap/pybind11/include/pybind11/detail/common.h index e37152a9a..5e225f8c1 100644 --- a/wrap/pybind11/include/pybind11/detail/common.h +++ b/wrap/pybind11/include/pybind11/detail/common.h @@ -9,13 +9,18 @@ #pragma once -#define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 13 -#define PYBIND11_VERSION_PATCH 1 +#include +#if PY_VERSION_HEX < 0x03080000 +# error "PYTHON < 3.8 IS UNSUPPORTED. pybind11 v2.13 was the last to support Python 3.7." +#endif + +#define PYBIND11_VERSION_MAJOR 3 +#define PYBIND11_VERSION_MINOR 0 +#define PYBIND11_VERSION_PATCH 0.dev1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020D0100 +#define PYBIND11_VERSION_HEX 0x030000D1 // Define some generic pybind11 helper macros for warning management. // @@ -41,7 +46,7 @@ # define PYBIND11_COMPILER_CLANG # define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) # define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) -# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic pop) #elif defined(__GNUC__) # define PYBIND11_COMPILER_GCC # define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) @@ -164,14 +169,6 @@ # endif #endif -#if !defined(PYBIND11_EXPORT_EXCEPTION) -# if defined(__apple_build_version__) -# define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT -# else -# define PYBIND11_EXPORT_EXCEPTION -# endif -#endif - // For CUDA, GCC7, GCC8: // PYBIND11_NOINLINE_FORCED is incompatible with `-Wattributes -Werror`. // When defining PYBIND11_NOINLINE_FORCED, it is best to also use `-Wno-attributes`. @@ -212,31 +209,6 @@ # define PYBIND11_MAYBE_UNUSED __attribute__((__unused__)) #endif -/* Don't let Python.h #define (v)snprintf as macro because they are implemented - properly in Visual Studio since 2015. */ -#if defined(_MSC_VER) -# define HAVE_SNPRINTF 1 -#endif - -/// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode -#if defined(_MSC_VER) -PYBIND11_WARNING_PUSH -PYBIND11_WARNING_DISABLE_MSVC(4505) -// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -# if defined(_DEBUG) && !defined(Py_DEBUG) -// Workaround for a VS 2022 issue. -// NOTE: This workaround knowingly violates the Python.h include order requirement: -// https://docs.python.org/3/c-api/intro.html#include-files -// See https://github.com/pybind/pybind11/pull/3497 for full context. -# include -# if _MSVC_STL_VERSION >= 143 -# include -# endif -# define PYBIND11_DEBUG_MARKER -# undef _DEBUG -# endif -#endif - // https://en.cppreference.com/w/c/chrono/localtime #if defined(__STDC_LIB_EXT1__) && !defined(__STDC_WANT_LIB_EXT1__) # define __STDC_WANT_LIB_EXT1__ @@ -271,46 +243,14 @@ PYBIND11_WARNING_DISABLE_MSVC(4505) # endif #endif -#include -#if PY_VERSION_HEX < 0x03070000 -# error "PYTHON < 3.7 IS UNSUPPORTED. pybind11 v2.12 was the last to support Python 3.6." -#endif -#include -#include - -/* Python #defines overrides on all sorts of core functions, which - tends to weak havok in C++ codebases that expect these to work - like regular functions (potentially with several overloads) */ -#if defined(isalnum) -# undef isalnum -# undef isalpha -# undef islower -# undef isspace -# undef isupper -# undef tolower -# undef toupper -#endif - -#if defined(copysign) -# undef copysign -#endif - #if defined(PYBIND11_NUMPY_1_ONLY) # define PYBIND11_INTERNAL_NUMPY_1_ONLY_DETECTED #endif -#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +#if (defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif -#if defined(_MSC_VER) -# if defined(PYBIND11_DEBUG_MARKER) -# define _DEBUG -# undef PYBIND11_DEBUG_MARKER -# endif -PYBIND11_WARNING_POP -#endif - #include #include #include @@ -329,6 +269,17 @@ PYBIND11_WARNING_POP # endif #endif +// For libc++, the exceptions should be exported, +// otherwise, the exception translation would be incorrect. +// IMPORTANT: This code block must stay BELOW the #include above (see PR #5390). +#if !defined(PYBIND11_EXPORT_EXCEPTION) +# if defined(_LIBCPP_EXCEPTION) +# define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION +# endif +#endif + // Must be after including or one of the other headers specified by the standard #if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L # define PYBIND11_HAS_U8STRING @@ -387,6 +338,20 @@ PYBIND11_WARNING_POP #define PYBIND11_CONCAT(first, second) first##second #define PYBIND11_ENSURE_INTERNALS_READY pybind11::detail::get_internals(); +#if !defined(GRAALVM_PYTHON) +# define PYBIND11_PYCFUNCTION_GET_DOC(func) ((func)->m_ml->ml_doc) +# define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ + do { \ + (func)->m_ml->ml_doc = (doc); \ + } while (0) +#else +# define PYBIND11_PYCFUNCTION_GET_DOC(func) (GraalPyCFunction_GetDoc((PyObject *) (func))) +# define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ + do { \ + GraalPyCFunction_SetDoc((PyObject *) (func), (doc)); \ + } while (0) +#endif + #define PYBIND11_CHECK_PYTHON_VERSION \ { \ const char *compiled_ver \ @@ -462,7 +427,25 @@ PYBIND11_WARNING_POP return "Hello, World!"; }); } + + The third macro argument is optional (available since 2.13.0), and can be used to + mark the extension module as safe to run without the GIL under a free-threaded CPython + interpreter. Passing this argument has no effect on other interpreters. + + .. code-block:: cpp + + PYBIND11_MODULE(example, m, py::mod_gil_not_used()) { + m.doc() = "pybind11 example module safe to run without the GIL"; + + // Add bindings here + m.def("foo", []() { + return "Hello, Free-threaded World!"; + }); + } + \endrst */ +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") #define PYBIND11_MODULE(name, variable, ...) \ static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name) \ PYBIND11_MAYBE_UNUSED; \ @@ -483,6 +466,7 @@ PYBIND11_WARNING_POP PYBIND11_CATCH_INIT_EXCEPTIONS \ } \ void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ & (variable)) +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -538,7 +522,7 @@ enum class return_value_policy : uint8_t { object without taking ownership similar to the above return_value_policy::reference policy. In contrast to that policy, the function or property's implicit this argument (called the parent) is - considered to be the the owner of the return value (the child). + considered to be the owner of the return value (the child). pybind11 then couples the lifetime of the parent to the child via a reference relationship that ensures that the parent cannot be garbage collected while Python is still using the child. More advanced @@ -621,6 +605,8 @@ struct instance { bool simple_instance_registered : 1; /// If true, get_internals().patients has an entry for this object bool has_patients : 1; + /// If true, this Python object needs to be kept alive for the lifetime of the C++ value. + bool is_alias : 1; /// Initializes all of the above type/values/holders data (but not the instance values /// themselves) @@ -643,6 +629,14 @@ struct instance { static_assert(std::is_standard_layout::value, "Internal error: `pybind11::detail::instance` is not standard layout!"); +// Some older compilers (e.g. gcc 9.4.0) require +// static_assert(always_false::value, "..."); +// instead of +// static_assert(false, "..."); +// to trigger the static_assert() in a template only if it is actually instantiated. +template +struct always_false : std::false_type {}; + /// from __cpp_future__ import (convenient aliases from C++14/17) #if defined(PYBIND11_CPP14) using std::conditional_t; @@ -1109,14 +1103,14 @@ struct overload_cast_impl { } template - constexpr auto operator()(Return (Class::*pmf)(Args...), - std::false_type = {}) const noexcept -> decltype(pmf) { + constexpr auto operator()(Return (Class::*pmf)(Args...), std::false_type = {}) const noexcept + -> decltype(pmf) { return pmf; } template - constexpr auto operator()(Return (Class::*pmf)(Args...) const, - std::true_type) const noexcept -> decltype(pmf) { + constexpr auto operator()(Return (Class::*pmf)(Args...) const, std::true_type) const noexcept + -> decltype(pmf) { return pmf; } }; @@ -1264,5 +1258,10 @@ constexpr # define PYBIND11_DETAILED_ERROR_MESSAGES #endif +// CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. +#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION) +# define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET +#endif + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/detail/cpp_conduit.h b/wrap/pybind11/include/pybind11/detail/cpp_conduit.h new file mode 100644 index 000000000..b66c2d39c --- /dev/null +++ b/wrap/pybind11/include/pybind11/detail/cpp_conduit.h @@ -0,0 +1,77 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include + +#include "common.h" +#include "internals.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Forward declaration needed here: Refactoring opportunity. +extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *); + +inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { +#if defined(PYPY_VERSION) + auto &internals = get_internals(); + return bool(internals.registered_types_py.find(type_obj) + != internals.registered_types_py.end()); +#else + return bool(type_obj->tp_new == pybind11_object_new); +#endif +} + +inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { + PyObject *descr = _PyType_Lookup(type_obj, attr_name); + return bool((descr != nullptr) && PyInstanceMethod_Check(descr)); +} + +inline object try_get_cpp_conduit_method(PyObject *obj) { + if (PyType_Check(obj)) { + return object(); + } + PyTypeObject *type_obj = Py_TYPE(obj); + str attr_name("_pybind11_conduit_v1_"); + bool assumed_to_be_callable = false; + if (type_is_managed_by_our_internals(type_obj)) { + if (!is_instance_method_of_type(type_obj, attr_name.ptr())) { + return object(); + } + assumed_to_be_callable = true; + } + PyObject *method = PyObject_GetAttr(obj, attr_name.ptr()); + if (method == nullptr) { + PyErr_Clear(); + return object(); + } + if (!assumed_to_be_callable && PyCallable_Check(method) == 0) { + Py_DECREF(method); + return object(); + } + return reinterpret_steal(method); +} + +inline void *try_raw_pointer_ephemeral_from_cpp_conduit(handle src, + const std::type_info *cpp_type_info) { + object method = try_get_cpp_conduit_method(src.ptr()); + if (method) { + capsule cpp_type_info_capsule(const_cast(static_cast(cpp_type_info)), + typeid(std::type_info).name()); + object cpp_conduit = method(bytes(PYBIND11_PLATFORM_ABI_ID), + cpp_type_info_capsule, + bytes("raw_pointer_ephemeral")); + if (isinstance(cpp_conduit)) { + return reinterpret_borrow(cpp_conduit).get_pointer(); + } + } + return nullptr; +} + +#define PYBIND11_HAS_CPP_CONDUIT 1 + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/detail/descr.h b/wrap/pybind11/include/pybind11/detail/descr.h index 7d546311e..a5f17f869 100644 --- a/wrap/pybind11/include/pybind11/detail/descr.h +++ b/wrap/pybind11/include/pybind11/detail/descr.h @@ -99,6 +99,13 @@ constexpr descr<1, Type> const_name() { return {'%'}; } +// Use a different name based on whether the parameter is used as input or output +template +constexpr descr io_name(char const (&text1)[N1], char const (&text2)[N2]) { + return const_name("@") + const_name(text1) + const_name("@") + const_name(text2) + + const_name("@"); +} + // If "_" is defined as a macro, py::detail::_ cannot be provided. // It is therefore best to use py::detail::const_name universally. // This block is for backward compatibility only. @@ -156,9 +163,8 @@ constexpr auto concat(const descr &d, const Args &...args) { } #else template -constexpr auto concat(const descr &d, - const Args &...args) -> decltype(std::declval>() - + concat(args...)) { +constexpr auto concat(const descr &d, const Args &...args) + -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } #endif @@ -168,5 +174,15 @@ constexpr descr type_descr(const descr &descr) { return const_name("{") + descr + const_name("}"); } +template +constexpr descr arg_descr(const descr &descr) { + return const_name("@^") + descr + const_name("@!"); +} + +template +constexpr descr return_descr(const descr &descr) { + return const_name("@$") + descr + const_name("@!"); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h b/wrap/pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h new file mode 100644 index 000000000..7c00fe98c --- /dev/null +++ b/wrap/pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h @@ -0,0 +1,39 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct dynamic_raw_ptr_cast_is_possible : std::false_type {}; + +template +struct dynamic_raw_ptr_cast_is_possible< + To, + From, + detail::enable_if_t::value && std::is_polymorphic::value>> + : std::true_type {}; + +template ::value, int> = 0> +To *dynamic_raw_ptr_cast_if_possible(From * /*ptr*/) { + return nullptr; +} + +template ::value, int> = 0> +To *dynamic_raw_ptr_cast_if_possible(From *ptr) { + return dynamic_cast(ptr); +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/detail/exception_translation.h b/wrap/pybind11/include/pybind11/detail/exception_translation.h new file mode 100644 index 000000000..22ae8a1c9 --- /dev/null +++ b/wrap/pybind11/include/pybind11/detail/exception_translation.h @@ -0,0 +1,71 @@ +/* + pybind11/detail/exception_translation.h: means to translate C++ exceptions to Python exceptions + + Copyright (c) 2024 The Pybind Development Team. + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "common.h" +#include "internals.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Apply all the extensions translators from a list +// Return true if one of the translators completed without raising an exception +// itself. Return of false indicates that if there are other translators +// available, they should be tried. +inline bool apply_exception_translators(std::forward_list &translators) { + auto last_exception = std::current_exception(); + + for (auto &translator : translators) { + try { + translator(last_exception); + return true; + } catch (...) { + last_exception = std::current_exception(); + } + } + return false; +} + +inline void try_translate_exceptions() { + /* When an exception is caught, give each registered exception + translator a chance to translate it to a Python exception. First + all module-local translators will be tried in reverse order of + registration. If none of the module-locale translators handle + the exception (or there are no module-locale translators) then + the global translators will be tried, also in reverse order of + registration. + + A translator may choose to do one of the following: + + - catch the exception and call py::set_error() + to set a standard (or custom) Python exception, or + - do nothing and let the exception fall through to the next translator, or + - delegate translation to the next translator by throwing a new type of exception. + */ + + bool handled = with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + if (detail::apply_exception_translators(local_exception_translators)) { + return true; + } + if (detail::apply_exception_translators(exception_translators)) { + return true; + } + return false; + }); + + if (!handled) { + set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); + } +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/detail/init.h b/wrap/pybind11/include/pybind11/detail/init.h index 4509bd131..ed95afe58 100644 --- a/wrap/pybind11/include/pybind11/detail/init.h +++ b/wrap/pybind11/include/pybind11/detail/init.h @@ -10,6 +10,7 @@ #pragma once #include "class.h" +#include "using_smart_holder.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -128,11 +129,13 @@ void construct(value_and_holder &v_h, Cpp *ptr, bool need_alias) { // the holder and destruction happens when we leave the C++ scope, and the holder // class gets to handle the destruction however it likes. v_h.value_ptr() = ptr; - v_h.set_instance_registered(true); // To prevent init_instance from registering it - v_h.type->init_instance(v_h.inst, nullptr); // Set up the holder + v_h.set_instance_registered(true); // Trick to prevent init_instance from registering it + // DANGER ZONE BEGIN: exceptions will leave v_h in an invalid state. + v_h.type->init_instance(v_h.inst, nullptr); // Set up the holder Holder temp_holder(std::move(v_h.holder>())); // Steal the holder v_h.type->dealloc(v_h); // Destroys the moved-out holder remains, resets value ptr to null v_h.set_instance_registered(false); + // DANGER ZONE END. construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(*ptr)); } else { @@ -153,7 +156,7 @@ void construct(value_and_holder &v_h, Alias *alias_ptr, bool) { // holder. This also handles types like std::shared_ptr and std::unique_ptr where T is a // derived type (through those holder's implicit conversion from derived class holder // constructors). -template +template >::value, int> = 0> void construct(value_and_holder &v_h, Holder holder, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); auto *ptr = holder_helper>::get(holder); @@ -195,6 +198,74 @@ void construct(value_and_holder &v_h, Alias &&result, bool) { v_h.value_ptr() = new Alias(std::move(result)); } +template +smart_holder init_smart_holder_from_unique_ptr(std::unique_ptr &&unq_ptr, + bool void_cast_raw_ptr) { + void *void_ptr = void_cast_raw_ptr ? static_cast(unq_ptr.get()) : nullptr; + return smart_holder::from_unique_ptr(std::move(unq_ptr), void_ptr); +} + +template >, + detail::enable_if_t>::value, int> = 0> +void construct(value_and_holder &v_h, std::unique_ptr, D> &&unq_ptr, bool need_alias) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); + auto *ptr = unq_ptr.get(); + no_nullptr(ptr); + if (Class::has_alias && need_alias && !is_alias(ptr)) { + throw type_error("pybind11::init(): construction failed: returned std::unique_ptr pointee " + "is not an alias instance"); + } + // Here and below: if the new object is a trampoline, the shared_from_this mechanism needs + // to be prevented from accessing the smart_holder vptr, because it does not keep the + // trampoline Python object alive. For types that don't inherit from enable_shared_from_this + // it does not matter if void_cast_raw_ptr is true or false, therefore it's not necessary + // to also inspect the type. + auto smhldr = init_smart_holder_from_unique_ptr( + std::move(unq_ptr), /*void_cast_raw_ptr*/ Class::has_alias && is_alias(ptr)); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >, + detail::enable_if_t>::value, int> = 0> +void construct(value_and_holder &v_h, + std::unique_ptr, D> &&unq_ptr, + bool /*need_alias*/) { + auto *ptr = unq_ptr.get(); + no_nullptr(ptr); + auto smhldr + = init_smart_holder_from_unique_ptr(std::move(unq_ptr), /*void_cast_raw_ptr*/ true); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >::value, int> = 0> +void construct(value_and_holder &v_h, std::shared_ptr> &&shd_ptr, bool need_alias) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); + auto *ptr = shd_ptr.get(); + no_nullptr(ptr); + if (Class::has_alias && need_alias && !is_alias(ptr)) { + throw type_error("pybind11::init(): construction failed: returned std::shared_ptr pointee " + "is not an alias instance"); + } + auto smhldr = smart_holder::from_shared_ptr(shd_ptr); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >::value, int> = 0> +void construct(value_and_holder &v_h, + std::shared_ptr> &&shd_ptr, + bool /*need_alias*/) { + auto *ptr = shd_ptr.get(); + no_nullptr(ptr); + auto smhldr = smart_holder::from_shared_ptr(shd_ptr); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + // Implementing class for py::init<...>() template struct constructor { @@ -408,7 +479,7 @@ struct pickle_factory { template void execute(Class &cl, const Extra &...extra) && { - cl.def("__getstate__", std::move(get)); + cl.def("__getstate__", std::move(get), pos_only()); #if defined(PYBIND11_CPP14) cl.def( diff --git a/wrap/pybind11/include/pybind11/detail/internals.h b/wrap/pybind11/include/pybind11/detail/internals.h index e61c1687f..841c8fe15 100644 --- a/wrap/pybind11/include/pybind11/detail/internals.h +++ b/wrap/pybind11/include/pybind11/detail/internals.h @@ -12,10 +12,11 @@ #include "common.h" #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) -# include "../gil.h" +# include #endif -#include "../pytypes.h" +#include +#include #include #include @@ -36,18 +37,12 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# if PY_VERSION_HEX >= 0x030C0000 || defined(_MSC_VER) -// Version bump for Python 3.12+, before first 3.12 beta release. -// Version bump for MSVC piggy-backed on PR #4779. See comments there. -# define PYBIND11_INTERNALS_VERSION 5 -# else -# define PYBIND11_INTERNALS_VERSION 4 -# endif +# define PYBIND11_INTERNALS_VERSION 7 #endif -// This requirement is mainly to reduce the support burden (see PR #4570). -static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5, - "pybind11 ABI version 5 is the minimum for Python 3.12+"); +#if PYBIND11_INTERNALS_VERSION < 7 +# error "PYBIND11_INTERNALS_VERSION 7 is the minimum for all platforms for pybind11v3." +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -66,40 +61,29 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // Thread Specific Storage (TSS) API. // Avoid unnecessary allocation of `Py_tss_t`, since we cannot use // `Py_LIMITED_API` anyway. -#if PYBIND11_INTERNALS_VERSION > 4 -# define PYBIND11_TLS_KEY_REF Py_tss_t & -# if defined(__clang__) -# define PYBIND11_TLS_KEY_INIT(var) \ - _Pragma("clang diagnostic push") /**/ \ - _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ - Py_tss_t var \ - = Py_tss_NEEDS_INIT; \ - _Pragma("clang diagnostic pop") -# elif defined(__GNUC__) && !defined(__INTEL_COMPILER) -# define PYBIND11_TLS_KEY_INIT(var) \ - _Pragma("GCC diagnostic push") /**/ \ - _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ - Py_tss_t var \ - = Py_tss_NEEDS_INIT; \ - _Pragma("GCC diagnostic pop") -# else -# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; -# endif -# define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) -# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) -# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) -# define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) +#define PYBIND11_TLS_KEY_REF Py_tss_t & +#if defined(__clang__) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("clang diagnostic push") /**/ \ + _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var \ + = Py_tss_NEEDS_INIT; \ + _Pragma("clang diagnostic pop") +#elif defined(__GNUC__) && !defined(__INTEL_COMPILER) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("GCC diagnostic push") /**/ \ + _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var \ + = Py_tss_NEEDS_INIT; \ + _Pragma("GCC diagnostic pop") #else -# define PYBIND11_TLS_KEY_REF Py_tss_t * -# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr; -# define PYBIND11_TLS_KEY_CREATE(var) \ - (((var) = PyThread_tss_alloc()) != nullptr && (PyThread_tss_create((var)) == 0)) -# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key)) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) -# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) -# define PYBIND11_TLS_FREE(key) PyThread_tss_free(key) +# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; #endif +#define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) +#define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) +#define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) +#define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) +#define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly // other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module @@ -107,8 +91,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ - || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) +#if !defined(_LIBCPP_VERSION) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash; using type_equal_to = std::equal_to; @@ -148,20 +131,36 @@ struct override_hash { using instance_map = std::unordered_multimap; +#ifdef Py_GIL_DISABLED +// Wrapper around PyMutex to provide BasicLockable semantics +class pymutex { + PyMutex mutex; + +public: + pymutex() : mutex({}) {} + void lock() { PyMutex_Lock(&mutex); } + void unlock() { PyMutex_Unlock(&mutex); } +}; + // Instance map shards are used to reduce mutex contention in free-threaded Python. struct instance_map_shard { - std::mutex mutex; instance_map registered_instances; + pymutex mutex; // alignas(64) would be better, but causes compile errors in macOS before 10.14 (see #5200) - char padding[64 - (sizeof(std::mutex) + sizeof(instance_map)) % 64]; + char padding[64 - (sizeof(instance_map) + sizeof(pymutex)) % 64]; }; +static_assert(sizeof(instance_map_shard) % 64 == 0, + "instance_map_shard size is not a multiple of 64 bytes"); +#endif + /// Internal data structure used to track registered instances and types. /// Whenever binary incompatible changes are made to this structure, /// `PYBIND11_INTERNALS_VERSION` must be incremented. struct internals { #ifdef Py_GIL_DISABLED - std::mutex mutex; + pymutex mutex; + pymutex exception_translator_mutex; #endif // std::type_index -> pybind11's type information type_map registered_types_cpp; @@ -180,35 +179,26 @@ struct internals { std::forward_list registered_exception_translators; std::unordered_map shared_data; // Custom data to be shared across // extensions -#if PYBIND11_INTERNALS_VERSION == 4 - std::vector unused_loader_patient_stack_remove_at_v5; -#endif - std::forward_list static_strings; // Stores the std::strings backing - // detail::c_str() + std::forward_list static_strings; // Stores the std::strings backing + // detail::c_str() PyTypeObject *static_property_type; PyTypeObject *default_metaclass; PyObject *instance_base; // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PYBIND11_TLS_KEY_INIT(tstate) -#if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) -#endif // PYBIND11_INTERNALS_VERSION > 4 // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; -#if PYBIND11_INTERNALS_VERSION > 4 // Note that we have to use a std::string to allocate memory to ensure a unique address // We want unique addresses since we use pointer equality to compare function records std::string function_record_capsule_name = internals_function_record_capsule_name; -#endif internals() = default; internals(const internals &other) = delete; internals &operator=(const internals &other) = delete; ~internals() { -#if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_FREE(loader_life_support_tls_key); -#endif // PYBIND11_INTERNALS_VERSION > 4 // This destructor is called *after* Py_Finalize() in finalize_interpreter(). // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is @@ -221,6 +211,17 @@ struct internals { } }; +// For backwards compatibility (i.e. #ifdef guards): +#define PYBIND11_HAS_INTERNALS_WITH_SMART_HOLDER_SUPPORT + +enum class holder_enum_t : uint8_t { + undefined, + std_unique_ptr, // Default, lacking interop with std::shared_ptr. + std_shared_ptr, // Lacking interop with std::unique_ptr. + smart_holder, // Full std::unique_ptr / std::shared_ptr interop. + custom_holder, +}; + /// Additional type information which does not fit into the PyTypeObject. /// Changes to this struct also require bumping `PYBIND11_INTERNALS_VERSION`. struct type_info { @@ -236,6 +237,7 @@ struct type_info { buffer_info *(*get_buffer)(PyObject *, void *) = nullptr; void *get_buffer_data = nullptr; void *(*module_local_load)(PyObject *, const type_info *) = nullptr; + holder_enum_t holder_enum_v = holder_enum_t::undefined; /* A simple type never occurs as a (direct or indirect) parent * of a class that makes use of multiple inheritance. * A type can be simple even if it has non-simple ancestors as long as it has no descendants. @@ -243,78 +245,17 @@ struct type_info { bool simple_type : 1; /* True if there is no multiple inheritance in this type's inheritance tree */ bool simple_ancestors : 1; - /* for base vs derived holder_type checks */ - bool default_holder : 1; /* true if this is a type registered with py::module_local */ bool module_local : 1; }; -/// On MSVC, debug and release builds are not ABI-compatible! -#if defined(_MSC_VER) && defined(_DEBUG) -# define PYBIND11_BUILD_TYPE "_debug" -#else -# define PYBIND11_BUILD_TYPE "" -#endif - -/// Let's assume that different compilers are ABI-incompatible. -/// A user can manually set this string if they know their -/// compiler is compatible. -#ifndef PYBIND11_COMPILER_TYPE -# if defined(_MSC_VER) -# define PYBIND11_COMPILER_TYPE "_msvc" -# elif defined(__INTEL_COMPILER) -# define PYBIND11_COMPILER_TYPE "_icc" -# elif defined(__clang__) -# define PYBIND11_COMPILER_TYPE "_clang" -# elif defined(__PGI) -# define PYBIND11_COMPILER_TYPE "_pgi" -# elif defined(__MINGW32__) -# define PYBIND11_COMPILER_TYPE "_mingw" -# elif defined(__CYGWIN__) -# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" -# elif defined(__GNUC__) -# define PYBIND11_COMPILER_TYPE "_gcc" -# else -# define PYBIND11_COMPILER_TYPE "_unknown" -# endif -#endif - -/// Also standard libs -#ifndef PYBIND11_STDLIB -# if defined(_LIBCPP_VERSION) -# define PYBIND11_STDLIB "_libcpp" -# elif defined(__GLIBCXX__) || defined(__GLIBCPP__) -# define PYBIND11_STDLIB "_libstdcpp" -# else -# define PYBIND11_STDLIB "" -# endif -#endif - -/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. -/// On MSVC, changes in _MSC_VER may indicate ABI incompatibility (#2898). -#ifndef PYBIND11_BUILD_ABI -# if defined(__GXX_ABI_VERSION) -# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) -# elif defined(_MSC_VER) -# define PYBIND11_BUILD_ABI "_mscver" PYBIND11_TOSTRING(_MSC_VER) -# else -# define PYBIND11_BUILD_ABI "" -# endif -#endif - -#ifndef PYBIND11_INTERNALS_KIND -# define PYBIND11_INTERNALS_KIND "" -#endif - #define PYBIND11_INTERNALS_ID \ "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB \ - PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" #define PYBIND11_MODULE_LOCAL_ID \ "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB \ - PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" /// Each module locally stores a pointer to the `internals` data. The data /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`. @@ -432,7 +373,7 @@ inline void translate_local_exception(std::exception_ptr p) { inline object get_python_state_dict() { object state_dict; -#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) state_dict = reinterpret_borrow(PyEval_GetBuiltins()); #else # if PY_VERSION_HEX < 0x03090000 @@ -530,15 +471,14 @@ PYBIND11_NOINLINE internals &get_internals() { } PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); -#if PYBIND11_INTERNALS_VERSION > 4 // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { pybind11_fail("get_internals: could not successfully initialize the " "loader_life_support TSS key!"); } -#endif + internals_ptr->istate = tstate->interp; - state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp); + state_dict[PYBIND11_INTERNALS_ID] = capsule(reinterpret_cast(internals_pp)); internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); @@ -566,40 +506,6 @@ PYBIND11_NOINLINE internals &get_internals() { struct local_internals { type_map registered_types_cpp; std::forward_list registered_exception_translators; -#if PYBIND11_INTERNALS_VERSION == 4 - - // For ABI compatibility, we can't store the loader_life_support TLS key in - // the `internals` struct directly. Instead, we store it in `shared_data` and - // cache a copy in `local_internals`. If we allocated a separate TLS key for - // each instance of `local_internals`, we could end up allocating hundreds of - // TLS keys if hundreds of different pybind11 modules are loaded (which is a - // plausible number). - PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) - - // Holds the shared TLS key for the loader_life_support stack. - struct shared_loader_life_support_data { - PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) - shared_loader_life_support_data() { - // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) - if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { - pybind11_fail("local_internals: could not successfully initialize the " - "loader_life_support TLS key!"); - } - } - // We can't help but leak the TLS key, because Python never unloads extension modules. - }; - - local_internals() { - auto &internals = get_internals(); - // Get or create the `loader_life_support_stack_key`. - auto &ptr = internals.shared_data["_life_support"]; - if (!ptr) { - ptr = new shared_loader_life_support_data; - } - loader_life_support_tls_key - = static_cast(ptr)->loader_life_support_tls_key; - } -#endif // PYBIND11_INTERNALS_VERSION == 4 }; /// Works like `get_internals`, but for things which are locally registered. @@ -614,7 +520,7 @@ inline local_internals &get_local_internals() { } #ifdef Py_GIL_DISABLED -# define PYBIND11_LOCK_INTERNALS(internals) std::unique_lock lock((internals).mutex) +# define PYBIND11_LOCK_INTERNALS(internals) std::unique_lock lock((internals).mutex) #else # define PYBIND11_LOCK_INTERNALS(internals) #endif @@ -626,6 +532,19 @@ inline auto with_internals(const F &cb) -> decltype(cb(get_internals())) { return cb(internals); } +template +inline auto with_exception_translators(const F &cb) + -> decltype(cb(get_internals().registered_exception_translators, + get_local_internals().registered_exception_translators)) { + auto &internals = get_internals(); +#ifdef Py_GIL_DISABLED + std::unique_lock lock((internals).exception_translator_mutex); +#endif + auto &local_internals = get_local_internals(); + return cb(internals.registered_exception_translators, + local_internals.registered_exception_translators); +} + inline std::uint64_t mix64(std::uint64_t z) { // David Stafford's variant 13 of the MurmurHash3 finalizer popularized // by the SplitMix PRNG. @@ -636,8 +555,8 @@ inline std::uint64_t mix64(std::uint64_t z) { } template -inline auto with_instance_map(const void *ptr, - const F &cb) -> decltype(cb(std::declval())) { +inline auto with_instance_map(const void *ptr, const F &cb) + -> decltype(cb(std::declval())) { auto &internals = get_internals(); #ifdef Py_GIL_DISABLED @@ -651,7 +570,7 @@ inline auto with_instance_map(const void *ptr, auto idx = static_cast(hash & internals.instance_shards_mask); auto &shard = internals.instance_shards[idx]; - std::unique_lock lock(shard.mutex); + std::unique_lock lock(shard.mutex); return cb(shard.registered_instances); #else (void) ptr; @@ -667,7 +586,7 @@ inline size_t num_registered_instances() { size_t count = 0; for (size_t i = 0; i <= internals.instance_shards_mask; ++i) { auto &shard = internals.instance_shards[i]; - std::unique_lock lock(shard.mutex); + std::unique_lock lock(shard.mutex); count += shard.registered_instances.size(); } return count; @@ -692,7 +611,8 @@ const char *c_str(Args &&...args) { } inline const char *get_function_record_capsule_name() { -#if PYBIND11_INTERNALS_VERSION > 4 + // On GraalPy, pointer equality of the names is currently not guaranteed +#if !defined(GRAALVM_PYTHON) return get_internals().function_record_capsule_name.c_str(); #else return nullptr; diff --git a/wrap/pybind11/include/pybind11/detail/struct_smart_holder.h b/wrap/pybind11/include/pybind11/detail/struct_smart_holder.h new file mode 100644 index 000000000..980fc3699 --- /dev/null +++ b/wrap/pybind11/include/pybind11/detail/struct_smart_holder.h @@ -0,0 +1,349 @@ +// Copyright (c) 2020-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/* Proof-of-Concept for smart pointer interoperability. + +High-level aspects: + +* Support all `unique_ptr`, `shared_ptr` interops that are feasible. + +* Cleanly and clearly report all interops that are infeasible. + +* Meant to fit into a `PyObject`, as a holder for C++ objects. + +* Support a system design that makes it impossible to trigger + C++ Undefined Behavior, especially from Python. + +* Support a system design with clean runtime inheritance casting. From this + it follows that the `smart_holder` needs to be type-erased (`void*`). + +* Handling of RTTI for the type-erased held pointer is NOT implemented here. + It is the responsibility of the caller to ensure that `static_cast` + is well-formed when calling `as_*` member functions. Inheritance casting + needs to be handled in a different layer (similar to the code organization + in boost/python/object/inheritance.hpp). + +Details: + +* The "root holder" chosen here is a `shared_ptr` (named `vptr` in this + implementation). This choice is practically inevitable because `shared_ptr` + has only very limited support for inspecting and accessing its deleter. + +* If created from a raw pointer, or a `unique_ptr` without a custom deleter, + `vptr` always uses a custom deleter, to support `unique_ptr`-like disowning. + The custom deleters could be extended to included life-time management for + external objects (e.g. `PyObject`). + +* If created from an external `shared_ptr`, or a `unique_ptr` with a custom + deleter, including life-time management for external objects is infeasible. + +* By choice, the smart_holder is movable but not copyable, to keep the design + simple, and to guard against accidental copying overhead. + +* The `void_cast_raw_ptr` option is needed to make the `smart_holder` `vptr` + member invisible to the `shared_from_this` mechanism, in case the lifetime + of a `PyObject` is tied to the pointee. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// pybindit = Python Bindings Innovation Track. +// Currently not in pybind11 namespace to signal that this POC does not depend +// on any existing pybind11 functionality. +namespace pybindit { +namespace memory { + +static constexpr bool type_has_shared_from_this(...) { return false; } + +template +static constexpr bool type_has_shared_from_this(const std::enable_shared_from_this *) { + return true; +} + +struct guarded_delete { + std::weak_ptr released_ptr; // Trick to keep the smart_holder memory footprint small. + std::function del_fun; // Rare case. + void (*del_ptr)(void *); // Common case. + bool use_del_fun; + bool armed_flag; + guarded_delete(std::function &&del_fun, bool armed_flag) + : del_fun{std::move(del_fun)}, del_ptr{nullptr}, use_del_fun{true}, + armed_flag{armed_flag} {} + guarded_delete(void (*del_ptr)(void *), bool armed_flag) + : del_ptr{del_ptr}, use_del_fun{false}, armed_flag{armed_flag} {} + void operator()(void *raw_ptr) const { + if (armed_flag) { + if (use_del_fun) { + del_fun(raw_ptr); + } else { + del_ptr(raw_ptr); + } + } + } +}; + +template ::value, int>::type = 0> +inline void builtin_delete_if_destructible(void *raw_ptr) { + std::default_delete{}(static_cast(raw_ptr)); +} + +template ::value, int>::type = 0> +inline void builtin_delete_if_destructible(void *) { + // This noop operator is needed to avoid a compilation error (for `delete raw_ptr;`), but + // throwing an exception from a destructor will std::terminate the process. Therefore the + // runtime check for lifetime-management correctness is implemented elsewhere (in + // ensure_pointee_is_destructible()). +} + +template +guarded_delete make_guarded_builtin_delete(bool armed_flag) { + return guarded_delete(builtin_delete_if_destructible, armed_flag); +} + +template +struct custom_deleter { + D deleter; + explicit custom_deleter(D &&deleter) : deleter{std::forward(deleter)} {} + void operator()(void *raw_ptr) { deleter(static_cast(raw_ptr)); } +}; + +template +guarded_delete make_guarded_custom_deleter(D &&uqp_del, bool armed_flag) { + return guarded_delete( + std::function(custom_deleter(std::forward(uqp_del))), armed_flag); +} + +template +inline bool is_std_default_delete(const std::type_info &rtti_deleter) { + return rtti_deleter == typeid(std::default_delete) + || rtti_deleter == typeid(std::default_delete); +} + +struct smart_holder { + const std::type_info *rtti_uqp_del = nullptr; + std::shared_ptr vptr; + bool vptr_is_using_noop_deleter : 1; + bool vptr_is_using_builtin_delete : 1; + bool vptr_is_external_shared_ptr : 1; + bool is_populated : 1; + bool is_disowned : 1; + + // Design choice: smart_holder is movable but not copyable. + smart_holder(smart_holder &&) = default; + smart_holder(const smart_holder &) = delete; + smart_holder &operator=(smart_holder &&) = delete; + smart_holder &operator=(const smart_holder &) = delete; + + smart_holder() + : vptr_is_using_noop_deleter{false}, vptr_is_using_builtin_delete{false}, + vptr_is_external_shared_ptr{false}, is_populated{false}, is_disowned{false} {} + + bool has_pointee() const { return vptr != nullptr; } + + template + static void ensure_pointee_is_destructible(const char *context) { + if (!std::is_destructible::value) { + throw std::invalid_argument(std::string("Pointee is not destructible (") + context + + ")."); + } + } + + void ensure_is_populated(const char *context) const { + if (!is_populated) { + throw std::runtime_error(std::string("Unpopulated holder (") + context + ")."); + } + } + void ensure_is_not_disowned(const char *context) const { + if (is_disowned) { + throw std::runtime_error(std::string("Holder was disowned already (") + context + + ")."); + } + } + + void ensure_vptr_is_using_builtin_delete(const char *context) const { + if (vptr_is_external_shared_ptr) { + throw std::invalid_argument(std::string("Cannot disown external shared_ptr (") + + context + ")."); + } + if (vptr_is_using_noop_deleter) { + throw std::invalid_argument(std::string("Cannot disown non-owning holder (") + context + + ")."); + } + if (!vptr_is_using_builtin_delete) { + throw std::invalid_argument(std::string("Cannot disown custom deleter (") + context + + ")."); + } + } + + template + void ensure_compatible_rtti_uqp_del(const char *context) const { + const std::type_info *rtti_requested = &typeid(D); + if (!rtti_uqp_del) { + if (!is_std_default_delete(*rtti_requested)) { + throw std::invalid_argument(std::string("Missing unique_ptr deleter (") + context + + ")."); + } + ensure_vptr_is_using_builtin_delete(context); + } else if (!(*rtti_requested == *rtti_uqp_del) + && !(vptr_is_using_builtin_delete + && is_std_default_delete(*rtti_requested))) { + throw std::invalid_argument(std::string("Incompatible unique_ptr deleter (") + context + + ")."); + } + } + + void ensure_has_pointee(const char *context) const { + if (!has_pointee()) { + throw std::invalid_argument(std::string("Disowned holder (") + context + ")."); + } + } + + void ensure_use_count_1(const char *context) const { + if (vptr == nullptr) { + throw std::invalid_argument(std::string("Cannot disown nullptr (") + context + ")."); + } + // In multithreaded environments accessing use_count can lead to + // race conditions, but in the context of Python it is a bug (elsewhere) + // if the Global Interpreter Lock (GIL) is not being held when this code + // is reached. + // PYBIND11:REMINDER: This may need to be protected by a mutex in free-threaded Python. + if (vptr.use_count() != 1) { + throw std::invalid_argument(std::string("Cannot disown use_count != 1 (") + context + + ")."); + } + } + + void reset_vptr_deleter_armed_flag(bool armed_flag) const { + auto *vptr_del_ptr = std::get_deleter(vptr); + if (vptr_del_ptr == nullptr) { + throw std::runtime_error( + "smart_holder::reset_vptr_deleter_armed_flag() called in an invalid context."); + } + vptr_del_ptr->armed_flag = armed_flag; + } + + // Caller is responsible for precondition: ensure_compatible_rtti_uqp_del() must succeed. + template + std::unique_ptr extract_deleter(const char *context) const { + const auto *gd = std::get_deleter(vptr); + if (gd && gd->use_del_fun) { + const auto &custom_deleter_ptr = gd->del_fun.template target>(); + if (custom_deleter_ptr == nullptr) { + throw std::runtime_error( + std::string("smart_holder::extract_deleter() precondition failure (") + context + + ")."); + } + static_assert(std::is_copy_constructible::value, + "Required for compatibility with smart_holder functionality."); + return std::unique_ptr(new D(custom_deleter_ptr->deleter)); + } + return nullptr; + } + + static smart_holder from_raw_ptr_unowned(void *raw_ptr) { + smart_holder hld; + hld.vptr.reset(raw_ptr, [](void *) {}); + hld.vptr_is_using_noop_deleter = true; + hld.is_populated = true; + return hld; + } + + template + T *as_raw_ptr_unowned() const { + return static_cast(vptr.get()); + } + + template + static smart_holder from_raw_ptr_take_ownership(T *raw_ptr, bool void_cast_raw_ptr = false) { + ensure_pointee_is_destructible("from_raw_ptr_take_ownership"); + smart_holder hld; + auto gd = make_guarded_builtin_delete(true); + if (void_cast_raw_ptr) { + hld.vptr.reset(static_cast(raw_ptr), std::move(gd)); + } else { + hld.vptr.reset(raw_ptr, std::move(gd)); + } + hld.vptr_is_using_builtin_delete = true; + hld.is_populated = true; + return hld; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void disown() { + reset_vptr_deleter_armed_flag(false); + is_disowned = true; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void reclaim_disowned() { + reset_vptr_deleter_armed_flag(true); + is_disowned = false; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void release_disowned() { vptr.reset(); } + + void ensure_can_release_ownership(const char *context = "ensure_can_release_ownership") const { + ensure_is_not_disowned(context); + ensure_vptr_is_using_builtin_delete(context); + ensure_use_count_1(context); + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void release_ownership() { + reset_vptr_deleter_armed_flag(false); + release_disowned(); + } + + template + static smart_holder from_unique_ptr(std::unique_ptr &&unq_ptr, + void *void_ptr = nullptr) { + smart_holder hld; + hld.rtti_uqp_del = &typeid(D); + hld.vptr_is_using_builtin_delete = is_std_default_delete(*hld.rtti_uqp_del); + guarded_delete gd{nullptr, false}; + if (hld.vptr_is_using_builtin_delete) { + gd = make_guarded_builtin_delete(true); + } else { + gd = make_guarded_custom_deleter(std::move(unq_ptr.get_deleter()), true); + } + if (void_ptr != nullptr) { + hld.vptr.reset(void_ptr, std::move(gd)); + } else { + hld.vptr.reset(unq_ptr.get(), std::move(gd)); + } + (void) unq_ptr.release(); + hld.is_populated = true; + return hld; + } + + template + static smart_holder from_shared_ptr(std::shared_ptr shd_ptr) { + smart_holder hld; + hld.vptr = std::static_pointer_cast(shd_ptr); + hld.vptr_is_external_shared_ptr = true; + hld.is_populated = true; + return hld; + } + + template + std::shared_ptr as_shared_ptr() const { + return std::static_pointer_cast(vptr); + } +}; + +} // namespace memory +} // namespace pybindit diff --git a/wrap/pybind11/include/pybind11/detail/type_caster_base.h b/wrap/pybind11/include/pybind11/detail/type_caster_base.h index fd8c81b9a..9618b2181 100644 --- a/wrap/pybind11/include/pybind11/detail/type_caster_base.h +++ b/wrap/pybind11/include/pybind11/detail/type_caster_base.h @@ -9,15 +9,24 @@ #pragma once -#include "../pytypes.h" +#include +#include +#include + #include "common.h" +#include "cpp_conduit.h" #include "descr.h" +#include "dynamic_raw_ptr_cast_if_possible.h" #include "internals.h" #include "typeid.h" +#include "using_smart_holder.h" +#include "value_and_holder.h" #include +#include #include #include +#include #include #include #include @@ -38,11 +47,7 @@ private: // Store stack pointer in thread-local storage. static PYBIND11_TLS_KEY_REF get_stack_tls_key() { -#if PYBIND11_INTERNALS_VERSION == 4 - return get_local_internals().loader_life_support_tls_key; -#else return get_internals().loader_life_support_tls_key; -#endif } static loader_life_support *get_stack_top() { return static_cast(PYBIND11_TLS_GET_VALUE(get_stack_tls_key())); @@ -112,7 +117,6 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector(t->tp_bases)) { check.push_back((PyTypeObject *) parent.ptr()); } - auto const &type_dict = get_internals().registered_types_py; for (size_t i = 0; i < check.size(); i++) { auto *type = check[i]; @@ -171,13 +175,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector &all_type_info(PyTypeObject *type) { - auto ins = all_type_info_get_cache(type); - if (ins.second) { - // New cache entry: populate it - all_type_info_populate(type, ins.first->second); - } - - return ins.first->second; + return all_type_info_get_cache(type).first->second; } /** @@ -243,6 +241,49 @@ PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, bool throw_if return handle(type_info ? ((PyObject *) type_info->type) : nullptr); } +inline bool try_incref(PyObject *obj) { + // Tries to increment the reference count of an object if it's not zero. + // TODO: Use PyUnstable_TryIncref when available. + // See https://github.com/python/cpython/issues/128844 +#ifdef Py_GIL_DISABLED + // See + // https://github.com/python/cpython/blob/d05140f9f77d7dfc753dd1e5ac3a5962aaa03eff/Include/internal/pycore_object.h#L761 + uint32_t local = _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local); + local += 1; + if (local == 0) { + // immortal + return true; + } + if (_Py_IsOwnedByCurrentThread(obj)) { + _Py_atomic_store_uint32_relaxed(&obj->ob_ref_local, local); +# ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +# endif + return true; + } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); + for (;;) { + // If the shared refcount is zero and the object is either merged + // or may not have weak references, then we cannot incref it. + if (shared == 0 || shared == _Py_REF_MERGED) { + return false; + } + + if (_Py_atomic_compare_exchange_ssize( + &obj->ob_ref_shared, &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) { +# ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +# endif + return true; + } + } +#else + assert(Py_REFCNT(obj) > 0); + Py_INCREF(obj); + return true; +#endif +} + // Searches the inheritance graph for a registered Python instance, using all_type_info(). PYBIND11_NOINLINE handle find_registered_python_instance(void *src, const detail::type_info *tinfo) { @@ -251,7 +292,10 @@ PYBIND11_NOINLINE handle find_registered_python_instance(void *src, for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { for (auto *instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { - return handle((PyObject *) it_i->second).inc_ref(); + auto *wrapper = reinterpret_cast(it_i->second); + if (try_incref(wrapper)) { + return handle(wrapper); + } } } } @@ -259,67 +303,6 @@ PYBIND11_NOINLINE handle find_registered_python_instance(void *src, }); } -struct value_and_holder { - instance *inst = nullptr; - size_t index = 0u; - const detail::type_info *type = nullptr; - void **vh = nullptr; - - // Main constructor for a found value/holder: - value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, - vh{inst->simple_layout ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} - - // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) - value_and_holder() = default; - - // Used for past-the-end iterator - explicit value_and_holder(size_t index) : index{index} {} - - template - V *&value_ptr() const { - return reinterpret_cast(vh[0]); - } - // True if this `value_and_holder` has a non-null value pointer - explicit operator bool() const { return value_ptr() != nullptr; } - - template - H &holder() const { - return reinterpret_cast(vh[1]); - } - bool holder_constructed() const { - return inst->simple_layout - ? inst->simple_holder_constructed - : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u; - } - // NOLINTNEXTLINE(readability-make-member-function-const) - void set_holder_constructed(bool v = true) { - if (inst->simple_layout) { - inst->simple_holder_constructed = v; - } else if (v) { - inst->nonsimple.status[index] |= instance::status_holder_constructed; - } else { - inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed; - } - } - bool instance_registered() const { - return inst->simple_layout - ? inst->simple_instance_registered - : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0); - } - // NOLINTNEXTLINE(readability-make-member-function-const) - void set_instance_registered(bool v = true) { - if (inst->simple_layout) { - inst->simple_instance_registered = v; - } else if (v) { - inst->nonsimple.status[index] |= instance::status_instance_registered; - } else { - inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered; - } - } -}; - // Container for accessing and iterating over an instance's values/holders struct values_and_holders { private: @@ -488,7 +471,7 @@ PYBIND11_NOINLINE void instance::allocate_layout() { // NOLINTNEXTLINE(readability-make-member-function-const) PYBIND11_NOINLINE void instance::deallocate_layout() { if (!simple_layout) { - PyMem_Free(nonsimple.values_and_holders); + PyMem_Free(reinterpret_cast(nonsimple.values_and_holders)); } } @@ -515,7 +498,7 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i } inline PyThreadState *get_thread_state_unchecked() { -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) return PyThreadState_GET(); #elif PY_VERSION_HEX < 0x030D0000 return _PyThreadState_UncheckedGet(); @@ -528,6 +511,361 @@ inline PyThreadState *get_thread_state_unchecked() { void keep_alive_impl(handle nurse, handle patient); inline PyObject *make_new_instance(PyTypeObject *type); +// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code. +inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); + +PYBIND11_NAMESPACE_BEGIN(smart_holder_type_caster_support) + +struct value_and_holder_helper { + value_and_holder loaded_v_h; + + bool have_holder() const { + return loaded_v_h.vh != nullptr && loaded_v_h.holder_constructed(); + } + + smart_holder &holder() const { return loaded_v_h.holder(); } + + void throw_if_uninitialized_or_disowned_holder(const char *typeid_name) const { + static const std::string missing_value_msg = "Missing value for wrapped C++ type `"; + if (!holder().is_populated) { + throw value_error(missing_value_msg + clean_type_id(typeid_name) + + "`: Python instance is uninitialized."); + } + if (!holder().has_pointee()) { + throw value_error(missing_value_msg + clean_type_id(typeid_name) + + "`: Python instance was disowned."); + } + } + + void throw_if_uninitialized_or_disowned_holder(const std::type_info &type_info) const { + throw_if_uninitialized_or_disowned_holder(type_info.name()); + } + + // have_holder() must be true or this function will fail. + void throw_if_instance_is_currently_owned_by_shared_ptr() const { + auto *vptr_gd_ptr = std::get_deleter(holder().vptr); + if (vptr_gd_ptr != nullptr && !vptr_gd_ptr->released_ptr.expired()) { + throw value_error("Python instance is currently owned by a std::shared_ptr."); + } + } + + void *get_void_ptr_or_nullptr() const { + if (have_holder()) { + auto &hld = holder(); + if (hld.is_populated && hld.has_pointee()) { + return hld.template as_raw_ptr_unowned(); + } + } + return nullptr; + } +}; + +template +handle smart_holder_from_unique_ptr(std::unique_ptr &&src, + return_value_policy policy, + handle parent, + const std::pair &st) { + if (policy == return_value_policy::copy) { + throw cast_error("return_value_policy::copy is invalid for unique_ptr."); + } + if (!src) { + return none().release(); + } + void *src_raw_void_ptr = const_cast(st.first); + assert(st.second != nullptr); + const detail::type_info *tinfo = st.second; + if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) { + auto *self_life_support + = dynamic_raw_ptr_cast_if_possible(src.get()); + if (self_life_support != nullptr) { + value_and_holder &v_h = self_life_support->v_h; + if (v_h.inst != nullptr && v_h.vh != nullptr) { + auto &holder = v_h.holder(); + if (!holder.is_disowned) { + pybind11_fail("smart_holder_from_unique_ptr: unexpected " + "smart_holder.is_disowned failure."); + } + // Critical transfer-of-ownership section. This must stay together. + self_life_support->deactivate_life_support(); + holder.reclaim_disowned(); + (void) src.release(); + // Critical section end. + return existing_inst; + } + } + throw cast_error("Invalid unique_ptr: another instance owns this pointer already."); + } + + auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + inst_raw_ptr->owned = true; + void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); + valueptr = src_raw_void_ptr; + + if (static_cast(src.get()) == src_raw_void_ptr) { + // This is a multiple-inheritance situation that is incompatible with the current + // shared_from_this handling (see PR #3023). Is there a better solution? + src_raw_void_ptr = nullptr; + } + auto smhldr = smart_holder::from_unique_ptr(std::move(src), src_raw_void_ptr); + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } + + return inst.release(); +} + +template +handle smart_holder_from_unique_ptr(std::unique_ptr &&src, + return_value_policy policy, + handle parent, + const std::pair &st) { + return smart_holder_from_unique_ptr( + std::unique_ptr(const_cast(src.release()), + std::move(src.get_deleter())), // Const2Mutbl + policy, + parent, + st); +} + +template +handle smart_holder_from_shared_ptr(const std::shared_ptr &src, + return_value_policy policy, + handle parent, + const std::pair &st) { + switch (policy) { + case return_value_policy::automatic: + case return_value_policy::automatic_reference: + break; + case return_value_policy::take_ownership: + throw cast_error("Invalid return_value_policy for shared_ptr (take_ownership)."); + case return_value_policy::copy: + case return_value_policy::move: + break; + case return_value_policy::reference: + throw cast_error("Invalid return_value_policy for shared_ptr (reference)."); + case return_value_policy::reference_internal: + break; + } + if (!src) { + return none().release(); + } + + auto src_raw_ptr = src.get(); + assert(st.second != nullptr); + void *src_raw_void_ptr = static_cast(src_raw_ptr); + const detail::type_info *tinfo = st.second; + if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) { + // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing smart_holder. + // PYBIND11:REMINDER: MISSING: keep_alive. + return existing_inst; + } + + auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + inst_raw_ptr->owned = true; + void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); + valueptr = src_raw_void_ptr; + + auto smhldr + = smart_holder::from_shared_ptr(std::shared_ptr(src, const_cast(st.first))); + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } + + return inst.release(); +} + +template +handle smart_holder_from_shared_ptr(const std::shared_ptr &src, + return_value_policy policy, + handle parent, + const std::pair &st) { + return smart_holder_from_shared_ptr(std::const_pointer_cast(src), // Const2Mutbl + policy, + parent, + st); +} + +struct shared_ptr_parent_life_support { + PyObject *parent; + explicit shared_ptr_parent_life_support(PyObject *parent) : parent{parent} { + Py_INCREF(parent); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void operator()(void *) { + gil_scoped_acquire gil; + Py_DECREF(parent); + } +}; + +struct shared_ptr_trampoline_self_life_support { + PyObject *self; + explicit shared_ptr_trampoline_self_life_support(instance *inst) + : self{reinterpret_cast(inst)} { + gil_scoped_acquire gil; + Py_INCREF(self); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void operator()(void *) { + gil_scoped_acquire gil; + Py_DECREF(self); + } +}; + +template ::value, int>::type = 0> +inline std::unique_ptr unique_with_deleter(T *raw_ptr, std::unique_ptr &&deleter) { + if (deleter == nullptr) { + return std::unique_ptr(raw_ptr); + } + return std::unique_ptr(raw_ptr, std::move(*deleter)); +} + +template ::value, int>::type = 0> +inline std::unique_ptr unique_with_deleter(T *raw_ptr, std::unique_ptr &&deleter) { + if (deleter == nullptr) { + pybind11_fail("smart_holder_type_casters: deleter is not default constructible and no" + " instance available to return."); + } + return std::unique_ptr(raw_ptr, std::move(*deleter)); +} + +template +struct load_helper : value_and_holder_helper { + bool was_populated = false; + bool python_instance_is_alias = false; + + void maybe_set_python_instance_is_alias(handle src) { + if (was_populated) { + python_instance_is_alias = reinterpret_cast(src.ptr())->is_alias; + } + } + + static std::shared_ptr make_shared_ptr_with_responsible_parent(T *raw_ptr, handle parent) { + return std::shared_ptr(raw_ptr, shared_ptr_parent_life_support(parent.ptr())); + } + + std::shared_ptr load_as_shared_ptr(void *void_raw_ptr, + handle responsible_parent = nullptr) const { + if (!have_holder()) { + return nullptr; + } + throw_if_uninitialized_or_disowned_holder(typeid(T)); + smart_holder &hld = holder(); + hld.ensure_is_not_disowned("load_as_shared_ptr"); + if (hld.vptr_is_using_noop_deleter) { + if (responsible_parent) { + return make_shared_ptr_with_responsible_parent(static_cast(void_raw_ptr), + responsible_parent); + } + throw std::runtime_error("Non-owning holder (load_as_shared_ptr)."); + } + auto *type_raw_ptr = static_cast(void_raw_ptr); + if (python_instance_is_alias) { + auto *vptr_gd_ptr = std::get_deleter(hld.vptr); + if (vptr_gd_ptr != nullptr) { + std::shared_ptr released_ptr = vptr_gd_ptr->released_ptr.lock(); + if (released_ptr) { + return std::shared_ptr(released_ptr, type_raw_ptr); + } + std::shared_ptr to_be_released( + type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst)); + vptr_gd_ptr->released_ptr = to_be_released; + return to_be_released; + } + auto *sptsls_ptr = std::get_deleter(hld.vptr); + if (sptsls_ptr != nullptr) { + // This code is reachable only if there are multiple registered_instances for the + // same pointee. + if (reinterpret_cast(loaded_v_h.inst) == sptsls_ptr->self) { + pybind11_fail("smart_holder_type_caster_support load_as_shared_ptr failure: " + "loaded_v_h.inst == sptsls_ptr->self"); + } + } + if (sptsls_ptr != nullptr + || !pybindit::memory::type_has_shared_from_this(type_raw_ptr)) { + return std::shared_ptr( + type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst)); + } + if (hld.vptr_is_external_shared_ptr) { + pybind11_fail("smart_holder_type_casters load_as_shared_ptr failure: not " + "implemented: trampoline-self-life-support for external shared_ptr " + "to type inheriting from std::enable_shared_from_this."); + } + pybind11_fail( + "smart_holder_type_casters: load_as_shared_ptr failure: internal inconsistency."); + } + std::shared_ptr void_shd_ptr = hld.template as_shared_ptr(); + return std::shared_ptr(void_shd_ptr, type_raw_ptr); + } + + template + std::unique_ptr load_as_unique_ptr(void *raw_void_ptr, + const char *context = "load_as_unique_ptr") { + if (!have_holder()) { + return unique_with_deleter(nullptr, std::unique_ptr()); + } + throw_if_uninitialized_or_disowned_holder(typeid(T)); + throw_if_instance_is_currently_owned_by_shared_ptr(); + holder().ensure_is_not_disowned(context); + holder().template ensure_compatible_rtti_uqp_del(context); + holder().ensure_use_count_1(context); + + T *raw_type_ptr = static_cast(raw_void_ptr); + + auto *self_life_support + = dynamic_raw_ptr_cast_if_possible(raw_type_ptr); + if (self_life_support == nullptr && python_instance_is_alias) { + throw value_error("Alias class (also known as trampoline) does not inherit from " + "py::trampoline_self_life_support, therefore the ownership of this " + "instance cannot safely be transferred to C++."); + } + + std::unique_ptr extracted_deleter = holder().template extract_deleter(context); + + // Critical transfer-of-ownership section. This must stay together. + if (self_life_support != nullptr) { + holder().disown(); + } else { + holder().release_ownership(); + } + auto result = unique_with_deleter(raw_type_ptr, std::move(extracted_deleter)); + if (self_life_support != nullptr) { + self_life_support->activate_life_support(loaded_v_h); + } else { + void *value_void_ptr = loaded_v_h.value_ptr(); + loaded_v_h.value_ptr() = nullptr; + deregister_instance(loaded_v_h.inst, value_void_ptr, loaded_v_h.type); + } + // Critical section end. + + return result; + } + + // This assumes load_as_shared_ptr succeeded(), and the returned shared_ptr is still alive. + // The returned unique_ptr is meant to never expire (the behavior is undefined otherwise). + template + std::unique_ptr + load_as_const_unique_ptr(T *raw_type_ptr, const char *context = "load_as_const_unique_ptr") { + if (!have_holder()) { + return unique_with_deleter(nullptr, std::unique_ptr()); + } + holder().template ensure_compatible_rtti_uqp_del(context); + return unique_with_deleter( + raw_type_ptr, std::move(holder().template extract_deleter(context))); + } +}; + +PYBIND11_NAMESPACE_END(smart_holder_type_caster_support) + class type_caster_generic { public: PYBIND11_NOINLINE explicit type_caster_generic(const std::type_info &type_info) @@ -632,6 +970,15 @@ public: // Base methods for generic caster; there are overridden in copyable_holder_caster void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + smart_holder_type_caster_support::value_and_holder_helper v_h_helper; + v_h_helper.loaded_v_h = v_h; + if (v_h_helper.have_holder()) { + v_h_helper.throw_if_uninitialized_or_disowned_holder(cpptype->name()); + value = v_h_helper.holder().template as_raw_ptr_unowned(); + return; + } + } auto *&vptr = v_h.value_ptr(); // Lazy allocation for unallocated values: if (vptr == nullptr) { @@ -670,6 +1017,13 @@ public: } return false; } + bool try_cpp_conduit(handle src) { + value = try_raw_pointer_ephemeral_from_cpp_conduit(src, cpptype); + if (value != nullptr) { + return true; + } + return false; + } void check_holder_compat() {} PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { @@ -801,6 +1155,10 @@ public: return true; } + if (convert && cpptype && this_.try_cpp_conduit(src)) { + return true; + } + return false; } @@ -828,6 +1186,32 @@ public: void *value = nullptr; }; +inline object cpp_conduit_method(handle self, + const bytes &pybind11_platform_abi_id, + const capsule &cpp_type_info_capsule, + const bytes &pointer_kind) { +#ifdef PYBIND11_HAS_STRING_VIEW + using cpp_str = std::string_view; +#else + using cpp_str = std::string; +#endif + if (cpp_str(pybind11_platform_abi_id) != PYBIND11_PLATFORM_ABI_ID) { + return none(); + } + if (std::strcmp(cpp_type_info_capsule.name(), typeid(std::type_info).name()) != 0) { + return none(); + } + if (cpp_str(pointer_kind) != "raw_pointer_ephemeral") { + throw std::runtime_error("Invalid pointer_kind: \"" + std::string(pointer_kind) + "\""); + } + const auto *cpp_type_info = cpp_type_info_capsule.get_pointer(); + type_caster_generic caster(*cpp_type_info); + if (!caster.load(self, false)) { + return none(); + } + return capsule(caster.value, cpp_type_info->name()); +} + /** * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster * needs to provide `operator T*()` and `operator T&()` operators. @@ -1180,14 +1564,14 @@ protected: does not have a private operator new implementation. A comma operator is used in the decltype argument to apply SFINAE to the public copy/move constructors.*/ template ::value>> - static auto make_copy_constructor(const T *) -> decltype(new T(std::declval()), - Constructor{}) { + static auto make_copy_constructor(const T *) + -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } template ::value>> - static auto make_move_constructor(const T *) -> decltype(new T(std::declval()), - Constructor{}) { + static auto make_move_constructor(const T *) + -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { return new T(std::move(*const_cast(reinterpret_cast(arg)))); }; diff --git a/wrap/pybind11/include/pybind11/detail/using_smart_holder.h b/wrap/pybind11/include/pybind11/detail/using_smart_holder.h new file mode 100644 index 000000000..57f99b95f --- /dev/null +++ b/wrap/pybind11/include/pybind11/detail/using_smart_holder.h @@ -0,0 +1,22 @@ +// Copyright (c) 2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" +#include "struct_smart_holder.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +using pybindit::memory::smart_holder; + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +using is_smart_holder = std::is_same; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/detail/value_and_holder.h b/wrap/pybind11/include/pybind11/detail/value_and_holder.h new file mode 100644 index 000000000..64c55cc59 --- /dev/null +++ b/wrap/pybind11/include/pybind11/detail/value_and_holder.h @@ -0,0 +1,78 @@ +// Copyright (c) 2016-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +struct value_and_holder { + instance *inst = nullptr; + size_t index = 0u; + const detail::type_info *type = nullptr; + void **vh = nullptr; + + // Main constructor for a found value/holder: + value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} + + // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) + value_and_holder() = default; + + // Used for past-the-end iterator + explicit value_and_holder(size_t index) : index{index} {} + + template + V *&value_ptr() const { + return reinterpret_cast(vh[0]); + } + // True if this `value_and_holder` has a non-null value pointer + explicit operator bool() const { return value_ptr() != nullptr; } + + template + H &holder() const { + return reinterpret_cast(vh[1]); + } + bool holder_constructed() const { + return inst->simple_layout + ? inst->simple_holder_constructed + : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u; + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void set_holder_constructed(bool v = true) { + if (inst->simple_layout) { + inst->simple_holder_constructed = v; + } else if (v) { + inst->nonsimple.status[index] |= instance::status_holder_constructed; + } else { + inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed; + } + } + bool instance_registered() const { + return inst->simple_layout + ? inst->simple_instance_registered + : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void set_instance_registered(bool v = true) { + if (inst->simple_layout) { + inst->simple_instance_registered = v; + } else if (v) { + inst->nonsimple.status[index] |= instance::status_instance_registered; + } else { + inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered; + } + } +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/eigen/matrix.h b/wrap/pybind11/include/pybind11/eigen/matrix.h index 8d4342f81..ca599c954 100644 --- a/wrap/pybind11/include/pybind11/eigen/matrix.h +++ b/wrap/pybind11/include/pybind11/eigen/matrix.h @@ -9,7 +9,8 @@ #pragma once -#include "../numpy.h" +#include + #include "common.h" /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. @@ -224,19 +225,22 @@ struct EigenProps { = !show_c_contiguous && show_order && requires_col_major; static constexpr auto descriptor - = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + = const_name("typing.Annotated[") + + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[") + + npy_format_descriptor::name + io_name("", "]") + const_name(", \"[") + const_name(const_name<(size_t) rows>(), const_name("m")) + const_name(", ") - + const_name(const_name<(size_t) cols>(), const_name("n")) + const_name("]") - + + + const_name(const_name<(size_t) cols>(), const_name("n")) + + const_name("]\"") // For a reference type (e.g. Ref) we have other constraints that might need to // be satisfied: writeable=True (for a mutable reference), and, depending on the map's // stride options, possibly f_contiguous or c_contiguous. We include them in the // descriptor output to provide some hint as to why a TypeError is occurring (otherwise - // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and - // an error message that you *gave* a numpy.ndarray of the right type and dimensions. - const_name(", flags.writeable", "") - + const_name(", flags.c_contiguous", "") - + const_name(", flags.f_contiguous", "") + const_name("]"); + // it can be confusing to see that a function accepts a + // 'typing.Annotated[numpy.typing.NDArray[numpy.float64], "[3,2]"]' and an error message + // that you *gave* a numpy.ndarray of the right type and dimensions. + + const_name(", \"flags.writeable\"", "") + + const_name(", \"flags.c_contiguous\"", "") + + const_name(", \"flags.f_contiguous\"", "") + const_name("]"); }; // Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, @@ -315,8 +319,11 @@ struct type_caster::value>> { return false; } + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") // See PR #5516 // Allocate the new type, then build a numpy reference into it value = Type(fits.rows, fits.cols); + PYBIND11_WARNING_POP auto ref = reinterpret_steal(eigen_ref_array(value)); if (dims == 1) { ref = ref.squeeze(); @@ -437,7 +444,9 @@ public: } } - static constexpr auto name = props::descriptor; + // return_descr forces the use of NDArray instead of ArrayLike in args + // since Ref<...> args can only accept arrays. + static constexpr auto name = return_descr(props::descriptor); // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return // types but not bound arguments). We still provide them (with an explicitly delete) so that diff --git a/wrap/pybind11/include/pybind11/eigen/tensor.h b/wrap/pybind11/include/pybind11/eigen/tensor.h index d4ed6c0ca..50e8b50b1 100644 --- a/wrap/pybind11/include/pybind11/eigen/tensor.h +++ b/wrap/pybind11/include/pybind11/eigen/tensor.h @@ -7,7 +7,8 @@ #pragma once -#include "../numpy.h" +#include + #include "common.h" #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) @@ -123,13 +124,16 @@ struct eigen_tensor_helper< template struct get_tensor_descriptor { static constexpr auto details - = const_name(", flags.writeable", "") - + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( - ", flags.c_contiguous", ", flags.f_contiguous"); + = const_name(", \"flags.writeable\"", "") + const_name + < static_cast(Type::Layout) + == static_cast(Eigen::RowMajor) + > (", \"flags.c_contiguous\"", ", \"flags.f_contiguous\""); static constexpr auto value - = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper>::dimensions_descriptor - + const_name("]") + const_name(details, const_name("")) + const_name("]"); + = const_name("typing.Annotated[") + + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[") + + npy_format_descriptor::name + io_name("", "]") + + const_name(", \"[") + eigen_tensor_helper>::dimensions_descriptor + + const_name("]\"") + const_name(details, const_name("")) + const_name("]"); }; // When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes does not have the begin() member @@ -469,9 +473,6 @@ struct type_caster, parent_object = reinterpret_borrow(parent); break; - case return_value_policy::take_ownership: - delete src; - // fallthrough default: // move, take_ownership don't make any sense for a ref/map: pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " @@ -504,7 +505,10 @@ protected: std::unique_ptr value; public: - static constexpr auto name = get_tensor_descriptor::value; + // return_descr forces the use of NDArray instead of ArrayLike since refs can only reference + // arrays + static constexpr auto name + = return_descr(get_tensor_descriptor::value); explicit operator MapType *() { return value.get(); } explicit operator MapType &() { return *value; } explicit operator MapType &&() && { return std::move(*value); } diff --git a/wrap/pybind11/include/pybind11/embed.h b/wrap/pybind11/include/pybind11/embed.h index 9d29eb824..0af777033 100644 --- a/wrap/pybind11/include/pybind11/embed.h +++ b/wrap/pybind11/include/pybind11/embed.h @@ -104,23 +104,13 @@ inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers, detail::precheck_interpreter(); Py_InitializeEx(init_signal_handlers ? 1 : 0); - // Before it was special-cased in python 3.8, passing an empty or null argv - // caused a segfault, so we have to reimplement the special case ourselves. - bool special_case = (argv == nullptr || argc <= 0); - - const char *const empty_argv[]{"\0"}; - const char *const *safe_argv = special_case ? empty_argv : argv; - if (special_case) { - argc = 1; - } - auto argv_size = static_cast(argc); // SetArgv* on python 3 takes wchar_t, so we have to convert. std::unique_ptr widened_argv(new wchar_t *[argv_size]); std::vector> widened_argv_entries; widened_argv_entries.reserve(argv_size); for (size_t ii = 0; ii < argv_size; ++ii) { - widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii])); + widened_argv_entries.emplace_back(detail::widen_chars(argv[ii])); if (!widened_argv_entries.back()) { // A null here indicates a character-encoding failure or the python // interpreter out of memory. Give up. diff --git a/wrap/pybind11/include/pybind11/eval.h b/wrap/pybind11/include/pybind11/eval.h index bd5f981f5..3ed1b5a4a 100644 --- a/wrap/pybind11/include/pybind11/eval.h +++ b/wrap/pybind11/include/pybind11/eval.h @@ -19,7 +19,7 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) inline void ensure_builtins_in_globals(object &global) { -#if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x03080000 +#if defined(PYPY_VERSION) // Running exec and eval adds `builtins` module under `__builtins__` key to // globals if not yet present. Python 3.8 made PyRun_String behave // similarly. Let's also do that for older versions, for consistency. This @@ -94,18 +94,18 @@ void exec(const char (&s)[N], object global = globals(), object local = object() eval(s, std::move(global), std::move(local)); } -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) template object eval_file(str, object, object) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } template object eval_file(str, object) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } template object eval_file(str) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } #else template diff --git a/wrap/pybind11/include/pybind11/functional.h b/wrap/pybind11/include/pybind11/functional.h index 6856119cd..4b3610117 100644 --- a/wrap/pybind11/include/pybind11/functional.h +++ b/wrap/pybind11/include/pybind11/functional.h @@ -9,12 +9,55 @@ #pragma once +#define PYBIND11_HAS_TYPE_CASTER_STD_FUNCTION_SPECIALIZATIONS + #include "pybind11.h" #include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +PYBIND11_NAMESPACE_BEGIN(type_caster_std_function_specializations) + +// ensure GIL is held during functor destruction +struct func_handle { + function f; +#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17)) + // This triggers a syntax error under very special conditions (very weird indeed). + explicit +#endif + func_handle(function &&f_) noexcept + : f(std::move(f_)) { + } + func_handle(const func_handle &f_) { operator=(f_); } + func_handle &operator=(const func_handle &f_) { + gil_scoped_acquire acq; + f = f_.f; + return *this; + } + ~func_handle() { + gil_scoped_acquire acq; + function kill_f(std::move(f)); + } +}; + +// to emulate 'move initialization capture' in C++11 +struct func_wrapper_base { + func_handle hfunc; + explicit func_wrapper_base(func_handle &&hf) noexcept : hfunc(hf) {} +}; + +template +struct func_wrapper : func_wrapper_base { + using func_wrapper_base::func_wrapper_base; + Return operator()(Args... args) const { + gil_scoped_acquire acq; + // casts the returned object as a rvalue to the return type + return hfunc.f(std::forward(args)...).template cast(); + } +}; + +PYBIND11_NAMESPACE_END(type_caster_std_function_specializations) template struct type_caster> { @@ -77,40 +120,8 @@ public: // See PR #1413 for full details } - // ensure GIL is held during functor destruction - struct func_handle { - function f; -#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17)) - // This triggers a syntax error under very special conditions (very weird indeed). - explicit -#endif - func_handle(function &&f_) noexcept - : f(std::move(f_)) { - } - func_handle(const func_handle &f_) { operator=(f_); } - func_handle &operator=(const func_handle &f_) { - gil_scoped_acquire acq; - f = f_.f; - return *this; - } - ~func_handle() { - gil_scoped_acquire acq; - function kill_f(std::move(f)); - } - }; - - // to emulate 'move initialization capture' in C++11 - struct func_wrapper { - func_handle hfunc; - explicit func_wrapper(func_handle &&hf) noexcept : hfunc(std::move(hf)) {} - Return operator()(Args... args) const { - gil_scoped_acquire acq; - // casts the returned object as a rvalue to the return type - return hfunc.f(std::forward(args)...).template cast(); - } - }; - - value = func_wrapper(func_handle(std::move(func))); + value = type_caster_std_function_specializations::func_wrapper( + type_caster_std_function_specializations::func_handle(std::move(func))); return true; } diff --git a/wrap/pybind11/include/pybind11/gil.h b/wrap/pybind11/include/pybind11/gil.h index 6b0edaee4..888810493 100644 --- a/wrap/pybind11/include/pybind11/gil.h +++ b/wrap/pybind11/include/pybind11/gil.h @@ -147,9 +147,7 @@ public: // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) tstate = PyEval_SaveThread(); if (disassoc) { - // Python >= 3.7 can remove this, it's an int before 3.7 - // NOLINTNEXTLINE(readability-qualified-auto) - auto key = internals.tstate; + auto key = internals.tstate; // NOLINT(readability-qualified-auto) PYBIND11_TLS_DELETE_VALUE(key); } } @@ -173,9 +171,7 @@ public: PyEval_RestoreThread(tstate); } if (disassoc) { - // Python >= 3.7 can remove this, it's an int before 3.7 - // NOLINTNEXTLINE(readability-qualified-auto) - auto key = detail::get_internals().tstate; + auto key = detail::get_internals().tstate; // NOLINT(readability-qualified-auto) PYBIND11_TLS_REPLACE_VALUE(key, tstate); } } diff --git a/wrap/pybind11/include/pybind11/gil_safe_call_once.h b/wrap/pybind11/include/pybind11/gil_safe_call_once.h index eaf84d16e..44e68f029 100644 --- a/wrap/pybind11/include/pybind11/gil_safe_call_once.h +++ b/wrap/pybind11/include/pybind11/gil_safe_call_once.h @@ -8,6 +8,10 @@ #include #include +#ifdef Py_GIL_DISABLED +# include +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) // Use the `gil_safe_call_once_and_store` class below instead of the naive @@ -42,6 +46,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) // get processed only when it is the main thread's turn again and it is running // normal Python code. However, this will be unnoticeable for quick call-once // functions, which is usually the case. +// +// For in-depth background, see docs/advanced/deadlock.md template class gil_safe_call_once_and_store { public: @@ -82,7 +88,12 @@ public: private: alignas(T) char storage_[sizeof(T)] = {}; std::once_flag once_flag_ = {}; - bool is_initialized_ = false; +#ifdef Py_GIL_DISABLED + std::atomic_bool +#else + bool +#endif + is_initialized_{false}; // The `is_initialized_`-`storage_` pair is very similar to `std::optional`, // but the latter does not have the triviality properties of former, // therefore `std::optional` is not a viable alternative here. diff --git a/wrap/pybind11/include/pybind11/numpy.h b/wrap/pybind11/include/pybind11/numpy.h index 05ef3918b..3a370fe5a 100644 --- a/wrap/pybind11/include/pybind11/numpy.h +++ b/wrap/pybind11/include/pybind11/numpy.h @@ -175,7 +175,6 @@ inline numpy_internals &get_numpy_internals() { PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { module_ numpy = module_::import("numpy"); str version_string = numpy.attr("__version__"); - module_ numpy_lib = module_::import("numpy.lib"); object numpy_version = numpy_lib.attr("NumpyVersion")(version_string); int major_version = numpy_version.attr("major").cast(); @@ -212,6 +211,7 @@ constexpr int platform_lookup(int I, Ints... Is) { } struct npy_api { + // If you change this code, please review `normalized_dtype_num` below. enum constants { NPY_ARRAY_C_CONTIGUOUS_ = 0x0001, NPY_ARRAY_F_CONTIGUOUS_ = 0x0002, @@ -384,6 +384,74 @@ private: } }; +// This table normalizes typenums by mapping NPY_INT_, NPY_LONG, ... to NPY_INT32_, NPY_INT64, ... +// This is needed to correctly handle situations where multiple typenums map to the same type, +// e.g. NPY_LONG_ may be equivalent to NPY_INT_ or NPY_LONGLONG_ despite having a different +// typenum. The normalized typenum should always match the values used in npy_format_descriptor. +// If you change this code, please review `enum constants` above. +static constexpr int normalized_dtype_num[npy_api::NPY_VOID_ + 1] = { + // NPY_BOOL_ => + npy_api::NPY_BOOL_, + // NPY_BYTE_ => + npy_api::NPY_BYTE_, + // NPY_UBYTE_ => + npy_api::NPY_UBYTE_, + // NPY_SHORT_ => + npy_api::NPY_INT16_, + // NPY_USHORT_ => + npy_api::NPY_UINT16_, + // NPY_INT_ => + sizeof(int) == sizeof(std::int16_t) ? npy_api::NPY_INT16_ + : sizeof(int) == sizeof(std::int32_t) ? npy_api::NPY_INT32_ + : sizeof(int) == sizeof(std::int64_t) ? npy_api::NPY_INT64_ + : npy_api::NPY_INT_, + // NPY_UINT_ => + sizeof(unsigned int) == sizeof(std::uint16_t) ? npy_api::NPY_UINT16_ + : sizeof(unsigned int) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_ + : sizeof(unsigned int) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_ + : npy_api::NPY_UINT_, + // NPY_LONG_ => + sizeof(long) == sizeof(std::int16_t) ? npy_api::NPY_INT16_ + : sizeof(long) == sizeof(std::int32_t) ? npy_api::NPY_INT32_ + : sizeof(long) == sizeof(std::int64_t) ? npy_api::NPY_INT64_ + : npy_api::NPY_LONG_, + // NPY_ULONG_ => + sizeof(unsigned long) == sizeof(std::uint16_t) ? npy_api::NPY_UINT16_ + : sizeof(unsigned long) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_ + : sizeof(unsigned long) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_ + : npy_api::NPY_ULONG_, + // NPY_LONGLONG_ => + sizeof(long long) == sizeof(std::int16_t) ? npy_api::NPY_INT16_ + : sizeof(long long) == sizeof(std::int32_t) ? npy_api::NPY_INT32_ + : sizeof(long long) == sizeof(std::int64_t) ? npy_api::NPY_INT64_ + : npy_api::NPY_LONGLONG_, + // NPY_ULONGLONG_ => + sizeof(unsigned long long) == sizeof(std::uint16_t) ? npy_api::NPY_UINT16_ + : sizeof(unsigned long long) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_ + : sizeof(unsigned long long) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_ + : npy_api::NPY_ULONGLONG_, + // NPY_FLOAT_ => + npy_api::NPY_FLOAT_, + // NPY_DOUBLE_ => + npy_api::NPY_DOUBLE_, + // NPY_LONGDOUBLE_ => + npy_api::NPY_LONGDOUBLE_, + // NPY_CFLOAT_ => + npy_api::NPY_CFLOAT_, + // NPY_CDOUBLE_ => + npy_api::NPY_CDOUBLE_, + // NPY_CLONGDOUBLE_ => + npy_api::NPY_CLONGDOUBLE_, + // NPY_OBJECT_ => + npy_api::NPY_OBJECT_, + // NPY_STRING_ => + npy_api::NPY_STRING_, + // NPY_UNICODE_ => + npy_api::NPY_UNICODE_, + // NPY_VOID_ => + npy_api::NPY_VOID_, +}; + inline PyArray_Proxy *array_proxy(void *ptr) { return reinterpret_cast(ptr); } inline const PyArray_Proxy *array_proxy(const void *ptr) { @@ -684,6 +752,13 @@ public: return detail::npy_format_descriptor::type>::dtype(); } + /// Return the type number associated with a C++ type. + /// This is the constexpr equivalent of `dtype::of().num()`. + template + static constexpr int num_of() { + return detail::npy_format_descriptor::type>::value; + } + /// Size of the data type in bytes. #ifdef PYBIND11_NUMPY_1_ONLY ssize_t itemsize() const { return detail::array_descriptor_proxy(m_ptr)->elsize; } @@ -725,7 +800,9 @@ public: return detail::array_descriptor_proxy(m_ptr)->type; } - /// type number of dtype. + /// Type number of dtype. Note that different values may be returned for equivalent types, + /// e.g. even though ``long`` may be equivalent to ``int`` or ``long long``, they still have + /// different type numbers. Consider using `normalized_num` to avoid this. int num() const { // Note: The signature, `dtype::num` follows the naming of NumPy's public // Python API (i.e., ``dtype.num``), rather than its internal @@ -733,6 +810,17 @@ public: return detail::array_descriptor_proxy(m_ptr)->type_num; } + /// Type number of dtype, normalized to match the return value of `num_of` for equivalent + /// types. This function can be used to write switch statements that correctly handle + /// equivalent types with different type numbers. + int normalized_num() const { + int value = num(); + if (value >= 0 && value <= detail::npy_api::NPY_VOID_) { + return detail::normalized_dtype_num[value]; + } + return value; + } + /// Single character for byteorder char byteorder() const { return detail::array_descriptor_proxy(m_ptr)->byteorder; } @@ -901,7 +989,11 @@ public: template array(ShapeContainer shape, StridesContainer strides, const T *ptr, handle base = handle()) - : array(pybind11::dtype::of(), std::move(shape), std::move(strides), ptr, base) {} + : array(pybind11::dtype::of(), + std::move(shape), + std::move(strides), + reinterpret_cast(ptr), + base) {} template array(ShapeContainer shape, const T *ptr, handle base = handle()) @@ -1424,7 +1516,11 @@ public: }; template -struct npy_format_descriptor::value>> { +struct npy_format_descriptor< + T, + enable_if_t::value + || ((std::is_same::value || std::is_same::value) + && sizeof(T) == sizeof(PyObject *))>> { static constexpr auto name = const_name("object"); static constexpr int value = npy_api::NPY_OBJECT_; @@ -1986,7 +2082,7 @@ private: // Pointers to values the function was called with; the vectorized ones set here will start // out as array_t pointers, but they will be changed them to T pointers before we make // call the wrapped function. Non-vectorized pointers are left as-is. - std::array params{{&args...}}; + std::array params{{reinterpret_cast(&args)...}}; // The array of `buffer_info`s of vectorized arguments: std::array buffers{ @@ -2086,7 +2182,8 @@ vectorize_helper vectorize_extractor(const Func &f, Retur template struct handle_type_name> { static constexpr auto name - = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("]"); + = io_name("typing.Annotated[numpy.typing.ArrayLike, ", "numpy.typing.NDArray[") + + npy_format_descriptor::name + const_name("]"); }; PYBIND11_NAMESPACE_END(detail) diff --git a/wrap/pybind11/include/pybind11/pybind11.h b/wrap/pybind11/include/pybind11/pybind11.h index 74919a7d5..4dee2c55f 100644 --- a/wrap/pybind11/include/pybind11/pybind11.h +++ b/wrap/pybind11/include/pybind11/pybind11.h @@ -9,9 +9,11 @@ */ #pragma once - #include "detail/class.h" +#include "detail/dynamic_raw_ptr_cast_if_possible.h" +#include "detail/exception_translation.h" #include "detail/init.h" +#include "detail/using_smart_holder.h" #include "attr.h" #include "gil.h" #include "gil_safe_call_once.h" @@ -22,10 +24,17 @@ #include #include #include +#include #include #include #include +// See PR #5448. This warning suppression is needed for the PYBIND11_OVERRIDE macro family. +// NOTE that this is NOT embedded in a push/pop pair because that is very difficult to achieve. +#if defined(__clang_major__) && __clang_major__ < 14 +PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") +#endif + #if defined(__cpp_lib_launder) && !(defined(_MSC_VER) && (_MSC_VER < 1914)) # define PYBIND11_STD_LAUNDER std::launder # define PYBIND11_HAS_STD_LAUNDER 1 @@ -95,24 +104,6 @@ inline std::string replace_newlines_and_squash(const char *text) { return result.substr(str_begin, str_range); } -// Apply all the extensions translators from a list -// Return true if one of the translators completed without raising an exception -// itself. Return of false indicates that if there are other translators -// available, they should be tried. -inline bool apply_exception_translators(std::forward_list &translators) { - auto last_exception = std::current_exception(); - - for (auto &translator : translators) { - try { - translator(last_exception); - return true; - } catch (...) { - last_exception = std::current_exception(); - } - } - return false; -} - #if defined(_MSC_VER) # define PYBIND11_COMPAT_STRDUP _strdup #else @@ -319,9 +310,20 @@ protected: constexpr bool has_kw_only_args = any_of...>::value, has_pos_only_args = any_of...>::value, has_arg_annotations = any_of...>::value; + constexpr bool has_is_method = any_of...>::value; + // The implicit `self` argument is not present and not counted in method definitions. + constexpr bool has_args = cast_in::args_pos >= 0; + constexpr bool is_method_with_self_arg_only = has_is_method && !has_args; static_assert(has_arg_annotations || !has_kw_only_args, "py::kw_only requires the use of argument annotations"); - static_assert(has_arg_annotations || !has_pos_only_args, + static_assert(((/* Need `py::arg("arg_name")` annotation in function/method. */ + has_arg_annotations) + || (/* Allow methods with no arguments `def method(self, /): ...`. + * A method has at least one argument `self`. There can be no + * `py::arg` annotation. E.g. `class.def("method", py::pos_only())`. + */ + is_method_with_self_arg_only)) + || !has_pos_only_args, "py::pos_only requires the use of argument annotations (for docstrings " "and aligning the annotations to the argument)"); @@ -441,6 +443,13 @@ protected: std::string signature; size_t type_index = 0, arg_index = 0; bool is_starred = false; + // `is_return_value.top()` is true if we are currently inside the return type of the + // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops + // back to the previous state. + std::stack is_return_value({false}); + // The following characters have special meaning in the signature parsing. Literals + // containing these are escaped with `!`. + std::string special_chars("!@%{}-"); for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; @@ -494,7 +503,57 @@ protected: } else { signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name())); } + } else if (c == '!' && special_chars.find(*(pc + 1)) != std::string::npos) { + // typing::Literal escapes special characters with ! + signature += *++pc; + } else if (c == '@') { + // `@^ ... @!` and `@$ ... @!` are used to force arg/return value type (see + // typing::Callable/detail::arg_descr/detail::return_descr) + if (*(pc + 1) == '^') { + is_return_value.emplace(false); + ++pc; + continue; + } + if (*(pc + 1) == '$') { + is_return_value.emplace(true); + ++pc; + continue; + } + if (*(pc + 1) == '!') { + is_return_value.pop(); + ++pc; + continue; + } + // Handle types that differ depending on whether they appear + // in an argument or a return value position (see io_name). + // For named arguments (py::arg()) with noconvert set, return value type is used. + ++pc; + if (!is_return_value.top() + && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) { + while (*pc != '\0' && *pc != '@') { + signature += *pc++; + } + if (*pc == '@') { + ++pc; + } + while (*pc != '\0' && *pc != '@') { + ++pc; + } + } else { + while (*pc != '\0' && *pc != '@') { + ++pc; + } + if (*pc == '@') { + ++pc; + } + while (*pc != '\0' && *pc != '@') { + signature += *pc++; + } + } } else { + if (c == '-' && *(pc + 1) == '>') { + is_return_value.emplace(true); + } signature += c; } } @@ -591,8 +650,7 @@ protected: // chain. chain_start = rec; rec->next = chain; - auto rec_capsule - = reinterpret_borrow(((PyCFunctionObject *) m_ptr)->m_self); + auto rec_capsule = reinterpret_borrow(PyCFunction_GET_SELF(m_ptr)); rec_capsule.set_pointer(unique_rec.release()); guarded_strdup.release(); } else { @@ -610,7 +668,8 @@ protected: int index = 0; /* Create a nice pydoc rec including all signatures and docstrings of the functions in the overload chain */ - if (chain && options::show_function_signatures()) { + if (chain && options::show_function_signatures() + && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) { // First a generic signature signatures += rec->name; signatures += "(*args, **kwargs)\n"; @@ -619,7 +678,8 @@ protected: // Then specific overload signatures bool first_user_def = true; for (auto *it = chain_start; it != nullptr; it = it->next) { - if (options::show_function_signatures()) { + if (options::show_function_signatures() + && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) { if (index > 0) { signatures += '\n'; } @@ -650,12 +710,11 @@ protected: } } - /* Install docstring */ auto *func = (PyCFunctionObject *) m_ptr; - std::free(const_cast(func->m_ml->ml_doc)); // Install docstring if it's non-empty (when at least one option is enabled) - func->m_ml->ml_doc - = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); + auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); + std::free(const_cast(PYBIND11_PYCFUNCTION_GET_DOC(func))); + PYBIND11_PYCFUNCTION_SET_DOC(func, doc); if (rec->is_method) { m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); @@ -1038,40 +1097,7 @@ protected: throw; #endif } catch (...) { - /* When an exception is caught, give each registered exception - translator a chance to translate it to a Python exception. First - all module-local translators will be tried in reverse order of - registration. If none of the module-locale translators handle - the exception (or there are no module-locale translators) then - the global translators will be tried, also in reverse order of - registration. - - A translator may choose to do one of the following: - - - catch the exception and call py::set_error() - to set a standard (or custom) Python exception, or - - do nothing and let the exception fall through to the next translator, or - - delegate translation to the next translator by throwing a new type of exception. - */ - - bool handled = with_internals([&](internals &internals) { - auto &local_exception_translators - = get_local_internals().registered_exception_translators; - if (detail::apply_exception_translators(local_exception_translators)) { - return true; - } - auto &exception_translators = internals.registered_exception_translators; - if (detail::apply_exception_translators(exception_translators)) { - return true; - } - return false; - }); - - if (handled) { - return nullptr; - } - - set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); + try_translate_exceptions(); return nullptr; } @@ -1356,7 +1382,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) template <> struct handle_type_name { - static constexpr auto name = const_name("module"); + static constexpr auto name = const_name("types.ModuleType"); }; PYBIND11_NAMESPACE_END(detail) @@ -1420,8 +1446,8 @@ protected: tinfo->dealloc = rec.dealloc; tinfo->simple_type = true; tinfo->simple_ancestors = true; - tinfo->default_holder = rec.default_holder; tinfo->module_local = rec.module_local; + tinfo->holder_enum_v = rec.holder_enum_v; with_internals([&](internals &internals) { auto tindex = std::type_index(*rec.type); @@ -1431,7 +1457,17 @@ protected: } else { internals.registered_types_cpp[tindex] = tinfo; } + + PYBIND11_WARNING_PUSH +#if defined(__GNUC__) && __GNUC__ == 12 + // When using GCC 12 these warnings are disabled as they trigger + // false positive warnings. Discussed here: + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115824. + PYBIND11_WARNING_DISABLE_GCC("-Warray-bounds") + PYBIND11_WARNING_DISABLE_GCC("-Wstringop-overread") +#endif internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo}; + PYBIND11_WARNING_POP }); if (rec.bases.size() > 1 || rec.multiple_inheritance) { @@ -1584,6 +1620,239 @@ auto method_adaptor(Return (Class::*pmf)(Args...) const) -> Return (Derived::*)( return pmf; } +PYBIND11_NAMESPACE_BEGIN(detail) + +// Helper for the property_cpp_function static member functions below. +// The only purpose of these functions is to support .def_readonly & .def_readwrite. +// In this context, the PM template parameter is certain to be a Pointer to a Member. +// The main purpose of must_be_member_function_pointer is to make this obvious, and to guard +// against accidents. As a side-effect, it also explains why the syntactical overhead for +// perfect forwarding is not needed. +template +using must_be_member_function_pointer = enable_if_t::value, int>; + +// Note that property_cpp_function is intentionally in the main pybind11 namespace, +// because user-defined specializations could be useful. + +// Classic (non-smart_holder) implementations for .def_readonly and .def_readwrite +// getter and setter functions. +// WARNING: This classic implementation can lead to dangling pointers for raw pointer members. +// See test_ptr() in tests/test_class_sh_property.py +// However, this implementation works as-is (and safely) for smart_holder std::shared_ptr members. +template +struct property_cpp_function_classic { + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + return cpp_function([pm](const T &c) -> const D & { return c.*pm; }, is_method(hdl)); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + return readonly(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl)); + } +}; + +PYBIND11_NAMESPACE_END(detail) + +template +struct property_cpp_function : detail::property_cpp_function_classic {}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct both_t_and_d_use_type_caster_base : std::false_type {}; + +// `T` is assumed to be equivalent to `intrinsic_t`. +// `D` is may or may not be equivalent to `intrinsic_t`. +template +struct both_t_and_d_use_type_caster_base< + T, + D, + enable_if_t, type_caster>, + std::is_base_of>, make_caster>>::value>> + : std::true_type {}; + +// Specialization for raw pointer members, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// WARNING: Like the classic implementation, this implementation can lead to dangling pointers. +// See test_ptr() in tests/test_class_sh_property.py +// However, the read functions return a shared_ptr to the member, emulating the PyCLIF approach: +// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233 +// This prevents disowning of the Python object owning the raw pointer member. +template +struct property_cpp_function_sh_raw_ptr_member { + using drp = typename std::remove_pointer::type; + + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + D ptr = (*c_sp).*pm; + return std::shared_ptr(c_sp, ptr); + }, + is_method(hdl)); + } + return property_cpp_function_classic::readonly(pm, hdl); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + return readonly(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function([pm](T &c, D value) { c.*pm = std::forward(std::move(value)); }, + is_method(hdl)); + } + return property_cpp_function_classic::write(pm, hdl); + } +}; + +// Specialization for members held by-value, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// The read functions return a shared_ptr to the member, emulating the PyCLIF approach: +// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233 +// This prevents disowning of the Python object owning the member. +template +struct property_cpp_function_sh_member_held_by_value { + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr::type> { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return std::shared_ptr::type>(c_sp, + &(c_sp.get()->*pm)); + }, + is_method(hdl)); + } + return property_cpp_function_classic::readonly(pm, hdl); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return std::shared_ptr(c_sp, &(c_sp.get()->*pm)); + }, + is_method(hdl)); + } + return property_cpp_function_classic::read(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl)); + } + return property_cpp_function_classic::write(pm, hdl); + } +}; + +// Specialization for std::unique_ptr members, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// read disowns the member unique_ptr. +// write disowns the passed Python object. +// readonly is disabled (static_assert) because there is no safe & intuitive way to make the member +// accessible as a Python object without disowning the member unique_ptr. A .def_readonly disowning +// the unique_ptr member is deemed highly prone to misunderstandings. +template +struct property_cpp_function_sh_unique_ptr_member { + template = 0> + static cpp_function readonly(PM, const handle &) { + static_assert(!is_instantiation::value, + "def_readonly cannot be used for std::unique_ptr members."); + return cpp_function{}; // Unreachable. + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> D { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return D{std::move(c_sp.get()->*pm)}; + }, + is_method(hdl)); + } + return property_cpp_function_classic::read(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + return cpp_function([pm](T &c, D &&value) { c.*pm = std::move(value); }, is_method(hdl)); + } +}; + +PYBIND11_NAMESPACE_END(detail) + +template +struct property_cpp_function< + T, + D, + detail::enable_if_t, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_raw_ptr_member {}; + +template +struct property_cpp_function, + std::is_array, + detail::is_instantiation, + detail::is_instantiation>, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_member_held_by_value {}; + +template +struct property_cpp_function< + T, + D, + detail::enable_if_t, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_unique_ptr_member {}; + +#ifdef PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE +// NOTE: THIS IS MEANT FOR STRESS-TESTING OR TRIAGING ONLY! +// Running the pybind11 unit tests with smart_holder as the default holder is to ensure +// that `py::smart_holder` / `py::classh` is backward-compatible with all pre-existing +// functionality. +// Be careful not to link translation units compiled with different default holders, because +// this will cause ODR violations (https://en.wikipedia.org/wiki/One_Definition_Rule). +template +using default_holder_type = smart_holder; +#else +template +using default_holder_type = std::unique_ptr; +#endif + template class class_ : public detail::generic_type { template @@ -1600,7 +1869,7 @@ public: using type = type_; using type_alias = detail::exactly_one_t; constexpr static bool has_alias = !std::is_void::value; - using holder_type = detail::exactly_one_t, options...>; + using holder_type = detail::exactly_one_t, options...>; static_assert(detail::all_of...>::value, "Unknown/invalid class_ template parameters provided"); @@ -1631,8 +1900,16 @@ public: record.type_align = alignof(conditional_t &); record.holder_size = sizeof(holder_type); record.init_instance = init_instance; - record.dealloc = dealloc; - record.default_holder = detail::is_instantiation::value; + + if (detail::is_instantiation::value) { + record.holder_enum_v = detail::holder_enum_t::std_unique_ptr; + } else if (detail::is_instantiation::value) { + record.holder_enum_v = detail::holder_enum_t::std_shared_ptr; + } else if (std::is_same::value) { + record.holder_enum_v = detail::holder_enum_t::smart_holder; + } else { + record.holder_enum_v = detail::holder_enum_t::custom_holder; + } set_operator_new(&record); @@ -1642,6 +1919,12 @@ public: /* Process optional arguments, if any */ process_attributes::init(extra..., &record); + if (record.release_gil_before_calling_cpp_dtor) { + record.dealloc = dealloc_release_gil_before_calling_cpp_dtor; + } else { + record.dealloc = dealloc_without_manipulating_gil; + } + generic_type::initialize(record); if (has_alias) { @@ -1652,6 +1935,7 @@ public: = instances[std::type_index(typeid(type))]; }); } + def("_pybind11_conduit_v1_", cpp_conduit_method); } template ::value, int> = 0> @@ -1764,9 +2048,11 @@ public: class_ &def_readwrite(const char *name, D C::*pm, const Extra &...extra) { static_assert(std::is_same::value || std::is_base_of::value, "def_readwrite() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)), - fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this)); - def_property(name, fget, fset, return_value_policy::reference_internal, extra...); + def_property(name, + property_cpp_function::read(pm, *this), + property_cpp_function::write(pm, *this), + return_value_policy::reference_internal, + extra...); return *this; } @@ -1774,8 +2060,10 @@ public: class_ &def_readonly(const char *name, const D C::*pm, const Extra &...extra) { static_assert(std::is_same::value || std::is_base_of::value, "def_readonly() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)); - def_property_readonly(name, fget, return_value_policy::reference_internal, extra...); + def_property_readonly(name, + property_cpp_function::readonly(pm, *this), + return_value_policy::reference_internal, + extra...); return *this; } @@ -1952,6 +2240,8 @@ private: /// instance. Should be called as soon as the `type` value_ptr is set for an instance. Takes /// an optional pointer to an existing holder to use; if not specified and the instance is /// `.owned`, a new holder will be constructed to manage the value pointer. + template ::value, int> = 0> static void init_instance(detail::instance *inst, const void *holder_ptr) { auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type))); if (!v_h.instance_registered()) { @@ -1961,15 +2251,73 @@ private: init_holder(inst, v_h, (const holder_type *) holder_ptr, v_h.value_ptr()); } - /// Deallocates an instance; via holder, if constructed; otherwise via operator delete. - static void dealloc(detail::value_and_holder &v_h) { - // We could be deallocating because we are cleaning up after a Python exception. - // If so, the Python error indicator will be set. We need to clear that before - // running the destructor, in case the destructor code calls more Python. - // If we don't, the Python API will exit with an exception, and pybind11 will - // throw error_already_set from the C++ destructor which is forbidden and triggers - // std::terminate(). - error_scope scope; + template + static bool try_initialization_using_shared_from_this(holder_type *, WrappedType *, ...) { + return false; + } + + // Adopting existing approach used by type_caster_base, although it leads to somewhat fuzzy + // ownership semantics: if we detected via shared_from_this that a shared_ptr exists already, + // it is reused, irrespective of the return_value_policy in effect. + // "SomeBaseOfWrappedType" is needed because std::enable_shared_from_this is not necessarily a + // direct base of WrappedType. + template + static bool try_initialization_using_shared_from_this( + holder_type *uninitialized_location, + WrappedType *value_ptr_w_t, + const std::enable_shared_from_this *) { + auto shd_ptr = std::dynamic_pointer_cast( + detail::try_get_shared_from_this(value_ptr_w_t)); + if (!shd_ptr) { + return false; + } + // Note: inst->owned ignored. + new (uninitialized_location) holder_type(holder_type::from_shared_ptr(shd_ptr)); + return true; + } + + template ::value, int> = 0> + static void init_instance(detail::instance *inst, const void *holder_const_void_ptr) { + // Need for const_cast is a consequence of the type_info::init_instance type: + // void (*init_instance)(instance *, const void *); + auto *holder_void_ptr = const_cast(holder_const_void_ptr); + + auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type))); + if (!v_h.instance_registered()) { + register_instance(inst, v_h.value_ptr(), v_h.type); + v_h.set_instance_registered(); + } + auto *uninitialized_location = std::addressof(v_h.holder()); + auto *value_ptr_w_t = v_h.value_ptr(); + // Try downcast from `type` to `type_alias`: + inst->is_alias + = detail::dynamic_raw_ptr_cast_if_possible(value_ptr_w_t) != nullptr; + if (holder_void_ptr) { + // Note: inst->owned ignored. + auto *holder_ptr = static_cast(holder_void_ptr); + new (uninitialized_location) holder_type(std::move(*holder_ptr)); + } else if (!try_initialization_using_shared_from_this( + uninitialized_location, value_ptr_w_t, value_ptr_w_t)) { + if (inst->owned) { + new (uninitialized_location) holder_type(holder_type::from_raw_ptr_take_ownership( + value_ptr_w_t, /*void_cast_raw_ptr*/ inst->is_alias)); + } else { + new (uninitialized_location) + holder_type(holder_type::from_raw_ptr_unowned(value_ptr_w_t)); + } + } + v_h.set_holder_constructed(); + } + + // Deallocates an instance; via holder, if constructed; otherwise via operator delete. + // NOTE: The Python error indicator needs to cleared BEFORE this function is called. + // This is because we could be deallocating while cleaning up after a Python exception. + // If the error indicator is not cleared but the C++ destructor code makes Python C API + // calls, those calls are likely to generate a new exception, and pybind11 will then + // throw `error_already_set` from the C++ destructor. This is forbidden and will + // trigger std::terminate(). + static void dealloc_impl(detail::value_and_holder &v_h) { if (v_h.holder_constructed()) { v_h.holder().~holder_type(); v_h.set_holder_constructed(false); @@ -1980,6 +2328,32 @@ private: v_h.value_ptr() = nullptr; } + static void dealloc_without_manipulating_gil(detail::value_and_holder &v_h) { + error_scope scope; + dealloc_impl(v_h); + } + + static void dealloc_release_gil_before_calling_cpp_dtor(detail::value_and_holder &v_h) { + error_scope scope; + // Intentionally not using `gil_scoped_release` because the non-simple + // version unconditionally calls `get_internals()`. + // `Py_BEGIN_ALLOW_THREADS`, `Py_END_ALLOW_THREADS` cannot be used + // because those macros include `{` and `}`. + PyThreadState *py_ts = PyEval_SaveThread(); + try { + dealloc_impl(v_h); + } catch (...) { + // This code path is expected to be unreachable unless there is a + // bug in pybind11 itself. + // An alternative would be to mark this function, or + // `dealloc_impl()`, with `nothrow`, but that would be a subtle + // behavior change and could make debugging more difficult. + PyEval_RestoreThread(py_ts); + throw; + } + PyEval_RestoreThread(py_ts); + } + static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); if (!h) { @@ -2001,6 +2375,11 @@ private: } }; +// Supports easier switching between py::class_ and py::class_: +// users can simply replace the `_` in `class_` with `h` or vice versa. +template +using classh = class_; + /// Binds an existing constructor taking arguments Args... template detail::initimpl::constructor init() { @@ -2062,9 +2441,11 @@ struct enum_base { .format(std::move(type_name), enum_name(arg), int_(arg)); }, name("__repr__"), - is_method(m_base)); + is_method(m_base), + pos_only()); - m_base.attr("name") = property(cpp_function(&enum_name, name("name"), is_method(m_base))); + m_base.attr("name") + = property(cpp_function(&enum_name, name("name"), is_method(m_base), pos_only())); m_base.attr("__str__") = cpp_function( [](handle arg) -> str { @@ -2072,7 +2453,8 @@ struct enum_base { return pybind11::str("{}.{}").format(std::move(type_name), enum_name(arg)); }, name("__str__"), - is_method(m_base)); + is_method(m_base), + pos_only()); if (options::show_enum_members_docstring()) { m_base.attr("__doc__") = static_property( @@ -2127,7 +2509,8 @@ struct enum_base { }, \ name(op), \ is_method(m_base), \ - arg("other")) + arg("other"), \ + pos_only()) #define PYBIND11_ENUM_OP_CONV(op, expr) \ m_base.attr(op) = cpp_function( \ @@ -2137,7 +2520,8 @@ struct enum_base { }, \ name(op), \ is_method(m_base), \ - arg("other")) + arg("other"), \ + pos_only()) #define PYBIND11_ENUM_OP_CONV_LHS(op, expr) \ m_base.attr(op) = cpp_function( \ @@ -2147,7 +2531,8 @@ struct enum_base { }, \ name(op), \ is_method(m_base), \ - arg("other")) + arg("other"), \ + pos_only()) if (is_convertible) { PYBIND11_ENUM_OP_CONV_LHS("__eq__", !b.is_none() && a.equal(b)); @@ -2167,7 +2552,8 @@ struct enum_base { m_base.attr("__invert__") = cpp_function([](const object &arg) { return ~(int_(arg)); }, name("__invert__"), - is_method(m_base)); + is_method(m_base), + pos_only()); } } else { PYBIND11_ENUM_OP_STRICT("__eq__", int_(a).equal(int_(b)), return false); @@ -2187,11 +2573,15 @@ struct enum_base { #undef PYBIND11_ENUM_OP_CONV #undef PYBIND11_ENUM_OP_STRICT - m_base.attr("__getstate__") = cpp_function( - [](const object &arg) { return int_(arg); }, name("__getstate__"), is_method(m_base)); + m_base.attr("__getstate__") = cpp_function([](const object &arg) { return int_(arg); }, + name("__getstate__"), + is_method(m_base), + pos_only()); - m_base.attr("__hash__") = cpp_function( - [](const object &arg) { return int_(arg); }, name("__hash__"), is_method(m_base)); + m_base.attr("__hash__") = cpp_function([](const object &arg) { return int_(arg); }, + name("__hash__"), + is_method(m_base), + pos_only()); } PYBIND11_NOINLINE void value(char const *name_, object value, const char *doc = nullptr) { @@ -2283,9 +2673,9 @@ public: m_base.init(is_arithmetic, is_convertible); def(init([](Scalar i) { return static_cast(i); }), arg("value")); - def_property_readonly("value", [](Type value) { return (Scalar) value; }); - def("__int__", [](Type value) { return (Scalar) value; }); - def("__index__", [](Type value) { return (Scalar) value; }); + def_property_readonly("value", [](Type value) { return (Scalar) value; }, pos_only()); + def("__int__", [](Type value) { return (Scalar) value; }, pos_only()); + def("__index__", [](Type value) { return (Scalar) value; }, pos_only()); attr("__setstate__") = cpp_function( [](detail::value_and_holder &v_h, Scalar arg) { detail::initimpl::setstate( @@ -2294,7 +2684,8 @@ public: detail::is_new_style_constructor(), pybind11::name("__setstate__"), is_method(*this), - arg("state")); + arg("state"), + pos_only()); } /// Export enumeration entries into the parent scope @@ -2366,13 +2757,20 @@ keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) { inline std::pair all_type_info_get_cache(PyTypeObject *type) { auto res = with_internals([type](internals &internals) { - return internals - .registered_types_py + auto ins = internals + .registered_types_py #ifdef __cpp_lib_unordered_map_try_emplace - .try_emplace(type); + .try_emplace(type); #else - .emplace(type, std::vector()); + .emplace(type, std::vector()); #endif + if (ins.second) { + // For free-threading mode, this call must be under + // the with_internals() mutex lock, to avoid that other threads + // continue running with the empty ins.first->second. + all_type_info_populate(type, ins.first->second); + } + return ins; }); if (res.second) { // New cache entry created; set up a weak reference to automatically remove it if the type @@ -2473,7 +2871,8 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { if (!detail::get_type_info(typeid(state), false)) { class_(handle(), "iterator", pybind11::module_local()) - .def("__iter__", [](state &s) -> state & { return s; }) + .def( + "__iter__", [](state &s) -> state & { return s; }, pos_only()) .def( "__next__", [](state &s) -> ValueType { @@ -2490,6 +2889,7 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 }, std::forward(extra)..., + pos_only(), Policy); } @@ -2624,10 +3024,12 @@ void implicitly_convertible() { } inline void register_exception_translator(ExceptionTranslator &&translator) { - detail::with_internals([&](detail::internals &internals) { - internals.registered_exception_translators.push_front( - std::forward(translator)); - }); + detail::with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + (void) local_exception_translators; + exception_translators.push_front(std::forward(translator)); + }); } /** @@ -2637,11 +3039,12 @@ inline void register_exception_translator(ExceptionTranslator &&translator) { * the exception. */ inline void register_local_exception_translator(ExceptionTranslator &&translator) { - detail::with_internals([&](detail::internals &internals) { - (void) internals; - detail::get_local_internals().registered_exception_translators.push_front( - std::forward(translator)); - }); + detail::with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + (void) exception_translators; + local_exception_translators.push_front(std::forward(translator)); + }); } /** @@ -2815,8 +3218,8 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * } /* Don't call dispatch code if invoked from overridden function. - Unfortunately this doesn't work on PyPy. */ -#if !defined(PYPY_VERSION) + Unfortunately this doesn't work on PyPy and GraalPy. */ +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) # if PY_VERSION_HEX >= 0x03090000 PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); if (frame != nullptr) { diff --git a/wrap/pybind11/include/pybind11/pytypes.h b/wrap/pybind11/include/pybind11/pytypes.h index f26c307a8..92e0a81f4 100644 --- a/wrap/pybind11/include/pybind11/pytypes.h +++ b/wrap/pybind11/include/pybind11/pytypes.h @@ -113,6 +113,17 @@ public: /// See above (the only difference is that the key is provided as a string literal) str_attr_accessor attr(const char *key) const; + /** \rst + Similar to the above attr functions with the difference that the templated Type + is used to set the `__annotations__` dict value to the corresponding key. Worth noting + that attr_with_type_hint is implemented in cast.h. + \endrst */ + template + obj_attr_accessor attr_with_type_hint(handle key) const; + /// See above (the only difference is that the key is provided as a string literal) + template + str_attr_accessor attr_with_type_hint(const char *key) const; + /** \rst Matches * unpacking in Python, e.g. to unpack arguments out of a ``tuple`` or ``list`` for a function call. Applying another * to the result yields @@ -182,6 +193,9 @@ public: /// Get or set the object's docstring, i.e. ``obj.__doc__``. str_attr_accessor doc() const; + /// Get or set the object's annotations, i.e. ``obj.__annotations__``. + object annotations() const; + /// Return the object's current reference count ssize_t ref_count() const { #ifdef PYPY_VERSION @@ -643,7 +657,7 @@ struct error_fetch_and_normalize { bool have_trace = false; if (m_trace) { -#if !defined(PYPY_VERSION) +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) auto *tb = reinterpret_cast(m_trace.ptr()); // Get the deepest trace possible. @@ -1259,6 +1273,7 @@ protected: using pointer = arrow_proxy; sequence_fast_readonly(handle obj, ssize_t n) : ptr(PySequence_Fast_ITEMS(obj.ptr()) + n) {} + sequence_fast_readonly() = default; // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference dereference() const { return *ptr; } @@ -1281,6 +1296,7 @@ protected: using pointer = arrow_proxy; sequence_slow_readwrite(handle obj, ssize_t index) : obj(obj), index(index) {} + sequence_slow_readwrite() = default; reference dereference() const { return {obj, static_cast(index)}; } void increment() { ++index; } @@ -1354,7 +1370,7 @@ inline bool PyUnicode_Check_Permissive(PyObject *o) { # define PYBIND11_STR_CHECK_FUN PyUnicode_Check #endif -inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; } +inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == &PyStaticMethod_Type; } class kwargs_proxy : public handle { public: @@ -1468,11 +1484,17 @@ public: PYBIND11_OBJECT_DEFAULT(iterator, object, PyIter_Check) iterator &operator++() { + init(); advance(); return *this; } iterator operator++(int) { + // Note: We must call init() first so that rv.value is + // the same as this->value just before calling advance(). + // Otherwise, dereferencing the returned iterator may call + // advance() again and return the 3rd item instead of the 1st. + init(); auto rv = *this; advance(); return rv; @@ -1480,15 +1502,12 @@ public: // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference operator*() const { - if (m_ptr && !value.ptr()) { - auto &self = const_cast(*this); - self.advance(); - } + init(); return value; } pointer operator->() const { - operator*(); + init(); return &value; } @@ -1511,6 +1530,13 @@ public: friend bool operator!=(const iterator &a, const iterator &b) { return a->ptr() != b->ptr(); } private: + void init() const { + if (m_ptr && !value.ptr()) { + auto &self = const_cast(*this); + self.advance(); + } + } + void advance() { value = reinterpret_steal(PyIter_Next(m_ptr)); if (value.ptr() == nullptr && PyErr_Occurred()) { @@ -2214,6 +2240,18 @@ class kwargs : public dict { PYBIND11_OBJECT_DEFAULT(kwargs, dict, PyDict_Check) }; +// Subclasses of args and kwargs to support type hinting +// as defined in PEP 484. See #5357 for more info. +template +class Args : public args { + using args::args; +}; + +template +class KWArgs : public kwargs { + using kwargs::kwargs; +}; + class anyset : public object { public: PYBIND11_OBJECT(anyset, object, PyAnySet_Check) @@ -2534,6 +2572,19 @@ str_attr_accessor object_api::doc() const { return attr("__doc__"); } +template +object object_api::annotations() const { +#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION <= 9 + // https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older + if (!hasattr(derived(), "__annotations__")) { + setattr(derived(), "__annotations__", dict()); + } + return attr("__annotations__"); +#else + return getattr(derived(), "__annotations__", dict()); +#endif +} + template handle object_api::get_type() const { return type::handle_of(derived()); diff --git a/wrap/pybind11/include/pybind11/stl.h b/wrap/pybind11/include/pybind11/stl.h index 71bc5902e..6a148e740 100644 --- a/wrap/pybind11/include/pybind11/stl.h +++ b/wrap/pybind11/include/pybind11/stl.h @@ -11,10 +11,14 @@ #include "pybind11.h" #include "detail/common.h" +#include "detail/descr.h" +#include "detail/type_caster_base.h" #include +#include #include #include +#include #include #include #include @@ -35,6 +39,89 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +// +// Begin: Equivalent of +// https://github.com/google/clif/blob/ae4eee1de07cdf115c0c9bf9fec9ff28efce6f6c/clif/python/runtime.cc#L388-L438 +/* +The three `PyObjectTypeIsConvertibleTo*()` functions below are +the result of converging the behaviors of pybind11 and PyCLIF +(http://github.com/google/clif). + +Originally PyCLIF was extremely far on the permissive side of the spectrum, +while pybind11 was very far on the strict side. Originally PyCLIF accepted any +Python iterable as input for a C++ `vector`/`set`/`map` argument, as long as +the elements were convertible. The obvious (in hindsight) problem was that +any empty Python iterable could be passed to any of these C++ types, e.g. `{}` +was accepted for C++ `vector`/`set` arguments, or `[]` for C++ `map` arguments. + +The functions below strike a practical permissive-vs-strict compromise, +informed by tens of thousands of use cases in the wild. A main objective is +to prevent accidents and improve readability: + +- Python literals must match the C++ types. + +- For C++ `set`: The potentially reducing conversion from a Python sequence + (e.g. Python `list` or `tuple`) to a C++ `set` must be explicit, by going + through a Python `set`. + +- However, a Python `set` can still be passed to a C++ `vector`. The rationale + is that this conversion is not reducing. Implicit conversions of this kind + are also fairly commonly used, therefore enforcing explicit conversions + would have an unfavorable cost : benefit ratio; more sloppily speaking, + such an enforcement would be more annoying than helpful. +*/ + +inline bool PyObjectIsInstanceWithOneOfTpNames(PyObject *obj, + std::initializer_list tp_names) { + if (PyType_Check(obj)) { + return false; + } + const char *obj_tp_name = Py_TYPE(obj)->tp_name; + for (const auto *tp_name : tp_names) { + if (std::strcmp(obj_tp_name, tp_name) == 0) { + return true; + } + } + return false; +} + +inline bool PyObjectTypeIsConvertibleToStdVector(PyObject *obj) { + if (PySequence_Check(obj) != 0) { + return !PyUnicode_Check(obj) && !PyBytes_Check(obj); + } + return (PyGen_Check(obj) != 0) || (PyAnySet_Check(obj) != 0) + || PyObjectIsInstanceWithOneOfTpNames( + obj, {"dict_keys", "dict_values", "dict_items", "map", "zip"}); +} + +inline bool PyObjectTypeIsConvertibleToStdSet(PyObject *obj) { + return (PyAnySet_Check(obj) != 0) || PyObjectIsInstanceWithOneOfTpNames(obj, {"dict_keys"}); +} + +inline bool PyObjectTypeIsConvertibleToStdMap(PyObject *obj) { + if (PyDict_Check(obj)) { + return true; + } + // Implicit requirement in the conditions below: + // A type with `.__getitem__()` & `.items()` methods must implement these + // to be compatible with https://docs.python.org/3/c-api/mapping.html + if (PyMapping_Check(obj) == 0) { + return false; + } + PyObject *items = PyObject_GetAttrString(obj, "items"); + if (items == nullptr) { + PyErr_Clear(); + return false; + } + bool is_convertible = (PyCallable_Check(items) != 0); + Py_DECREF(items); + return is_convertible; +} + +// +// End: Equivalent of clif/python/runtime.cc +// + /// Extracts an const lvalue reference or rvalue reference for U based on the type of T (e.g. for /// forwarding a container element). Typically used indirect via forwarded_type(), below. template @@ -66,17 +153,10 @@ private: } void reserve_maybe(const anyset &, void *) {} -public: - bool load(handle src, bool convert) { - if (!isinstance(src)) { - return false; - } - auto s = reinterpret_borrow(src); - value.clear(); - reserve_maybe(s, &value); - for (auto entry : s) { + bool convert_iterable(const iterable &itbl, bool convert) { + for (const auto &it : itbl) { key_conv conv; - if (!conv.load(entry, convert)) { + if (!conv.load(it, convert)) { return false; } value.insert(cast_op(std::move(conv))); @@ -84,6 +164,29 @@ public: return true; } + bool convert_anyset(anyset s, bool convert) { + value.clear(); + reserve_maybe(s, &value); + return convert_iterable(s, convert); + } + +public: + bool load(handle src, bool convert) { + if (!PyObjectTypeIsConvertibleToStdSet(src.ptr())) { + return false; + } + if (isinstance(src)) { + value.clear(); + return convert_anyset(reinterpret_borrow(src), convert); + } + if (!convert) { + return false; + } + assert(isinstance(src)); + value.clear(); + return convert_iterable(reinterpret_borrow(src), convert); + } + template static handle cast(T &&src, return_value_policy policy, handle parent) { if (!std::is_lvalue_reference::value) { @@ -115,15 +218,10 @@ private: } void reserve_maybe(const dict &, void *) {} -public: - bool load(handle src, bool convert) { - if (!isinstance(src)) { - return false; - } - auto d = reinterpret_borrow(src); + bool convert_elements(const dict &d, bool convert) { value.clear(); reserve_maybe(d, &value); - for (auto it : d) { + for (const auto &it : d) { key_conv kconv; value_conv vconv; if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) { @@ -134,6 +232,25 @@ public: return true; } +public: + bool load(handle src, bool convert) { + if (!PyObjectTypeIsConvertibleToStdMap(src.ptr())) { + return false; + } + if (isinstance(src)) { + return convert_elements(reinterpret_borrow(src), convert); + } + if (!convert) { + return false; + } + auto items = reinterpret_steal(PyMapping_Items(src.ptr())); + if (!items) { + throw error_already_set(); + } + assert(isinstance(items)); + return convert_elements(dict(reinterpret_borrow(items)), convert); + } + template static handle cast(T &&src, return_value_policy policy, handle parent) { dict d; @@ -166,20 +283,21 @@ struct list_caster { using value_conv = make_caster; bool load(handle src, bool convert) { - if (!isinstance(src) || isinstance(src) || isinstance(src)) { + if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) { return false; } - auto s = reinterpret_borrow(src); - value.clear(); - reserve_maybe(s, &value); - for (const auto &it : s) { - value_conv conv; - if (!conv.load(it, convert)) { - return false; - } - value.push_back(cast_op(std::move(conv))); + if (isinstance(src)) { + return convert_elements(src, convert); } - return true; + if (!convert) { + return false; + } + // Designed to be behavior-equivalent to passing tuple(src) from Python: + // The conversion to a tuple will first exhaust the generator object, to ensure that + // the generator is not left in an unpredictable (to the caller) partially-consumed + // state. + assert(isinstance(src)); + return convert_elements(tuple(reinterpret_borrow(src)), convert); } private: @@ -189,6 +307,20 @@ private: } void reserve_maybe(const sequence &, void *) {} + bool convert_elements(handle seq, bool convert) { + auto s = reinterpret_borrow(seq); + value.clear(); + reserve_maybe(s, &value); + for (const auto &it : seq) { + value_conv conv; + if (!conv.load(it, convert)) { + return false; + } + value.push_back(cast_op(std::move(conv))); + } + return true; + } + public: template static handle cast(T &&src, return_value_policy policy, handle parent) { @@ -220,43 +352,87 @@ struct type_caster> : list_caster struct type_caster> : list_caster, Type> {}; +template +ArrayType vector_to_array_impl(V &&v, index_sequence) { + return {{std::move(v[I])...}}; +} + +// Based on https://en.cppreference.com/w/cpp/container/array/to_array +template +ArrayType vector_to_array(V &&v) { + return vector_to_array_impl(std::forward(v), make_index_sequence{}); +} + template struct array_caster { using value_conv = make_caster; private: - template - bool require_size(enable_if_t size) { - if (value.size() != size) { - value.resize(size); - } - return true; - } - template - bool require_size(enable_if_t size) { - return size == Size; - } + std::unique_ptr value; -public: - bool load(handle src, bool convert) { - if (!isinstance(src)) { - return false; - } - auto l = reinterpret_borrow(src); - if (!require_size(l.size())) { - return false; - } + template = 0> + bool convert_elements(handle seq, bool convert) { + auto l = reinterpret_borrow(seq); + value.reset(new ArrayType{}); + // Using `resize` to preserve the behavior exactly as it was before PR #5305 + // For the `resize` to work, `Value` must be default constructible. + // For `std::valarray`, this is a requirement: + // https://en.cppreference.com/w/cpp/named_req/NumericType + value->resize(l.size()); size_t ctr = 0; for (const auto &it : l) { value_conv conv; if (!conv.load(it, convert)) { return false; } - value[ctr++] = cast_op(std::move(conv)); + (*value)[ctr++] = cast_op(std::move(conv)); } return true; } + template = 0> + bool convert_elements(handle seq, bool convert) { + auto l = reinterpret_borrow(seq); + if (l.size() != Size) { + return false; + } + // The `temp` storage is needed to support `Value` types that are not + // default-constructible. + // Deliberate choice: no template specializations, for simplicity, and + // because the compile time overhead for the specializations is deemed + // more significant than the runtime overhead for the `temp` storage. + std::vector temp; + temp.reserve(l.size()); + for (auto it : l) { + value_conv conv; + if (!conv.load(it, convert)) { + return false; + } + temp.emplace_back(cast_op(std::move(conv))); + } + value.reset(new ArrayType(vector_to_array(std::move(temp)))); + return true; + } + +public: + bool load(handle src, bool convert) { + if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) { + return false; + } + if (isinstance(src)) { + return convert_elements(src, convert); + } + if (!convert) { + return false; + } + // Designed to be behavior-equivalent to passing tuple(src) from Python: + // The conversion to a tuple will first exhaust the generator object, to ensure that + // the generator is not left in an unpredictable (to the caller) partially-consumed + // state. + assert(isinstance(src)); + return convert_elements(tuple(reinterpret_borrow(src)), convert); + } + template static handle cast(T &&src, return_value_policy policy, handle parent) { list l(src.size()); @@ -272,12 +448,36 @@ public: return l.release(); } - PYBIND11_TYPE_CASTER(ArrayType, - const_name(const_name(""), const_name("Annotated[")) - + const_name("list[") + value_conv::name + const_name("]") - + const_name(const_name(""), - const_name(", FixedSize(") - + const_name() + const_name(")]"))); + // Code copied from PYBIND11_TYPE_CASTER macro. + // Intentionally preserving the behavior exactly as it was before PR #5305 + template >::value, int> = 0> + static handle cast(T_ *src, return_value_policy policy, handle parent) { + if (!src) { + return none().release(); + } + if (policy == return_value_policy::take_ownership) { + auto h = cast(std::move(*src), policy, parent); + delete src; // WARNING: Assumes `src` was allocated with `new`. + return h; + } + return cast(*src, policy, parent); + } + + // NOLINTNEXTLINE(google-explicit-constructor) + operator ArrayType *() { return &(*value); } + // NOLINTNEXTLINE(google-explicit-constructor) + operator ArrayType &() { return *value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator ArrayType &&() && { return std::move(*value); } + + template + using cast_op_type = movable_cast_op_type; + + static constexpr auto name + = const_name(const_name(""), const_name("Annotated[")) + const_name("list[") + + value_conv::name + const_name("]") + + const_name( + const_name(""), const_name(", FixedSize(") + const_name() + const_name(")]")); }; template diff --git a/wrap/pybind11/include/pybind11/stl/filesystem.h b/wrap/pybind11/include/pybind11/stl/filesystem.h index 85c131efe..fb8164e0d 100644 --- a/wrap/pybind11/include/pybind11/stl/filesystem.h +++ b/wrap/pybind11/include/pybind11/stl/filesystem.h @@ -4,11 +4,11 @@ #pragma once -#include "../pybind11.h" -#include "../detail/common.h" -#include "../detail/descr.h" -#include "../cast.h" -#include "../pytypes.h" +#include +#include +#include +#include +#include #include @@ -33,6 +33,13 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +#ifdef PYPY_VERSION +# define PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(...) (__VA_ARGS__) +#else +# define PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(...) \ + (reinterpret_cast(__VA_ARGS__)) +#endif + #if defined(PYBIND11_HAS_FILESYSTEM) || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) template struct path_caster { @@ -72,7 +79,8 @@ public: } PyObject *native = nullptr; if constexpr (std::is_same_v) { - if (PyUnicode_FSConverter(buf, &native) != 0) { + if (PyUnicode_FSConverter(buf, PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(&native)) + != 0) { if (auto *c_str = PyBytes_AsString(native)) { // AsString returns a pointer to the internal buffer, which // must not be free'd. @@ -80,7 +88,8 @@ public: } } } else if constexpr (std::is_same_v) { - if (PyUnicode_FSDecoder(buf, &native) != 0) { + if (PyUnicode_FSDecoder(buf, PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(&native)) + != 0) { if (auto *c_str = PyUnicode_AsWideCharString(native, nullptr)) { // AsWideCharString returns a new string that must be free'd. value = c_str; // Copies the string. @@ -97,7 +106,7 @@ public: return true; } - PYBIND11_TYPE_CASTER(T, const_name("os.PathLike")); + PYBIND11_TYPE_CASTER(T, io_name("Union[os.PathLike, str, bytes]", "pathlib.Path")); }; #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) diff --git a/wrap/pybind11/include/pybind11/stl_bind.h b/wrap/pybind11/include/pybind11/stl_bind.h index 66c452ea7..3eb1e53f4 100644 --- a/wrap/pybind11/include/pybind11/stl_bind.h +++ b/wrap/pybind11/include/pybind11/stl_bind.h @@ -180,7 +180,7 @@ void vector_modifiers( v.end()); try { v.shrink_to_fit(); - } catch (const std::exception &) { + } catch (const std::exception &) { // NOLINT(bugprone-empty-catch) // Do nothing } throw; @@ -487,7 +487,7 @@ PYBIND11_NAMESPACE_END(detail) // // std::vector // -template , typename... Args> +template , typename... Args> class_ bind_vector(handle scope, std::string const &name, Args &&...args) { using Class_ = class_; @@ -694,9 +694,43 @@ struct ItemsViewImpl : public detail::items_view { Map ↦ }; +inline str format_message_key_error_key_object(handle py_key) { + str message = "pybind11::bind_map key"; + if (!py_key) { + return message; + } + try { + message = str(py_key); + } catch (const std::exception &) { + try { + message = repr(py_key); + } catch (const std::exception &) { + return message; + } + } + const ssize_t cut_length = 100; + if (len(message) > 2 * cut_length + 3) { + return str(message[slice(0, cut_length, 1)]) + str("✄✄✄") + + str(message[slice(-cut_length, static_cast(len(message)), 1)]); + } + return message; +} + +template +str format_message_key_error(const KeyType &key) { + object py_key; + try { + py_key = cast(key); + } catch (const std::exception &) { + do { // Trick to avoid "empty catch" warning/error. + } while (false); + } + return format_message_key_error_key_object(py_key); +} + PYBIND11_NAMESPACE_END(detail) -template , typename... Args> +template , typename... Args> class_ bind_map(handle scope, const std::string &name, Args &&...args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; @@ -785,7 +819,8 @@ class_ bind_map(handle scope, const std::string &name, Args && [](Map &m, const KeyType &k) -> MappedType & { auto it = m.find(k); if (it == m.end()) { - throw key_error(); + set_error(PyExc_KeyError, detail::format_message_key_error(k)); + throw error_already_set(); } return it->second; }, @@ -808,7 +843,8 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def("__delitem__", [](Map &m, const KeyType &k) { auto it = m.find(k); if (it == m.end()) { - throw key_error(); + set_error(PyExc_KeyError, detail::format_message_key_error(k)); + throw error_already_set(); } m.erase(it); }); diff --git a/wrap/pybind11/include/pybind11/trampoline_self_life_support.h b/wrap/pybind11/include/pybind11/trampoline_self_life_support.h new file mode 100644 index 000000000..484045bb1 --- /dev/null +++ b/wrap/pybind11/include/pybind11/trampoline_self_life_support.h @@ -0,0 +1,60 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "detail/common.h" +#include "detail/using_smart_holder.h" +#include "detail/value_and_holder.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) +// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code. +inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); +PYBIND11_NAMESPACE_END(detail) + +// The original core idea for this struct goes back to PyCLIF: +// https://github.com/google/clif/blob/07f95d7e69dca2fcf7022978a55ef3acff506c19/clif/python/runtime.cc#L37 +// URL provided here mainly to give proper credit. +struct trampoline_self_life_support { + detail::value_and_holder v_h; + + trampoline_self_life_support() = default; + + void activate_life_support(const detail::value_and_holder &v_h_) { + Py_INCREF((PyObject *) v_h_.inst); + v_h = v_h_; + } + + void deactivate_life_support() { + Py_DECREF((PyObject *) v_h.inst); + v_h = detail::value_and_holder(); + } + + ~trampoline_self_life_support() { + if (v_h.inst != nullptr && v_h.vh != nullptr) { + void *value_void_ptr = v_h.value_ptr(); + if (value_void_ptr != nullptr) { + PyGILState_STATE threadstate = PyGILState_Ensure(); + v_h.value_ptr() = nullptr; + v_h.holder().release_disowned(); + detail::deregister_instance(v_h.inst, value_void_ptr, v_h.type); + Py_DECREF((PyObject *) v_h.inst); // Must be after deregister. + PyGILState_Release(threadstate); + } + } + } + + // For the next two, the default implementations generate undefined behavior (ASAN failures + // manually verified). The reason is that v_h needs to be kept default-initialized. + trampoline_self_life_support(const trampoline_self_life_support &) {} + trampoline_self_life_support(trampoline_self_life_support &&) noexcept {} + + // These should never be needed (please provide test cases if you think they are). + trampoline_self_life_support &operator=(const trampoline_self_life_support &) = delete; + trampoline_self_life_support &operator=(trampoline_self_life_support &&) = delete; +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/include/pybind11/typing.h b/wrap/pybind11/include/pybind11/typing.h index 1442cdc7f..c5f342aed 100644 --- a/wrap/pybind11/include/pybind11/typing.h +++ b/wrap/pybind11/include/pybind11/typing.h @@ -14,6 +14,15 @@ #include "cast.h" #include "pytypes.h" +#include + +#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L +# define PYBIND11_TYPING_H_HAS_STRING_LITERAL +# include +# include +# include +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(typing) @@ -80,6 +89,18 @@ class Optional : public object { using object::object; }; +template +class Final : public object { + PYBIND11_OBJECT_DEFAULT(Final, object, PyObject_Type) + using object::object; +}; + +template +class ClassVar : public object { + PYBIND11_OBJECT_DEFAULT(ClassVar, object, PyObject_Type) + using object::object; +}; + template class TypeGuard : public bool_ { using bool_::bool_; @@ -98,7 +119,7 @@ class Never : public none { using none::none; }; -#if defined(__cpp_nontype_template_parameter_class) +#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) template struct StringLiteral { constexpr StringLiteral(const char (&str)[N]) { std::copy_n(str, N, name); } @@ -173,16 +194,19 @@ template struct handle_type_name> { using retval_type = conditional_t::value, void_type, Return>; static constexpr auto name - = const_name("Callable[[") + ::pybind11::detail::concat(make_caster::name...) - + const_name("], ") + make_caster::name + const_name("]"); + = const_name("Callable[[") + + ::pybind11::detail::concat(::pybind11::detail::arg_descr(make_caster::name)...) + + const_name("], ") + ::pybind11::detail::return_descr(make_caster::name) + + const_name("]"); }; template struct handle_type_name> { // PEP 484 specifies this syntax for defining only return types of callables using retval_type = conditional_t::value, void_type, Return>; - static constexpr auto name - = const_name("Callable[..., ") + make_caster::name + const_name("]"); + static constexpr auto name = const_name("Callable[..., ") + + ::pybind11::detail::return_descr(make_caster::name) + + const_name("]"); }; template @@ -202,6 +226,16 @@ struct handle_type_name> { static constexpr auto name = const_name("Optional[") + make_caster::name + const_name("]"); }; +template +struct handle_type_name> { + static constexpr auto name = const_name("Final[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("ClassVar[") + make_caster::name + const_name("]"); +}; + template struct handle_type_name> { static constexpr auto name = const_name("TypeGuard[") + make_caster::name + const_name("]"); @@ -222,16 +256,36 @@ struct handle_type_name { static constexpr auto name = const_name("Never"); }; -#if defined(__cpp_nontype_template_parameter_class) +#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) +template +consteval auto sanitize_string_literal() { + constexpr std::string_view v(StrLit.name); + constexpr std::string_view special_chars("!@%{}-"); + constexpr auto num_special_chars = std::accumulate( + special_chars.begin(), special_chars.end(), (size_t) 0, [&v](auto acc, const char &c) { + return std::move(acc) + std::ranges::count(v, c); + }); + char result[v.size() + num_special_chars + 1]; + size_t i = 0; + for (auto c : StrLit.name) { + if (special_chars.find(c) != std::string_view::npos) { + result[i++] = '!'; + } + result[i++] = c; + } + return typing::StringLiteral(result); +} + template struct handle_type_name> { - static constexpr auto name = const_name("Literal[") - + pybind11::detail::concat(const_name(Literals.name)...) - + const_name("]"); + static constexpr auto name + = const_name("Literal[") + + pybind11::detail::concat(const_name(sanitize_string_literal().name)...) + + const_name("]"); }; template struct handle_type_name> { - static constexpr auto name = const_name(StrLit.name); + static constexpr auto name = const_name(sanitize_string_literal().name); }; #endif diff --git a/wrap/pybind11/include/pybind11/warnings.h b/wrap/pybind11/include/pybind11/warnings.h new file mode 100644 index 000000000..263b2990e --- /dev/null +++ b/wrap/pybind11/include/pybind11/warnings.h @@ -0,0 +1,75 @@ +/* + pybind11/warnings.h: Python warnings wrappers. + + Copyright (c) 2024 Jan Iwaszkiewicz + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "pybind11.h" +#include "detail/common.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) + +inline bool PyWarning_Check(PyObject *obj) { + int result = PyObject_IsSubclass(obj, PyExc_Warning); + if (result == 1) { + return true; + } + if (result == -1) { + raise_from(PyExc_SystemError, + "pybind11::detail::PyWarning_Check(): PyObject_IsSubclass() call failed."); + throw error_already_set(); + } + return false; +} + +PYBIND11_NAMESPACE_END(detail) + +PYBIND11_NAMESPACE_BEGIN(warnings) + +inline object +new_warning_type(handle scope, const char *name, handle base = PyExc_RuntimeWarning) { + if (!detail::PyWarning_Check(base.ptr())) { + pybind11_fail("pybind11::warnings::new_warning_type(): cannot create custom warning, base " + "must be a subclass of " + "PyExc_Warning!"); + } + if (hasattr(scope, name)) { + pybind11_fail("pybind11::warnings::new_warning_type(): an attribute with name \"" + + std::string(name) + "\" exists already."); + } + std::string full_name = scope.attr("__name__").cast() + std::string(".") + name; + handle h(PyErr_NewException(full_name.c_str(), base.ptr(), nullptr)); + if (!h) { + raise_from(PyExc_SystemError, + "pybind11::warnings::new_warning_type(): PyErr_NewException() call failed."); + throw error_already_set(); + } + auto obj = reinterpret_steal(h); + scope.attr(name) = obj; + return obj; +} + +// Similar to Python `warnings.warn()` +inline void +warn(const char *message, handle category = PyExc_RuntimeWarning, int stack_level = 2) { + if (!detail::PyWarning_Check(category.ptr())) { + pybind11_fail( + "pybind11::warnings::warn(): cannot raise warning, category must be a subclass of " + "PyExc_Warning!"); + } + + if (PyErr_WarnEx(category.ptr(), message, stack_level) == -1) { + throw error_already_set(); + } +} + +PYBIND11_NAMESPACE_END(warnings) + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/wrap/pybind11/pybind11/__init__.py b/wrap/pybind11/pybind11/__init__.py index b14660cae..e9d033c4b 100644 --- a/wrap/pybind11/pybind11/__init__.py +++ b/wrap/pybind11/pybind11/__init__.py @@ -2,8 +2,8 @@ from __future__ import annotations import sys -if sys.version_info < (3, 7): # noqa: UP036 - msg = "pybind11 does not support Python < 3.7. v2.12 was the last release supporting Python 3.6." +if sys.version_info < (3, 8): # noqa: UP036 + msg = "pybind11 does not support Python < 3.8. v2.13 was the last release supporting Python 3.7." raise ImportError(msg) diff --git a/wrap/pybind11/pybind11/__main__.py b/wrap/pybind11/pybind11/__main__.py index b656ce6fe..28be9f165 100644 --- a/wrap/pybind11/pybind11/__main__.py +++ b/wrap/pybind11/pybind11/__main__.py @@ -2,12 +2,35 @@ from __future__ import annotations import argparse +import re import sys import sysconfig from ._version import __version__ from .commands import get_cmake_dir, get_include, get_pkgconfig_dir +# This is the conditional used for os.path being posixpath +if "posix" in sys.builtin_module_names: + from shlex import quote +elif "nt" in sys.builtin_module_names: + # See https://github.com/mesonbuild/meson/blob/db22551ed9d2dd7889abea01cc1c7bba02bf1c75/mesonbuild/utils/universal.py#L1092-L1121 + # and the original documents: + # https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments and + # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + UNSAFE = re.compile("[ \t\n\r]") + + def quote(s: str) -> str: + if s and not UNSAFE.search(s): + return s + + # Paths cannot contain a '"' on Windows, so we don't need to worry + # about nuanced counting here. + return f'"{s}\\"' if s.endswith("\\") else f'"{s}"' +else: + + def quote(s: str) -> str: + return s + def print_includes() -> None: dirs = [ @@ -22,7 +45,7 @@ def print_includes() -> None: if d and d not in unique_dirs: unique_dirs.append(d) - print(" ".join("-I" + d for d in unique_dirs)) + print(" ".join(quote(f"-I{d}") for d in unique_dirs)) def main() -> None: @@ -48,15 +71,22 @@ def main() -> None: action="store_true", help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.", ) + parser.add_argument( + "--extension-suffix", + action="store_true", + help="Print the extension for a Python module", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() if args.includes: print_includes() if args.cmakedir: - print(get_cmake_dir()) + print(quote(get_cmake_dir())) if args.pkgconfigdir: - print(get_pkgconfig_dir()) + print(quote(get_pkgconfig_dir())) + if args.extension_suffix: + print(sysconfig.get_config_var("EXT_SUFFIX")) if __name__ == "__main__": diff --git a/wrap/pybind11/pybind11/_version.py b/wrap/pybind11/pybind11/_version.py index 18b72c07c..2dfe67616 100644 --- a/wrap/pybind11/pybind11/_version.py +++ b/wrap/pybind11/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> int | str: return s -__version__ = "2.13.1" +__version__ = "3.0.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/wrap/pybind11/pybind11/setup_helpers.py b/wrap/pybind11/pybind11/setup_helpers.py index ced506f8c..f24291818 100644 --- a/wrap/pybind11/pybind11/setup_helpers.py +++ b/wrap/pybind11/pybind11/setup_helpers.py @@ -249,7 +249,7 @@ def has_flag(compiler: Any, flag: str) -> bool: cpp_flag_cache = None -@lru_cache() +@lru_cache def auto_cpp_level(compiler: Any) -> str | int: """ Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows. diff --git a/wrap/pybind11/pyproject.toml b/wrap/pybind11/pyproject.toml index 71e7f5617..13dd04a51 100644 --- a/wrap/pybind11/pyproject.toml +++ b/wrap/pybind11/pyproject.toml @@ -30,7 +30,7 @@ ignore_missing_imports = true [tool.pylint] -master.py-version = "3.7" +master.py-version = "3.8" reports.output-format = "colorized" messages_control.disable = [ "design", @@ -45,7 +45,7 @@ messages_control.disable = [ ] [tool.ruff] -target-version = "py37" +target-version = "py38" src = ["src"] [tool.ruff.lint] @@ -71,7 +71,6 @@ ignore = [ "PLR", # Design related pylint "E501", # Line too long (Black is enough) "PT011", # Too broad with raises in pytest - "PT004", # Fixture that doesn't return needs underscore (no, it is fine) "SIM118", # iter(x) is not always the same as iter(x.keys()) ] unfixable = ["T20"] diff --git a/wrap/pybind11/setup.cfg b/wrap/pybind11/setup.cfg index 2beca780f..bb5b744aa 100644 --- a/wrap/pybind11/setup.cfg +++ b/wrap/pybind11/setup.cfg @@ -14,7 +14,6 @@ classifiers = Topic :: Utilities Programming Language :: C++ Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 @@ -39,5 +38,5 @@ project_urls = Chat = https://gitter.im/pybind/Lobby [options] -python_requires = >=3.7 +python_requires = >=3.8 zip_safe = False diff --git a/wrap/pybind11/setup.py b/wrap/pybind11/setup.py index 96563c1a5..c65bbc627 100644 --- a/wrap/pybind11/setup.py +++ b/wrap/pybind11/setup.py @@ -144,6 +144,10 @@ with remove_output("pybind11/include", "pybind11/share"): stderr=sys.stderr, ) + # pkgconf-pypi needs pybind11/share/pkgconfig to be importable + Path("pybind11/share/__init__.py").touch() + Path("pybind11/share/pkgconfig/__init__.py").touch() + txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") exec(code, {"SDist": SDist}) diff --git a/wrap/pybind11/tests/CMakeLists.txt b/wrap/pybind11/tests/CMakeLists.txt index f182e2499..e2fab9c13 100644 --- a/wrap/pybind11/tests/CMakeLists.txt +++ b/wrap/pybind11/tests/CMakeLists.txt @@ -5,16 +5,7 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.5) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) # Filter out items; print an optional message if any items filtered. This ignores extensions. # @@ -76,8 +67,8 @@ project(pybind11_tests CXX) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/../tools") option(PYBIND11_WERROR "Report all warnings as errors" OFF) -option(DOWNLOAD_EIGEN "Download EIGEN (requires CMake 3.11+)" OFF) -option(PYBIND11_CUDA_TESTS "Enable building CUDA tests (requires CMake 3.12+)" OFF) +option(DOWNLOAD_EIGEN "Download EIGEN" OFF) +option(PYBIND11_CUDA_TESTS "Enable building CUDA tests" OFF) set(PYBIND11_TEST_OVERRIDE "" CACHE STRING "Tests from ;-separated list of *.cpp files will be built instead of all tests") @@ -88,7 +79,12 @@ set(PYBIND11_TEST_FILTER if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) # We're being loaded directly, i.e. not via add_subdirectory, so make this # work as its own project and load the pybind11Config to get the tools we need - find_package(pybind11 REQUIRED CONFIG) + + if(SKBUILD) + add_subdirectory(.. pybind11_src) + else() + find_package(pybind11 REQUIRED CONFIG) + endif() endif() if(NOT CMAKE_BUILD_TYPE AND NOT DEFINED CMAKE_CONFIGURATION_TYPES) @@ -119,12 +115,32 @@ set(PYBIND11_TEST_FILES test_callbacks test_chrono test_class + test_class_release_gil_before_calling_cpp_dtor + test_class_sh_basic + test_class_sh_disowning + test_class_sh_disowning_mi + test_class_sh_factory_constructors + test_class_sh_inheritance + test_class_sh_mi_thunks + test_class_sh_property + test_class_sh_property_non_owning + test_class_sh_shared_ptr_copy_move + test_class_sh_trampoline_basic + test_class_sh_trampoline_self_life_support + test_class_sh_trampoline_shared_from_this + test_class_sh_trampoline_shared_ptr_cpp_arg + test_class_sh_trampoline_unique_ptr + test_class_sh_unique_ptr_custom_deleter + test_class_sh_unique_ptr_member + test_class_sh_virtual_py_cpp_mix test_const_name test_constants_and_functions test_copy_move + test_cpp_conduit test_custom_type_casters test_custom_type_setup test_docstring_options + test_docs_advanced_cast_custom test_eigen_matrix test_eigen_tensor test_enum @@ -153,11 +169,13 @@ set(PYBIND11_TEST_FILES test_tagbased_polymorphic test_thread test_type_caster_pyobject_ptr + test_type_caster_std_function_specializations test_union test_unnamed_namespace_a test_unnamed_namespace_b test_vector_unique_ptr_member - test_virtual_functions) + test_virtual_functions + test_warnings) # Invoking cmake with something like: # cmake -DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp" .. @@ -220,6 +238,8 @@ tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_ # And add additional targets for other tests. tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set") tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils") +tests_extra_targets("test_cpp_conduit.py" + "exo_planet_pybind11;exo_planet_c_api;home_planet_very_lonely_traveler") set(PYBIND11_EIGEN_REPO "https://gitlab.com/libeigen/eigen.git" @@ -243,25 +263,21 @@ endif() if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake). # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also - # produces a fatal error if loaded from a pre-3.0 cmake. if(DOWNLOAD_EIGEN) - if(CMAKE_VERSION VERSION_LESS 3.11) - message(FATAL_ERROR "CMake 3.11+ required when using DOWNLOAD_EIGEN") + if(CMAKE_VERSION VERSION_LESS 3.18) + set(_opts) + else() + set(_opts SOURCE_SUBDIR no-cmake-build) endif() - include(FetchContent) FetchContent_Declare( eigen GIT_REPOSITORY "${PYBIND11_EIGEN_REPO}" - GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}") - - FetchContent_GetProperties(eigen) - if(NOT eigen_POPULATED) - message( - STATUS - "Downloading Eigen ${PYBIND11_EIGEN_VERSION_STRING} (${PYBIND11_EIGEN_VERSION_HASH}) from ${PYBIND11_EIGEN_REPO}" - ) - FetchContent_Populate(eigen) + GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}" + ${_opts}) + FetchContent_MakeAvailable(eigen) + if(NOT CMAKE_VERSION VERSION_LESS 3.18) + set(EIGEN3_INCLUDE_DIR "${eigen_SOURCE_DIR}") endif() set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR}) @@ -309,8 +325,7 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) endif() - message( - STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download") + message(STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON to download") endif() endif() @@ -390,6 +405,9 @@ function(pybind11_enable_warnings target_name) -Wdeprecated -Wundef -Wnon-virtual-dtor) + if(DEFINED CMAKE_CXX_STANDARD AND NOT CMAKE_CXX_STANDARD VERSION_LESS 20) + target_compile_options(${target_name} PRIVATE -Wpedantic) + endif() endif() if(PYBIND11_WERROR) @@ -489,15 +507,22 @@ foreach(target ${test_targets}) endforeach() endif() endif() + if(SKBUILD) + install(TARGETS ${target} LIBRARY DESTINATION .) + endif() + + if("${target}" STREQUAL "exo_planet_c_api") + if(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang|NVHPC)") + target_compile_options(${target} PRIVATE -fno-exceptions) + endif() + endif() endforeach() # Provide nice organisation in IDEs -if(NOT CMAKE_VERSION VERSION_LESS 3.8) - source_group( - TREE "${CMAKE_CURRENT_SOURCE_DIR}/../include" - PREFIX "Header Files" - FILES ${PYBIND11_HEADERS}) -endif() +source_group( + TREE "${CMAKE_CURRENT_SOURCE_DIR}/../include" + PREFIX "Header Files" + FILES ${PYBIND11_HEADERS}) # Make sure pytest is found or produce a warning pybind11_find_import(pytest VERSION 3.1) @@ -581,6 +606,9 @@ add_custom_command( ${CMAKE_CURRENT_BINARY_DIR}/sosize-$.txt) if(NOT PYBIND11_CUDA_TESTS) + # Test pure C++ code (not depending on Python). Provides the `test_pure_cpp` target. + add_subdirectory(pure_cpp) + # Test embedding the interpreter. Provides the `cpptest` target. add_subdirectory(test_embed) diff --git a/wrap/pybind11/tests/conftest.py b/wrap/pybind11/tests/conftest.py index 7de6c2ace..a018a3f79 100644 --- a/wrap/pybind11/tests/conftest.py +++ b/wrap/pybind11/tests/conftest.py @@ -28,8 +28,8 @@ except Exception: @pytest.fixture(scope="session", autouse=True) def use_multiprocessing_forkserver_on_linux(): - if sys.platform != "linux": - # The default on Windows and macOS is "spawn": If it's not broken, don't fix it. + if sys.platform != "linux" or sys.implementation.name == "graalpy": + # The default on Windows, macOS and GraalPy is "spawn": If it's not broken, don't fix it. return # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 @@ -136,7 +136,7 @@ class Capture: return Output(self.err) -@pytest.fixture() +@pytest.fixture def capture(capsys): """Extended `capsys` with context manager and custom equality operators""" return Capture(capsys) @@ -172,7 +172,7 @@ def _sanitize_docstring(thing): return _sanitize_general(s) -@pytest.fixture() +@pytest.fixture def doc(): """Sanitize docstrings and add custom failure explanation""" return SanitizedString(_sanitize_docstring) @@ -184,7 +184,7 @@ def _sanitize_message(thing): return _hexadecimal.sub("0", s) -@pytest.fixture() +@pytest.fixture def msg(): """Sanitize messages and add custom failure explanation""" return SanitizedString(_sanitize_message) @@ -198,10 +198,11 @@ def pytest_assertrepr_compare(op, left, right): # noqa: ARG001 def gc_collect(): - """Run the garbage collector twice (needed when running + """Run the garbage collector three times (needed when running reference counting tests with PyPy)""" gc.collect() gc.collect() + gc.collect() def pytest_configure(): @@ -211,9 +212,9 @@ def pytest_configure(): def pytest_report_header(config): del config # Unused. - assert ( - pybind11_tests.compiler_info is not None - ), "Please update pybind11_tests.cpp if this assert fails." + assert pybind11_tests.compiler_info is not None, ( + "Please update pybind11_tests.cpp if this assert fails." + ) return ( "C++ Info:" f" {pybind11_tests.compiler_info}" diff --git a/wrap/pybind11/tests/constructor_stats.h b/wrap/pybind11/tests/constructor_stats.h index 937f6c233..352b1b6ca 100644 --- a/wrap/pybind11/tests/constructor_stats.h +++ b/wrap/pybind11/tests/constructor_stats.h @@ -190,7 +190,7 @@ public: t1 = &p.first; } } - } catch (const std::out_of_range &) { + } catch (const std::out_of_range &) { // NOLINT(bugprone-empty-catch) } if (!t1) { throw std::runtime_error("Unknown class passed to ConstructorStats::get()"); @@ -312,8 +312,16 @@ void print_created(T *inst, Values &&...values) { } template void print_destroyed(T *inst, Values &&...values) { // Prints but doesn't store given values + /* + * On GraalPy, destructors can trigger anywhere and this can cause random + * failures in unrelated tests. + */ +#if !defined(GRAALVM_PYTHON) print_constr_details(inst, "destroyed", values...); track_destroyed(inst); +#else + py::detail::silence_unused_warnings(inst, values...); +#endif } template void print_values(T *inst, Values &&...values) { diff --git a/wrap/pybind11/tests/custom_exceptions.py b/wrap/pybind11/tests/custom_exceptions.py new file mode 100644 index 000000000..b1a092d76 --- /dev/null +++ b/wrap/pybind11/tests/custom_exceptions.py @@ -0,0 +1,10 @@ +from __future__ import annotations + + +class PythonMyException7(Exception): + def __init__(self, message): + self.message = message + super().__init__(message) + + def __str__(self): + return "[PythonMyException7]: " + self.message.a diff --git a/wrap/pybind11/tests/env.py b/wrap/pybind11/tests/env.py index 9f5347f2e..b513e455d 100644 --- a/wrap/pybind11/tests/env.py +++ b/wrap/pybind11/tests/env.py @@ -12,6 +12,7 @@ WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") CPYTHON = platform.python_implementation() == "CPython" PYPY = platform.python_implementation() == "PyPy" +GRAALPY = sys.implementation.name == "graalpy" PY_GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) diff --git a/wrap/pybind11/tests/exo_planet_c_api.cpp b/wrap/pybind11/tests/exo_planet_c_api.cpp new file mode 100644 index 000000000..151010717 --- /dev/null +++ b/wrap/pybind11/tests/exo_planet_c_api.cpp @@ -0,0 +1,76 @@ +// Copyright (c) 2024 The pybind Community. + +// In production situations it is totally fine to build with +// C++ Exception Handling enabled. However, here we want to ensure that +// C++ Exception Handling is not required. +#if defined(_MSC_VER) || defined(__EMSCRIPTEN__) +// Too much trouble making the required cmake changes (see PR #5375). +#else +# ifdef __cpp_exceptions +// https://isocpp.org/std/standing-documents/sd-6-sg10-feature-test-recommendations#__cpp_exceptions +# error This test is meant to be built with C++ Exception Handling disabled, but __cpp_exceptions is defined. +# endif +# ifdef __EXCEPTIONS +// https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html +# error This test is meant to be built with C++ Exception Handling disabled, but __EXCEPTIONS is defined. +# endif +#endif + +// THIS MUST STAY AT THE TOP! +#include // VERY light-weight dependency. + +#include "test_cpp_conduit_traveler_types.h" + +#include + +namespace { + +extern "C" PyObject *wrapGetLuggage(PyObject * /*self*/, PyObject *traveler) { + const auto *cpp_traveler = pybind11_conduit_v1::get_type_pointer_ephemeral< + pybind11_tests::test_cpp_conduit::Traveler>(traveler); + if (cpp_traveler == nullptr) { + return nullptr; + } + return PyUnicode_FromString(cpp_traveler->luggage.c_str()); +} + +extern "C" PyObject *wrapGetPoints(PyObject * /*self*/, PyObject *premium_traveler) { + const auto *cpp_premium_traveler = pybind11_conduit_v1::get_type_pointer_ephemeral< + pybind11_tests::test_cpp_conduit::PremiumTraveler>(premium_traveler); + if (cpp_premium_traveler == nullptr) { + return nullptr; + } + return PyLong_FromLong(static_cast(cpp_premium_traveler->points)); +} + +PyMethodDef ThisMethodDef[] = {{"GetLuggage", wrapGetLuggage, METH_O, nullptr}, + {"GetPoints", wrapGetPoints, METH_O, nullptr}, + {nullptr, nullptr, 0, nullptr}}; + +struct PyModuleDef ThisModuleDef = { + PyModuleDef_HEAD_INIT, // m_base + "exo_planet_c_api", // m_name + nullptr, // m_doc + -1, // m_size + ThisMethodDef, // m_methods + nullptr, // m_slots + nullptr, // m_traverse + nullptr, // m_clear + nullptr // m_free +}; + +} // namespace + +#if defined(WIN32) || defined(_WIN32) +# define EXO_PLANET_C_API_EXPORT __declspec(dllexport) +#else +# define EXO_PLANET_C_API_EXPORT __attribute__((visibility("default"))) +#endif + +extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() { + PyObject *m = PyModule_Create(&ThisModuleDef); + if (m == nullptr) { + return nullptr; + } + return m; +} diff --git a/wrap/pybind11/tests/exo_planet_pybind11.cpp b/wrap/pybind11/tests/exo_planet_pybind11.cpp new file mode 100644 index 000000000..9d1a2b84b --- /dev/null +++ b/wrap/pybind11/tests/exo_planet_pybind11.cpp @@ -0,0 +1,19 @@ +// Copyright (c) 2024 The pybind Community. + +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 900000001 + +#include "test_cpp_conduit_traveler_bindings.h" + +namespace pybind11_tests { +namespace test_cpp_conduit { + +PYBIND11_MODULE(exo_planet_pybind11, m) { + wrap_traveler(m); + m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); }); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/extra_python_package/test_files.py b/wrap/pybind11/tests/extra_python_package/test_files.py index 5a3f779a7..5d7299c2a 100644 --- a/wrap/pybind11/tests/extra_python_package/test_files.py +++ b/wrap/pybind11/tests/extra_python_package/test_files.py @@ -9,7 +9,6 @@ import tarfile import zipfile # These tests must be run explicitly -# They require CMake 3.15+ (--install) DIR = os.path.abspath(os.path.dirname(__file__)) MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) @@ -46,18 +45,33 @@ main_headers = { "include/pybind11/pytypes.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", + "include/pybind11/trampoline_self_life_support.h", "include/pybind11/type_caster_pyobject_ptr.h", "include/pybind11/typing.h", + "include/pybind11/warnings.h", +} + +conduit_headers = { + "include/pybind11/conduit/README.txt", + "include/pybind11/conduit/pybind11_conduit_v1.h", + "include/pybind11/conduit/pybind11_platform_abi_id.h", + "include/pybind11/conduit/wrap_include_python_h.h", } detail_headers = { "include/pybind11/detail/class.h", "include/pybind11/detail/common.h", + "include/pybind11/detail/cpp_conduit.h", "include/pybind11/detail/descr.h", + "include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", + "include/pybind11/detail/struct_smart_holder.h", "include/pybind11/detail/type_caster_base.h", "include/pybind11/detail/typeid.h", + "include/pybind11/detail/using_smart_holder.h", + "include/pybind11/detail/value_and_holder.h", + "include/pybind11/detail/exception_translation.h", } eigen_headers = { @@ -92,9 +106,11 @@ py_files = { "commands.py", "py.typed", "setup_helpers.py", + "share/__init__.py", + "share/pkgconfig/__init__.py", } -headers = main_headers | detail_headers | eigen_headers | stl_headers +headers = main_headers | conduit_headers | detail_headers | eigen_headers | stl_headers src_files = headers | cmake_files | pkgconfig_files all_files = src_files | py_files @@ -103,6 +119,7 @@ sdist_files = { "pybind11", "pybind11/include", "pybind11/include/pybind11", + "pybind11/include/pybind11/conduit", "pybind11/include/pybind11/detail", "pybind11/include/pybind11/eigen", "pybind11/include/pybind11/stl", diff --git a/wrap/pybind11/tests/home_planet_very_lonely_traveler.cpp b/wrap/pybind11/tests/home_planet_very_lonely_traveler.cpp new file mode 100644 index 000000000..78d50cff5 --- /dev/null +++ b/wrap/pybind11/tests/home_planet_very_lonely_traveler.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2024 The pybind Community. + +#include "test_cpp_conduit_traveler_bindings.h" + +namespace pybind11_tests { +namespace test_cpp_conduit { + +PYBIND11_MODULE(home_planet_very_lonely_traveler, m) { + m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); }); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/local_bindings.h b/wrap/pybind11/tests/local_bindings.h index 01d278535..dea181310 100644 --- a/wrap/pybind11/tests/local_bindings.h +++ b/wrap/pybind11/tests/local_bindings.h @@ -56,13 +56,13 @@ private: std::string message = ""; }; -PYBIND11_MAKE_OPAQUE(LocalVec); -PYBIND11_MAKE_OPAQUE(LocalVec2); -PYBIND11_MAKE_OPAQUE(LocalMap); -PYBIND11_MAKE_OPAQUE(NonLocalVec); -// PYBIND11_MAKE_OPAQUE(NonLocalVec2); // same type as LocalVec2 -PYBIND11_MAKE_OPAQUE(NonLocalMap); -PYBIND11_MAKE_OPAQUE(NonLocalMap2); +PYBIND11_MAKE_OPAQUE(LocalVec) +PYBIND11_MAKE_OPAQUE(LocalVec2) +PYBIND11_MAKE_OPAQUE(LocalMap) +PYBIND11_MAKE_OPAQUE(NonLocalVec) +// PYBIND11_MAKE_OPAQUE(NonLocalVec2) // same type as LocalVec2 +PYBIND11_MAKE_OPAQUE(NonLocalMap) +PYBIND11_MAKE_OPAQUE(NonLocalMap2) // Simple bindings (used with the above): template @@ -70,7 +70,7 @@ py::class_ bind_local(Args &&...args) { return py::class_(std::forward(args)...).def(py::init()).def("get", [](T &i) { return i.i + Adjust; }); -}; +} // Simulate a foreign library base class (to match the example in the docs): namespace pets { diff --git a/wrap/pybind11/tests/pure_cpp/CMakeLists.txt b/wrap/pybind11/tests/pure_cpp/CMakeLists.txt new file mode 100644 index 000000000..17be74b7f --- /dev/null +++ b/wrap/pybind11/tests/pure_cpp/CMakeLists.txt @@ -0,0 +1,20 @@ +find_package(Catch 2.13.2) + +if(CATCH_FOUND) + message(STATUS "Building pure C++ tests (not depending on Python) using Catch v${CATCH_VERSION}") +else() + message(STATUS "Catch not detected. Interpreter tests will be skipped. Install Catch headers" + " manually or use `cmake -DDOWNLOAD_CATCH=ON` to fetch them automatically.") + return() +endif() + +add_executable(smart_holder_poc_test smart_holder_poc_test.cpp) +pybind11_enable_warnings(smart_holder_poc_test) +target_link_libraries(smart_holder_poc_test PRIVATE pybind11::headers Catch2::Catch2) + +add_custom_target( + test_pure_cpp + COMMAND "$" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + +add_dependencies(check test_pure_cpp) diff --git a/wrap/pybind11/tests/pure_cpp/smart_holder_poc.h b/wrap/pybind11/tests/pure_cpp/smart_holder_poc.h new file mode 100644 index 000000000..320311b7d --- /dev/null +++ b/wrap/pybind11/tests/pure_cpp/smart_holder_poc.h @@ -0,0 +1,51 @@ +// Copyright (c) 2020-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "pybind11/detail/struct_smart_holder.h" + +namespace pybindit { +namespace memory { +namespace smart_holder_poc { // Proof-of-Concept implementations. + +template +T &as_lvalue_ref(const smart_holder &hld) { + static const char *context = "as_lvalue_ref"; + hld.ensure_is_populated(context); + hld.ensure_has_pointee(context); + return *hld.as_raw_ptr_unowned(); +} + +template +T &&as_rvalue_ref(const smart_holder &hld) { + static const char *context = "as_rvalue_ref"; + hld.ensure_is_populated(context); + hld.ensure_has_pointee(context); + return std::move(*hld.as_raw_ptr_unowned()); +} + +template +T *as_raw_ptr_release_ownership(smart_holder &hld, + const char *context = "as_raw_ptr_release_ownership") { + hld.ensure_can_release_ownership(context); + T *raw_ptr = hld.as_raw_ptr_unowned(); + hld.release_ownership(); + return raw_ptr; +} + +template > +std::unique_ptr as_unique_ptr(smart_holder &hld) { + static const char *context = "as_unique_ptr"; + hld.ensure_compatible_rtti_uqp_del(context); + hld.ensure_use_count_1(context); + T *raw_ptr = hld.as_raw_ptr_unowned(); + hld.release_ownership(); + // KNOWN DEFECT (see PR #4850): Does not copy the deleter. + return std::unique_ptr(raw_ptr); +} + +} // namespace smart_holder_poc +} // namespace memory +} // namespace pybindit diff --git a/wrap/pybind11/tests/pure_cpp/smart_holder_poc_test.cpp b/wrap/pybind11/tests/pure_cpp/smart_holder_poc_test.cpp new file mode 100644 index 000000000..24ab643ee --- /dev/null +++ b/wrap/pybind11/tests/pure_cpp/smart_holder_poc_test.cpp @@ -0,0 +1,415 @@ +#include "smart_holder_poc.h" + +#include +#include +#include +#include + +// Catch uses _ internally, which breaks gettext style defines +#ifdef _ +# undef _ +#endif + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +using pybindit::memory::smart_holder; +namespace poc = pybindit::memory::smart_holder_poc; + +namespace helpers { + +struct movable_int { + int valu; + explicit movable_int(int v) : valu{v} {} + movable_int(movable_int &&other) noexcept : valu(other.valu) { other.valu = 91; } +}; + +template +struct functor_builtin_delete { + void operator()(T *ptr) { delete ptr; } +#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8) \ + || (defined(__clang_major__) && __clang_major__ == 3 && __clang_minor__ == 6) + // Workaround for these errors: + // gcc 4.8.5: too many initializers for 'helpers::functor_builtin_delete' + // clang 3.6: excess elements in struct initializer + functor_builtin_delete() = default; + functor_builtin_delete(const functor_builtin_delete &) {} + functor_builtin_delete(functor_builtin_delete &&) {} +#endif +}; + +template +struct functor_other_delete : functor_builtin_delete {}; + +struct indestructible_int { + int valu; + explicit indestructible_int(int v) : valu{v} {} + +private: + ~indestructible_int() = default; +}; + +struct base { + virtual int get() { return 10; } + virtual ~base() = default; +}; + +struct derived : public base { + int get() override { return 100; } +}; + +} // namespace helpers + +TEST_CASE("from_raw_ptr_unowned+as_raw_ptr_unowned", "[S]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE(*hld.as_raw_ptr_unowned() == 19); +} + +TEST_CASE("from_raw_ptr_unowned+as_lvalue_ref", "[S]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_raw_ptr_unowned+as_rvalue_ref", "[S]") { + helpers::movable_int orig(19); + { + auto hld = smart_holder::from_raw_ptr_unowned(&orig); + helpers::movable_int othr(poc::as_rvalue_ref(hld)); + REQUIRE(othr.valu == 19); + REQUIRE(orig.valu == 91); + } +} + +TEST_CASE("from_raw_ptr_unowned+as_raw_ptr_release_ownership", "[E]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown non-owning holder (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_raw_ptr_unowned+as_unique_ptr", "[E]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown non-owning holder (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_unowned+as_unique_ptr_with_deleter", "[E]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Missing unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_unowned+as_shared_ptr", "[S]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE(*hld.as_shared_ptr() == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_lvalue_ref", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + REQUIRE(hld.has_pointee()); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_raw_ptr_release_ownership1", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + auto new_owner = std::unique_ptr(poc::as_raw_ptr_release_ownership(hld)); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_raw_ptr_release_ownership2", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown use_count != 1 (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr1", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::unique_ptr new_owner = poc::as_unique_ptr(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr2", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown use_count != 1 (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr_with_deleter", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Missing unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_shared_ptr", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::shared_ptr new_owner = hld.as_shared_ptr(); + REQUIRE(hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+disown+reclaim_disowned", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::unique_ptr new_owner(hld.as_raw_ptr_unowned()); + hld.disown(); + REQUIRE(poc::as_lvalue_ref(hld) == 19); + REQUIRE(*new_owner == 19); + hld.reclaim_disowned(); // Manually veriified: without this, clang++ -fsanitize=address reports + // "detected memory leaks". + // NOLINTNEXTLINE(bugprone-unused-return-value) + (void) new_owner.release(); // Manually verified: without this, clang++ -fsanitize=address + // reports "attempting double-free". + REQUIRE(poc::as_lvalue_ref(hld) == 19); + REQUIRE(new_owner.get() == nullptr); +} + +TEST_CASE("from_raw_ptr_take_ownership+disown+release_disowned", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::unique_ptr new_owner(hld.as_raw_ptr_unowned()); + hld.disown(); + REQUIRE(poc::as_lvalue_ref(hld) == 19); + REQUIRE(*new_owner == 19); + hld.release_disowned(); + REQUIRE(!hld.has_pointee()); +} + +TEST_CASE("from_raw_ptr_take_ownership+disown+ensure_is_not_disowned", "[E]") { + const char *context = "test_case"; + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + hld.ensure_is_not_disowned(context); // Does not throw. + std::unique_ptr new_owner(hld.as_raw_ptr_unowned()); + hld.disown(); + REQUIRE_THROWS_WITH(hld.ensure_is_not_disowned(context), + "Holder was disowned already (test_case)."); +} + +TEST_CASE("from_unique_ptr+as_lvalue_ref", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_unique_ptr+as_raw_ptr_release_ownership1", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + auto new_owner = std::unique_ptr(poc::as_raw_ptr_release_ownership(hld)); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr+as_raw_ptr_release_ownership2", "[E]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown use_count != 1 (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_unique_ptr+as_unique_ptr1", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::unique_ptr new_owner = poc::as_unique_ptr(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr+as_unique_ptr2", "[E]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown use_count != 1 (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr+as_unique_ptr_with_deleter", "[E]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr+as_shared_ptr", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::shared_ptr new_owner = hld.as_shared_ptr(); + REQUIRE(hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr_derived+as_unique_ptr_base", "[S]") { + std::unique_ptr orig_owner(new helpers::derived()); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::unique_ptr new_owner = poc::as_unique_ptr(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(new_owner->get() == 100); +} + +TEST_CASE("from_unique_ptr_derived+as_unique_ptr_base2", "[E]") { + std::unique_ptr> orig_owner( + new helpers::derived()); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH( + (poc::as_unique_ptr>(hld)), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_lvalue_ref", "[S]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_unique_ptr_with_std_function_deleter+as_lvalue_ref", "[S]") { + std::unique_ptr> orig_owner( + new int(19), [](const int *raw_ptr) { delete raw_ptr; }); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_raw_ptr_release_ownership", "[E]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown custom deleter (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr", "[E]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr_with_deleter1", "[S]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::unique_ptr> new_owner + = poc::as_unique_ptr>(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr_with_deleter2", "[E]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_shared_ptr", "[S]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::shared_ptr new_owner = hld.as_shared_ptr(); + REQUIRE(hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_shared_ptr+as_lvalue_ref", "[S]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_shared_ptr+as_raw_ptr_release_ownership", "[E]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown external shared_ptr (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_shared_ptr+as_unique_ptr", "[E]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown external shared_ptr (as_unique_ptr)."); +} + +TEST_CASE("from_shared_ptr+as_unique_ptr_with_deleter", "[E]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Missing unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_shared_ptr+as_shared_ptr", "[S]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE(*hld.as_shared_ptr() == 19); +} + +TEST_CASE("error_unpopulated_holder", "[E]") { + smart_holder hld; + REQUIRE_THROWS_WITH(poc::as_lvalue_ref(hld), "Unpopulated holder (as_lvalue_ref)."); +} + +TEST_CASE("error_disowned_holder", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + poc::as_unique_ptr(hld); + REQUIRE_THROWS_WITH(poc::as_lvalue_ref(hld), "Disowned holder (as_lvalue_ref)."); +} + +TEST_CASE("error_cannot_disown_nullptr", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + poc::as_unique_ptr(hld); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), "Cannot disown nullptr (as_unique_ptr)."); +} + +TEST_CASE("indestructible_int-from_raw_ptr_unowned+as_raw_ptr_unowned", "[S]") { + using zombie = helpers::indestructible_int; + // Using placement new instead of plain new, to not trigger leak sanitizer errors. + static std::aligned_storage::type memory_block[1]; + auto *value = new (memory_block) zombie(19); + auto hld = smart_holder::from_raw_ptr_unowned(value); + REQUIRE(hld.as_raw_ptr_unowned()->valu == 19); +} + +TEST_CASE("indestructible_int-from_raw_ptr_take_ownership", "[E]") { + helpers::indestructible_int *value = nullptr; + REQUIRE_THROWS_WITH(smart_holder::from_raw_ptr_take_ownership(value), + "Pointee is not destructible (from_raw_ptr_take_ownership)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_shared_ptr-outliving_smart_holder", "[S]") { + // Exercises guarded_builtin_delete flag_ptr validity past destruction of smart_holder. + std::shared_ptr longer_living; + { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + longer_living = hld.as_shared_ptr(); + } + REQUIRE(*longer_living == 19); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_shared_ptr-outliving_smart_holder", "[S]") { + // Exercises guarded_custom_deleter flag_ptr validity past destruction of smart_holder. + std::shared_ptr longer_living; + { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + longer_living = hld.as_shared_ptr(); + } + REQUIRE(*longer_living == 19); +} diff --git a/wrap/pybind11/tests/pybind11_tests.cpp b/wrap/pybind11/tests/pybind11_tests.cpp index 3d2d84e77..818d53a54 100644 --- a/wrap/pybind11/tests/pybind11_tests.cpp +++ b/wrap/pybind11/tests/pybind11_tests.cpp @@ -128,4 +128,9 @@ PYBIND11_MODULE(pybind11_tests, m, py::mod_gil_not_used()) { for (const auto &initializer : initializers()) { initializer(m); } + + py::class_(m, "TestContext") + .def(py::init<>(&TestContext::createNewContextForInit)) + .def("__enter__", &TestContext::contextEnter) + .def("__exit__", &TestContext::contextExit); } diff --git a/wrap/pybind11/tests/pybind11_tests.h b/wrap/pybind11/tests/pybind11_tests.h index 7be58feb6..0eb0398df 100644 --- a/wrap/pybind11/tests/pybind11_tests.h +++ b/wrap/pybind11/tests/pybind11_tests.h @@ -96,3 +96,24 @@ void ignoreOldStyleInitWarnings(F &&body) { )", py::dict(py::arg("body") = py::cpp_function(body))); } + +// See PR #5419 for background. +class TestContext { +public: + TestContext() = delete; + TestContext(const TestContext &) = delete; + TestContext(TestContext &&) = delete; + static TestContext *createNewContextForInit() { return new TestContext("new-context"); } + + pybind11::object contextEnter() { + py::object contextObj = py::cast(*this); + return contextObj; + } + void contextExit(const pybind11::object & /*excType*/, + const pybind11::object & /*excVal*/, + const pybind11::object & /*excTb*/) {} + +private: + explicit TestContext(const std::string &context) : context(context) {} + std::string context; +}; diff --git a/wrap/pybind11/tests/pyproject.toml b/wrap/pybind11/tests/pyproject.toml new file mode 100644 index 000000000..469c145df --- /dev/null +++ b/wrap/pybind11/tests/pyproject.toml @@ -0,0 +1,17 @@ +# Warning: this is currently used for pyodide, and is not a general out-of-tree +# builder for the tests (yet). Specifically, wheels can't be built from SDists. + +[build-system] +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +[project] +name = "pybind11_tests" +version = "0.0.1" +dependencies = ["pytest", "pytest-timeout", "numpy", "scipy"] + +[tool.scikit-build.cmake.define] +PYBIND11_FINDPYTHON = true + +[tool.cibuildwheel] +test-command = "pytest -o timeout=0 -p no:cacheprovider {project}/tests/test_*.py" diff --git a/wrap/pybind11/tests/requirements.txt b/wrap/pybind11/tests/requirements.txt index 337897bd2..688ff6fe8 100644 --- a/wrap/pybind11/tests/requirements.txt +++ b/wrap/pybind11/tests/requirements.txt @@ -1,13 +1,13 @@ --only-binary=:all: -build~=1.0; python_version>="3.7" -numpy~=1.20.0; python_version=="3.7" and platform_python_implementation=="PyPy" +build~=1.0; python_version>="3.8" numpy~=1.23.0; python_version=="3.8" and platform_python_implementation=="PyPy" numpy~=1.25.0; python_version=="3.9" and platform_python_implementation=='PyPy' -numpy~=1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10" -numpy~=1.22.2; platform_python_implementation!="PyPy" and python_version=="3.10" -numpy~=1.26.0; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13" +numpy~=1.26.0; platform_python_implementation=="GraalVM" and sys_platform=="linux" +numpy~=1.21.5; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.8" and python_version<"3.10" +numpy~=1.22.2; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10" +numpy~=1.26.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13" pytest~=7.0 pytest-timeout -scipy~=1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" -scipy~=1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" and sys_platform!='win32' -scipy~=1.11.1; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32' +scipy~=1.5.4; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version<"3.10" +scipy~=1.8.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10" and sys_platform!='win32' +scipy~=1.11.1; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32' diff --git a/wrap/pybind11/tests/test_async.py b/wrap/pybind11/tests/test_async.py index 4d33ba65f..64f4d6a77 100644 --- a/wrap/pybind11/tests/test_async.py +++ b/wrap/pybind11/tests/test_async.py @@ -1,12 +1,17 @@ from __future__ import annotations +import sys + import pytest asyncio = pytest.importorskip("asyncio") m = pytest.importorskip("pybind11_tests.async_module") +if sys.platform.startswith("emscripten"): + pytest.skip("Can't run a new event_loop in pyodide", allow_module_level=True) -@pytest.fixture() + +@pytest.fixture def event_loop(): loop = asyncio.new_event_loop() yield loop diff --git a/wrap/pybind11/tests/test_buffers.cpp b/wrap/pybind11/tests/test_buffers.cpp index b5b8c355b..ac4489f70 100644 --- a/wrap/pybind11/tests/test_buffers.cpp +++ b/wrap/pybind11/tests/test_buffers.cpp @@ -167,6 +167,137 @@ TEST_SUBMODULE(buffers, m) { sizeof(float)}); }); + // A matrix that uses Fortran storage order. + class FortranMatrix : public Matrix { + public: + FortranMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(cols, rows) { + print_created(this, + std::to_string(rows) + "x" + std::to_string(cols) + " Fortran matrix"); + } + + float operator()(py::ssize_t i, py::ssize_t j) const { return Matrix::operator()(j, i); } + + float &operator()(py::ssize_t i, py::ssize_t j) { return Matrix::operator()(j, i); } + + using Matrix::data; + + py::ssize_t rows() const { return Matrix::cols(); } + py::ssize_t cols() const { return Matrix::rows(); } + }; + py::class_(m, "FortranMatrix", py::buffer_protocol()) + .def(py::init()) + + .def("rows", &FortranMatrix::rows) + .def("cols", &FortranMatrix::cols) + + /// Bare bones interface + .def("__getitem__", + [](const FortranMatrix &m, std::pair i) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + return m(i.first, i.second); + }) + .def("__setitem__", + [](FortranMatrix &m, std::pair i, float v) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + m(i.first, i.second) = v; + }) + /// Provide buffer access + .def_buffer([](FortranMatrix &m) -> py::buffer_info { + return py::buffer_info(m.data(), /* Pointer to buffer */ + {m.rows(), m.cols()}, /* Buffer dimensions */ + /* Strides (in bytes) for each index */ + {sizeof(float), sizeof(float) * size_t(m.rows())}); + }); + + // A matrix that uses a discontiguous underlying memory block. + class DiscontiguousMatrix : public Matrix { + public: + DiscontiguousMatrix(py::ssize_t rows, + py::ssize_t cols, + py::ssize_t row_factor, + py::ssize_t col_factor) + : Matrix(rows * row_factor, cols * col_factor), m_row_factor(row_factor), + m_col_factor(col_factor) { + print_created(this, + std::to_string(rows) + "(*" + std::to_string(row_factor) + ")x" + + std::to_string(cols) + "(*" + std::to_string(col_factor) + + ") matrix"); + } + + ~DiscontiguousMatrix() { + print_destroyed(this, + std::to_string(rows() / m_row_factor) + "(*" + + std::to_string(m_row_factor) + ")x" + + std::to_string(cols() / m_col_factor) + "(*" + + std::to_string(m_col_factor) + ") matrix"); + } + + float operator()(py::ssize_t i, py::ssize_t j) const { + return Matrix::operator()(i * m_row_factor, j * m_col_factor); + } + + float &operator()(py::ssize_t i, py::ssize_t j) { + return Matrix::operator()(i * m_row_factor, j * m_col_factor); + } + + using Matrix::data; + + py::ssize_t rows() const { return Matrix::rows() / m_row_factor; } + py::ssize_t cols() const { return Matrix::cols() / m_col_factor; } + py::ssize_t row_factor() const { return m_row_factor; } + py::ssize_t col_factor() const { return m_col_factor; } + + private: + py::ssize_t m_row_factor; + py::ssize_t m_col_factor; + }; + py::class_(m, "DiscontiguousMatrix", py::buffer_protocol()) + .def(py::init()) + + .def("rows", &DiscontiguousMatrix::rows) + .def("cols", &DiscontiguousMatrix::cols) + + /// Bare bones interface + .def("__getitem__", + [](const DiscontiguousMatrix &m, std::pair i) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + return m(i.first, i.second); + }) + .def("__setitem__", + [](DiscontiguousMatrix &m, std::pair i, float v) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + m(i.first, i.second) = v; + }) + /// Provide buffer access + .def_buffer([](DiscontiguousMatrix &m) -> py::buffer_info { + return py::buffer_info(m.data(), /* Pointer to buffer */ + {m.rows(), m.cols()}, /* Buffer dimensions */ + /* Strides (in bytes) for each index */ + {size_t(m.col_factor()) * sizeof(float) * size_t(m.cols()) + * size_t(m.row_factor()), + size_t(m.col_factor()) * sizeof(float)}); + }); + + class BrokenMatrix : public Matrix { + public: + BrokenMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(rows, cols) {} + void throw_runtime_error() { throw std::runtime_error("See PR #5324 for context."); } + }; + py::class_(m, "BrokenMatrix", py::buffer_protocol()) + .def(py::init()) + .def_buffer([](BrokenMatrix &m) { + m.throw_runtime_error(); + return py::buffer_info(); + }); + // test_inherited_protocol class SquareMatrix : public Matrix { public: @@ -256,4 +387,56 @@ TEST_SUBMODULE(buffers, m) { }); m.def("get_buffer_info", [](const py::buffer &buffer) { return buffer.request(); }); + + // Expose Py_buffer for testing. + m.attr("PyBUF_FORMAT") = PyBUF_FORMAT; + m.attr("PyBUF_SIMPLE") = PyBUF_SIMPLE; + m.attr("PyBUF_ND") = PyBUF_ND; + m.attr("PyBUF_STRIDES") = PyBUF_STRIDES; + m.attr("PyBUF_INDIRECT") = PyBUF_INDIRECT; + m.attr("PyBUF_C_CONTIGUOUS") = PyBUF_C_CONTIGUOUS; + m.attr("PyBUF_F_CONTIGUOUS") = PyBUF_F_CONTIGUOUS; + m.attr("PyBUF_ANY_CONTIGUOUS") = PyBUF_ANY_CONTIGUOUS; + + m.def("get_py_buffer", [](const py::object &object, int flags) { + Py_buffer buffer; + memset(&buffer, 0, sizeof(Py_buffer)); + if (PyObject_GetBuffer(object.ptr(), &buffer, flags) == -1) { + throw py::error_already_set(); + } + + auto SimpleNamespace = py::module_::import("types").attr("SimpleNamespace"); + py::object result = SimpleNamespace("len"_a = buffer.len, + "readonly"_a = buffer.readonly, + "itemsize"_a = buffer.itemsize, + "format"_a = buffer.format, + "ndim"_a = buffer.ndim, + "shape"_a = py::none(), + "strides"_a = py::none(), + "suboffsets"_a = py::none()); + if (buffer.shape != nullptr) { + py::list l; + for (auto i = 0; i < buffer.ndim; i++) { + l.append(buffer.shape[i]); + } + py::setattr(result, "shape", l); + } + if (buffer.strides != nullptr) { + py::list l; + for (auto i = 0; i < buffer.ndim; i++) { + l.append(buffer.strides[i]); + } + py::setattr(result, "strides", l); + } + if (buffer.suboffsets != nullptr) { + py::list l; + for (auto i = 0; i < buffer.ndim; i++) { + l.append(buffer.suboffsets[i]); + } + py::setattr(result, "suboffsets", l); + } + + PyBuffer_Release(&buffer); + return result; + }); } diff --git a/wrap/pybind11/tests/test_buffers.py b/wrap/pybind11/tests/test_buffers.py index 84a301e25..2612edb27 100644 --- a/wrap/pybind11/tests/test_buffers.py +++ b/wrap/pybind11/tests/test_buffers.py @@ -82,6 +82,8 @@ def test_from_python(): for j in range(m4.cols()): assert m3[i, j] == m4[i, j] + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") cstats = ConstructorStats.get(m.Matrix) assert cstats.alive() == 1 del m3, m4 @@ -118,6 +120,8 @@ def test_to_python(): mat2[2, 3] = 5 assert mat2[2, 3] == 5 + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") cstats = ConstructorStats.get(m.Matrix) assert cstats.alive() == 1 del mat @@ -228,3 +232,170 @@ def test_buffer_docstring(): m.get_buffer_info.__doc__.strip() == "get_buffer_info(arg0: Buffer) -> pybind11_tests.buffers.buffer_info" ) + + +def test_buffer_exception(): + with pytest.raises(BufferError, match="Error getting buffer") as excinfo: + memoryview(m.BrokenMatrix(1, 1)) + assert isinstance(excinfo.value.__cause__, RuntimeError) + assert "for context" in str(excinfo.value.__cause__) + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_c_contiguous_to_pybuffer(type): + if type == "pybind11": + mat = m.Matrix(5, 4) + elif type == "numpy": + mat = np.empty((5, 4), dtype=np.float32) + else: + raise ValueError(f"Unknown parametrization {type}") + + info = m.get_py_buffer(mat, m.PyBUF_SIMPLE) + assert info.format is None + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 0 # See discussion on PR #5407. + assert info.shape is None + assert info.strides is None + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_SIMPLE | m.PyBUF_FORMAT) + assert info.format == "f" + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 0 # See discussion on PR #5407. + assert info.shape is None + assert info.strides is None + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_ND) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides is None + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_STRIDES) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [4 * info.itemsize, info.itemsize] + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_INDIRECT) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [4 * info.itemsize, info.itemsize] + assert info.suboffsets is None # Should be filled in here, but we don't use it. + assert not info.readonly + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_fortran_contiguous_to_pybuffer(type): + if type == "pybind11": + mat = m.FortranMatrix(5, 4) + elif type == "numpy": + mat = np.empty((5, 4), dtype=np.float32, order="F") + else: + raise ValueError(f"Unknown parametrization {type}") + + # A Fortran-shaped buffer can only be accessed at PyBUF_STRIDES level or higher. + info = m.get_py_buffer(mat, m.PyBUF_STRIDES) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [info.itemsize, 5 * info.itemsize] + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_INDIRECT) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [info.itemsize, 5 * info.itemsize] + assert info.suboffsets is None # Should be filled in here, but we don't use it. + assert not info.readonly + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_discontiguous_to_pybuffer(type): + if type == "pybind11": + mat = m.DiscontiguousMatrix(5, 4, 2, 3) + elif type == "numpy": + mat = np.empty((5 * 2, 4 * 3), dtype=np.float32)[::2, ::3] + else: + raise ValueError(f"Unknown parametrization {type}") + + info = m.get_py_buffer(mat, m.PyBUF_STRIDES) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [2 * 4 * 3 * info.itemsize, 3 * info.itemsize] + assert info.suboffsets is None + assert not info.readonly + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_to_pybuffer_contiguity(type): + def check_strides(mat): + # The full block is memset to 0, so fill it with non-zero in real spots. + expected = np.arange(1, 5 * 4 + 1).reshape((5, 4)) + for i in range(5): + for j in range(4): + mat[i, j] = expected[i, j] + # If all strides are correct, the exposed buffer should match the input. + np.testing.assert_array_equal(np.array(mat), expected) + + if type == "pybind11": + cmat = m.Matrix(5, 4) # C contiguous. + fmat = m.FortranMatrix(5, 4) # Fortran contiguous. + dmat = m.DiscontiguousMatrix(5, 4, 2, 3) # Not contiguous. + expected_exception = BufferError + elif type == "numpy": + cmat = np.empty((5, 4), dtype=np.float32) # C contiguous. + fmat = np.empty((5, 4), dtype=np.float32, order="F") # Fortran contiguous. + dmat = np.empty((5 * 2, 4 * 3), dtype=np.float32)[::2, ::3] # Not contiguous. + # NumPy incorrectly raises ValueError; when the minimum NumPy requirement is + # above the version that fixes https://github.com/numpy/numpy/issues/3634 then + # BufferError can be used everywhere. + expected_exception = (BufferError, ValueError) + else: + raise ValueError(f"Unknown parametrization {type}") + + check_strides(cmat) + # Should work in C-contiguous mode, but not Fortran order. + m.get_py_buffer(cmat, m.PyBUF_C_CONTIGUOUS) + m.get_py_buffer(cmat, m.PyBUF_ANY_CONTIGUOUS) + with pytest.raises(expected_exception): + m.get_py_buffer(cmat, m.PyBUF_F_CONTIGUOUS) + + check_strides(fmat) + # These flags imply C-contiguity, so won't work. + with pytest.raises(expected_exception): + m.get_py_buffer(fmat, m.PyBUF_SIMPLE) + with pytest.raises(expected_exception): + m.get_py_buffer(fmat, m.PyBUF_ND) + # Should work in Fortran-contiguous mode, but not C order. + with pytest.raises(expected_exception): + m.get_py_buffer(fmat, m.PyBUF_C_CONTIGUOUS) + m.get_py_buffer(fmat, m.PyBUF_ANY_CONTIGUOUS) + m.get_py_buffer(fmat, m.PyBUF_F_CONTIGUOUS) + + check_strides(dmat) + # Should never work. + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_SIMPLE) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_ND) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_C_CONTIGUOUS) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_ANY_CONTIGUOUS) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_F_CONTIGUOUS) diff --git a/wrap/pybind11/tests/test_builtin_casters.py b/wrap/pybind11/tests/test_builtin_casters.py index 9aa5926e9..240be85e7 100644 --- a/wrap/pybind11/tests/test_builtin_casters.py +++ b/wrap/pybind11/tests/test_builtin_casters.py @@ -297,7 +297,7 @@ def test_int_convert(): cant_convert(3.14159) # TODO: Avoid DeprecationWarning in `PyLong_AsLong` (and similar) # TODO: PyPy 3.8 does not behave like CPython 3.8 here yet (7.3.7) - if (3, 8) <= sys.version_info < (3, 10) and env.CPYTHON: + if sys.version_info < (3, 10) and env.CPYTHON: with env.deprecated_call(): assert convert(Int()) == 42 else: @@ -368,6 +368,8 @@ def test_tuple(doc): """ ) + assert doc(m.empty_tuple) == """empty_tuple() -> tuple[()]""" + assert m.rvalue_pair() == ("rvalue", "rvalue") assert m.lvalue_pair() == ("lvalue", "lvalue") assert m.rvalue_tuple() == ("rvalue", "rvalue", "rvalue") diff --git a/wrap/pybind11/tests/test_call_policies.cpp b/wrap/pybind11/tests/test_call_policies.cpp index 92924cb45..9140f7e9f 100644 --- a/wrap/pybind11/tests/test_call_policies.cpp +++ b/wrap/pybind11/tests/test_call_policies.cpp @@ -95,8 +95,8 @@ TEST_SUBMODULE(call_policies, m) { }, py::call_guard()); -#if !defined(PYPY_VERSION) - // `py::call_guard()` should work in PyPy as well, +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) + // `py::call_guard()` should work in PyPy/GraalPy as well, // but it's unclear how to test it without `PyGILState_GetThisThreadState`. auto report_gil_status = []() { auto is_gil_held = false; diff --git a/wrap/pybind11/tests/test_call_policies.py b/wrap/pybind11/tests/test_call_policies.py index 91670deb3..11aab9fd9 100644 --- a/wrap/pybind11/tests/test_call_policies.py +++ b/wrap/pybind11/tests/test_call_policies.py @@ -8,6 +8,7 @@ from pybind11_tests import call_policies as m @pytest.mark.xfail("env.PYPY", reason="sometimes comes out 1 off on PyPy", strict=False) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_keep_alive_argument(capture): n_inst = ConstructorStats.detail_reg_inst() with capture: @@ -60,6 +61,7 @@ def test_keep_alive_argument(capture): assert str(excinfo.value) == "Could not activate keep_alive!" +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_keep_alive_return_value(capture): n_inst = ConstructorStats.detail_reg_inst() with capture: @@ -118,6 +120,7 @@ def test_keep_alive_return_value(capture): # https://foss.heptapod.net/pypy/pypy/-/issues/2447 @pytest.mark.xfail("env.PYPY", reason="_PyObject_GetDictPtr is unimplemented") +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alive_gc(capture): n_inst = ConstructorStats.detail_reg_inst() p = m.ParentGC() @@ -137,6 +140,7 @@ def test_alive_gc(capture): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alive_gc_derived(capture): class Derived(m.Parent): pass @@ -159,6 +163,7 @@ def test_alive_gc_derived(capture): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alive_gc_multi_derived(capture): class Derived(m.Parent, m.Child): def __init__(self): @@ -185,6 +190,7 @@ def test_alive_gc_multi_derived(capture): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_return_none(capture): n_inst = ConstructorStats.detail_reg_inst() with capture: @@ -212,6 +218,7 @@ def test_return_none(capture): assert capture == "Releasing parent." +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_keep_alive_constructor(capture): n_inst = ConstructorStats.detail_reg_inst() diff --git a/wrap/pybind11/tests/test_callbacks.cpp b/wrap/pybind11/tests/test_callbacks.cpp index 2fd05dec7..20dbaa6b2 100644 --- a/wrap/pybind11/tests/test_callbacks.cpp +++ b/wrap/pybind11/tests/test_callbacks.cpp @@ -148,7 +148,7 @@ TEST_SUBMODULE(callbacks, m) { m.def("dummy_function2", [](int i, int j) { return i + j; }); m.def( "roundtrip", - [](std::function f, bool expect_none = false) { + [](std::function f, bool expect_none) { if (expect_none && f) { throw std::runtime_error("Expected None to be converted to empty std::function"); } @@ -269,12 +269,7 @@ TEST_SUBMODULE(callbacks, m) { rec_capsule.set_name(rec_capsule_name); m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr())); - // This test requires a new ABI version to pass -#if PYBIND11_INTERNALS_VERSION > 4 // rec_capsule with nullptr name py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); }); m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr())); -#else - m.add_object("custom_function2", py::none()); -#endif } diff --git a/wrap/pybind11/tests/test_callbacks.py b/wrap/pybind11/tests/test_callbacks.py index ce2a6d254..7919ad0ae 100644 --- a/wrap/pybind11/tests/test_callbacks.py +++ b/wrap/pybind11/tests/test_callbacks.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys import time from threading import Thread @@ -89,6 +90,7 @@ def test_keyword_args_and_generalized_unpacking(): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_lambda_closure_cleanup(): m.test_lambda_closure_cleanup() cstats = m.payload_cstats() @@ -97,6 +99,7 @@ def test_lambda_closure_cleanup(): assert cstats.move_constructions >= 1 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_cpp_callable_cleanup(): alive_counts = m.test_cpp_callable_cleanup() assert alive_counts == [0, 1, 2, 1, 2, 1, 0] @@ -153,6 +156,7 @@ def test_python_builtins(): assert m.test_sum_builtin(sum, []) == 0 +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_async_callbacks(): # serves as state for async callback class Item: @@ -176,6 +180,7 @@ def test_async_callbacks(): assert sum(res) == sum(x + 3 for x in work) +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_async_async_callbacks(): t = Thread(target=test_async_callbacks) t.start() @@ -212,9 +217,7 @@ def test_custom_func(): assert m.roundtrip(m.custom_function)(4) == 36 -@pytest.mark.skipif( - m.custom_function2 is None, reason="Current PYBIND11_INTERNALS_VERSION too low" -) +@pytest.mark.skipif("env.GRAALPY", reason="TODO debug segfault") def test_custom_func2(): assert m.custom_function2(3) == 27 assert m.roundtrip(m.custom_function2)(3) == 27 diff --git a/wrap/pybind11/tests/test_class.cpp b/wrap/pybind11/tests/test_class.cpp index 9001d86b1..28dc31333 100644 --- a/wrap/pybind11/tests/test_class.cpp +++ b/wrap/pybind11/tests/test_class.cpp @@ -52,8 +52,24 @@ void bind_empty0(py::module_ &m) { } } // namespace pr4220_tripped_over_this + +namespace pr5396_forward_declared_class { +class ForwardClass; +class Args : public py::args {}; +} // namespace pr5396_forward_declared_class + } // namespace test_class +static_assert(py::detail::is_same_or_base_of::value, ""); +static_assert( + py::detail::is_same_or_base_of::value, + ""); +static_assert(!py::detail::is_same_or_base_of< + py::args, + test_class::pr5396_forward_declared_class::ForwardClass>::value, + ""); + TEST_SUBMODULE(class_, m) { m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); @@ -89,6 +105,12 @@ TEST_SUBMODULE(class_, m) { .def_static("__new__", [](const py::object &) { return NoConstructorNew::new_instance(); }); + // test_pass_unique_ptr + struct ToBeHeldByUniquePtr {}; + py::class_>(m, "ToBeHeldByUniquePtr") + .def(py::init<>()); + m.def("pass_unique_ptr", [](std::unique_ptr &&) {}); + // test_inheritance class Pet { public: @@ -211,11 +233,12 @@ TEST_SUBMODULE(class_, m) { m.def("mismatched_holder_1", []() { auto mod = py::module_::import("__main__"); py::class_>(mod, "MismatchBase1"); - py::class_(mod, "MismatchDerived1"); + py::class_, MismatchBase1>( + mod, "MismatchDerived1"); }); m.def("mismatched_holder_2", []() { auto mod = py::module_::import("__main__"); - py::class_(mod, "MismatchBase2"); + py::class_>(mod, "MismatchBase2"); py::class_, MismatchBase2>( mod, "MismatchDerived2"); }); @@ -609,8 +632,10 @@ CHECK_NOALIAS(8); CHECK_HOLDER(1, unique); CHECK_HOLDER(2, unique); CHECK_HOLDER(3, unique); +#ifndef PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE CHECK_HOLDER(4, unique); CHECK_HOLDER(5, unique); +#endif CHECK_HOLDER(6, shared); CHECK_HOLDER(7, shared); CHECK_HOLDER(8, shared); diff --git a/wrap/pybind11/tests/test_class.py b/wrap/pybind11/tests/test_class.py index 9b2b1d834..2e11feb7b 100644 --- a/wrap/pybind11/tests/test_class.py +++ b/wrap/pybind11/tests/test_class.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys from unittest import mock import pytest @@ -27,6 +28,9 @@ def test_instance(msg): instance = m.NoConstructor.new_instance() + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.NoConstructor) assert cstats.alive() == 1 del instance @@ -35,12 +39,26 @@ def test_instance(msg): def test_instance_new(): instance = m.NoConstructorNew() # .__new__(m.NoConstructor.__class__) + + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.NoConstructorNew) assert cstats.alive() == 1 del instance assert cstats.alive() == 0 +def test_pass_unique_ptr(): + obj = m.ToBeHeldByUniquePtr() + with pytest.raises(RuntimeError) as execinfo: + m.pass_unique_ptr(obj) + assert str(execinfo.value).startswith( + "Passing `std::unique_ptr` from Python to C++ requires `py::class_` (with T = " + ) + assert "ToBeHeldByUniquePtr" in str(execinfo.value) + + def test_type(): assert m.check_type(1) == m.DerivedClass1 with pytest.raises(RuntimeError) as execinfo: @@ -361,7 +379,7 @@ def test_brace_initialization(): assert b.vec == [123, 456] -@pytest.mark.xfail("env.PYPY") +@pytest.mark.xfail("env.PYPY or env.GRAALPY") def test_class_refcount(): """Instances must correctly increase/decrease the reference count of their types (#1029)""" from sys import getrefcount @@ -501,3 +519,31 @@ def test_pr4220_tripped_over_this(): m.Empty0().get_msg() == "This is really only meant to exercise successful compilation." ) + + +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") +def test_all_type_info_multithreaded(): + # See PR #5419 for background. + import threading + + from pybind11_tests import TestContext + + class Context(TestContext): + pass + + num_runs = 10 + num_threads = 4 + barrier = threading.Barrier(num_threads) + + def func(): + barrier.wait() + with Context(): + pass + + for _ in range(num_runs): + threads = [threading.Thread(target=func) for _ in range(num_threads)] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() diff --git a/wrap/pybind11/tests/test_class_release_gil_before_calling_cpp_dtor.cpp b/wrap/pybind11/tests/test_class_release_gil_before_calling_cpp_dtor.cpp new file mode 100644 index 000000000..a4869a846 --- /dev/null +++ b/wrap/pybind11/tests/test_class_release_gil_before_calling_cpp_dtor.cpp @@ -0,0 +1,53 @@ +#include + +#include "pybind11_tests.h" + +#include +#include + +namespace pybind11_tests { +namespace class_release_gil_before_calling_cpp_dtor { + +using RegistryType = std::unordered_map; + +static RegistryType &PyGILState_Check_Results() { + static RegistryType singleton; // Local static variables have thread-safe initialization. + return singleton; +} + +template // Using int as a trick to easily generate a series of types. +struct ProbeType { +private: + std::string unique_key; + +public: + explicit ProbeType(const std::string &unique_key) : unique_key{unique_key} {} + + ~ProbeType() { + RegistryType ® = PyGILState_Check_Results(); + assert(reg.count(unique_key) == 0); + reg[unique_key] = PyGILState_Check(); + } +}; + +} // namespace class_release_gil_before_calling_cpp_dtor +} // namespace pybind11_tests + +TEST_SUBMODULE(class_release_gil_before_calling_cpp_dtor, m) { + using namespace pybind11_tests::class_release_gil_before_calling_cpp_dtor; + + py::class_>(m, "ProbeType0").def(py::init()); + + py::class_>(m, "ProbeType1", py::release_gil_before_calling_cpp_dtor()) + .def(py::init()); + + m.def("PopPyGILState_Check_Result", [](const std::string &unique_key) -> std::string { + RegistryType ® = PyGILState_Check_Results(); + if (reg.count(unique_key) == 0) { + return "MISSING"; + } + int res = reg[unique_key]; + reg.erase(unique_key); + return std::to_string(res); + }); +} diff --git a/wrap/pybind11/tests/test_class_release_gil_before_calling_cpp_dtor.py b/wrap/pybind11/tests/test_class_release_gil_before_calling_cpp_dtor.py new file mode 100644 index 000000000..0b1f246bb --- /dev/null +++ b/wrap/pybind11/tests/test_class_release_gil_before_calling_cpp_dtor.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import gc + +import pytest + +from pybind11_tests import class_release_gil_before_calling_cpp_dtor as m + + +@pytest.mark.parametrize( + ("probe_type", "unique_key", "expected_result"), + [ + (m.ProbeType0, "without_manipulating_gil", "1"), + (m.ProbeType1, "release_gil_before_calling_cpp_dtor", "0"), + ], +) +def test_gil_state_check_results(probe_type, unique_key, expected_result): + probe_type(unique_key) + gc.collect() + result = m.PopPyGILState_Check_Result(unique_key) + assert result == expected_result diff --git a/wrap/pybind11/tests/test_class_sh_basic.cpp b/wrap/pybind11/tests/test_class_sh_basic.cpp new file mode 100644 index 000000000..b89372b19 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_basic.cpp @@ -0,0 +1,247 @@ +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace class_sh_basic { + +struct atyp { // Short for "any type". + std::string mtxt; + atyp() : mtxt("DefaultConstructor") {} + explicit atyp(const std::string &mtxt_) : mtxt(mtxt_) {} + atyp(const atyp &other) { mtxt = other.mtxt + "_CpCtor"; } + atyp(atyp &&other) noexcept { mtxt = other.mtxt + "_MvCtor"; } +}; + +struct uconsumer { // unique_ptr consumer + std::unique_ptr held; + bool valid() const { return static_cast(held); } + + void pass_valu(std::unique_ptr obj) { held = std::move(obj); } + void pass_rref(std::unique_ptr &&obj) { held = std::move(obj); } + std::unique_ptr rtrn_valu() { return std::move(held); } + std::unique_ptr &rtrn_lref() { return held; } + const std::unique_ptr &rtrn_cref() const { return held; } +}; + +/// Custom deleter that is default constructible. +struct custom_deleter { + std::string trace_txt; + + custom_deleter() = default; + explicit custom_deleter(const std::string &trace_txt_) : trace_txt(trace_txt_) {} + + custom_deleter(const custom_deleter &other) { trace_txt = other.trace_txt + "_CpCtor"; } + + custom_deleter &operator=(const custom_deleter &rhs) { + trace_txt = rhs.trace_txt + "_CpLhs"; + return *this; + } + + custom_deleter(custom_deleter &&other) noexcept { + trace_txt = other.trace_txt + "_MvCtorTo"; + other.trace_txt += "_MvCtorFrom"; + } + + custom_deleter &operator=(custom_deleter &&rhs) noexcept { + trace_txt = rhs.trace_txt + "_MvLhs"; + rhs.trace_txt += "_MvRhs"; + return *this; + } + + void operator()(atyp *p) const { std::default_delete()(p); } + void operator()(const atyp *p) const { std::default_delete()(p); } +}; +static_assert(std::is_default_constructible::value, ""); + +/// Custom deleter that is not default constructible. +struct custom_deleter_nd : custom_deleter { + custom_deleter_nd() = delete; + explicit custom_deleter_nd(const std::string &trace_txt_) : custom_deleter(trace_txt_) {} +}; +static_assert(!std::is_default_constructible::value, ""); + +// clang-format off + +atyp rtrn_valu() { atyp obj{"rtrn_valu"}; return obj; } +atyp&& rtrn_rref() { static atyp obj; obj.mtxt = "rtrn_rref"; return std::move(obj); } +atyp const& rtrn_cref() { static atyp obj; obj.mtxt = "rtrn_cref"; return obj; } +atyp& rtrn_mref() { static atyp obj; obj.mtxt = "rtrn_mref"; return obj; } +atyp const* rtrn_cptr() { return new atyp{"rtrn_cptr"}; } +atyp* rtrn_mptr() { return new atyp{"rtrn_mptr"}; } + +std::string pass_valu(atyp obj) { return "pass_valu:" + obj.mtxt; } // NOLINT +std::string pass_cref(atyp const& obj) { return "pass_cref:" + obj.mtxt; } +std::string pass_mref(atyp& obj) { return "pass_mref:" + obj.mtxt; } +std::string pass_cptr(atyp const* obj) { return "pass_cptr:" + obj->mtxt; } +std::string pass_mptr(atyp* obj) { return "pass_mptr:" + obj->mtxt; } + +std::shared_ptr rtrn_shmp() { return std::make_shared("rtrn_shmp"); } +std::shared_ptr rtrn_shcp() { return std::shared_ptr(new atyp{"rtrn_shcp"}); } + +std::string pass_shmp(std::shared_ptr obj) { return "pass_shmp:" + obj->mtxt; } // NOLINT +std::string pass_shcp(std::shared_ptr obj) { return "pass_shcp:" + obj->mtxt; } // NOLINT + +std::unique_ptr rtrn_uqmp() { return std::unique_ptr(new atyp{"rtrn_uqmp"}); } +std::unique_ptr rtrn_uqcp() { return std::unique_ptr(new atyp{"rtrn_uqcp"}); } + +std::string pass_uqmp(std::unique_ptr obj) { return "pass_uqmp:" + obj->mtxt; } +std::string pass_uqcp(std::unique_ptr obj) { return "pass_uqcp:" + obj->mtxt; } + +struct sddm : std::default_delete {}; +struct sddc : std::default_delete {}; + +std::unique_ptr rtrn_udmp() { return std::unique_ptr(new atyp{"rtrn_udmp"}); } +std::unique_ptr rtrn_udcp() { return std::unique_ptr(new atyp{"rtrn_udcp"}); } + +std::string pass_udmp(std::unique_ptr obj) { return "pass_udmp:" + obj->mtxt; } +std::string pass_udcp(std::unique_ptr obj) { return "pass_udcp:" + obj->mtxt; } + +std::unique_ptr rtrn_udmp_del() { return std::unique_ptr(new atyp{"rtrn_udmp_del"}, custom_deleter{"udmp_deleter"}); } +std::unique_ptr rtrn_udcp_del() { return std::unique_ptr(new atyp{"rtrn_udcp_del"}, custom_deleter{"udcp_deleter"}); } + +std::string pass_udmp_del(std::unique_ptr obj) { return "pass_udmp_del:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } +std::string pass_udcp_del(std::unique_ptr obj) { return "pass_udcp_del:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } + +std::unique_ptr rtrn_udmp_del_nd() { return std::unique_ptr(new atyp{"rtrn_udmp_del_nd"}, custom_deleter_nd{"udmp_deleter_nd"}); } +std::unique_ptr rtrn_udcp_del_nd() { return std::unique_ptr(new atyp{"rtrn_udcp_del_nd"}, custom_deleter_nd{"udcp_deleter_nd"}); } + +std::string pass_udmp_del_nd(std::unique_ptr obj) { return "pass_udmp_del_nd:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } +std::string pass_udcp_del_nd(std::unique_ptr obj) { return "pass_udcp_del_nd:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } + +// clang-format on + +// Helpers for testing. +std::string get_mtxt(atyp const &obj) { return obj.mtxt; } +std::ptrdiff_t get_ptr(atyp const &obj) { return reinterpret_cast(&obj); } + +std::unique_ptr unique_ptr_roundtrip(std::unique_ptr obj) { return obj; } + +std::string pass_unique_ptr_cref(const std::unique_ptr &obj) { return obj->mtxt; } + +const std::unique_ptr &rtrn_unique_ptr_cref(const std::string &mtxt) { + static std::unique_ptr obj{new atyp{"static_ctor_arg"}}; + if (!mtxt.empty()) { + obj->mtxt = mtxt; + } + return obj; +} + +const std::unique_ptr &unique_ptr_cref_roundtrip(const std::unique_ptr &obj) { + return obj; +} + +struct SharedPtrStash { + std::vector> stash; + void Add(const std::shared_ptr &obj) { stash.push_back(obj); } +}; + +class LocalUnusualOpRef : UnusualOpRef {}; // To avoid clashing with `py::class_`. +py::object CastUnusualOpRefConstRef(const LocalUnusualOpRef &cref) { return py::cast(cref); } +py::object CastUnusualOpRefMovable(LocalUnusualOpRef &&mvbl) { return py::cast(std::move(mvbl)); } + +TEST_SUBMODULE(class_sh_basic, m) { + namespace py = pybind11; + + py::classh(m, "atyp").def(py::init<>()).def(py::init([](const std::string &mtxt) { + atyp obj; + obj.mtxt = mtxt; + return obj; + })); + + m.def("rtrn_valu", rtrn_valu); + m.def("rtrn_rref", rtrn_rref); + m.def("rtrn_cref", rtrn_cref); + m.def("rtrn_mref", rtrn_mref); + m.def("rtrn_cptr", rtrn_cptr); + m.def("rtrn_mptr", rtrn_mptr); + + m.def("pass_valu", pass_valu); + m.def("pass_cref", pass_cref); + m.def("pass_mref", pass_mref); + m.def("pass_cptr", pass_cptr); + m.def("pass_mptr", pass_mptr); + + m.def("rtrn_shmp", rtrn_shmp); + m.def("rtrn_shcp", rtrn_shcp); + + m.def("pass_shmp", pass_shmp); + m.def("pass_shcp", pass_shcp); + + m.def("rtrn_uqmp", rtrn_uqmp); + m.def("rtrn_uqcp", rtrn_uqcp); + + m.def("pass_uqmp", pass_uqmp); + m.def("pass_uqcp", pass_uqcp); + + m.def("rtrn_udmp", rtrn_udmp); + m.def("rtrn_udcp", rtrn_udcp); + + m.def("pass_udmp", pass_udmp); + m.def("pass_udcp", pass_udcp); + + m.def("rtrn_udmp_del", rtrn_udmp_del); + m.def("rtrn_udcp_del", rtrn_udcp_del); + + m.def("pass_udmp_del", pass_udmp_del); + m.def("pass_udcp_del", pass_udcp_del); + + m.def("rtrn_udmp_del_nd", rtrn_udmp_del_nd); + m.def("rtrn_udcp_del_nd", rtrn_udcp_del_nd); + + m.def("pass_udmp_del_nd", pass_udmp_del_nd); + m.def("pass_udcp_del_nd", pass_udcp_del_nd); + + py::classh(m, "uconsumer") + .def(py::init<>()) + .def("valid", &uconsumer::valid) + .def("pass_valu", &uconsumer::pass_valu) + .def("pass_rref", &uconsumer::pass_rref) + .def("rtrn_valu", &uconsumer::rtrn_valu) + .def("rtrn_lref", &uconsumer::rtrn_lref) + .def("rtrn_cref", &uconsumer::rtrn_cref); + + // Helpers for testing. + // These require selected functions above to work first, as indicated: + m.def("get_mtxt", get_mtxt); // pass_cref + m.def("get_ptr", get_ptr); // pass_cref + + m.def("unique_ptr_roundtrip", unique_ptr_roundtrip); // pass_uqmp, rtrn_uqmp + + m.def("pass_unique_ptr_cref", pass_unique_ptr_cref); + m.def("rtrn_unique_ptr_cref", rtrn_unique_ptr_cref); + m.def("unique_ptr_cref_roundtrip", unique_ptr_cref_roundtrip); + + py::classh(m, "SharedPtrStash") + .def(py::init<>()) + .def("Add", &SharedPtrStash::Add, py::arg("obj")); + + m.def("py_type_handle_of_atyp", []() { + return py::type::handle_of(); // Exercises static_cast in this function. + }); + + // Checks for type names used as arguments + m.def("args_shared_ptr", [](std::shared_ptr p) { return p; }); + m.def("args_shared_ptr_const", [](std::shared_ptr p) { return p; }); + m.def("args_unique_ptr", [](std::unique_ptr p) { return p; }); + m.def("args_unique_ptr_const", [](std::unique_ptr p) { return p; }); + + // Make sure unique_ptr type caster accept automatic_reference return value policy. + m.def( + "rtrn_uq_automatic_reference", + []() { return std::unique_ptr(new atyp("rtrn_uq_automatic_reference")); }, + pybind11::return_value_policy::automatic_reference); + + m.def("pass_shared_ptr_ptr", [](std::shared_ptr *) {}); + + py::classh(m, "LocalUnusualOpRef"); + m.def("CallCastUnusualOpRefConstRef", + []() { return CastUnusualOpRefConstRef(LocalUnusualOpRef()); }); + m.def("CallCastUnusualOpRefMovable", + []() { return CastUnusualOpRefMovable(LocalUnusualOpRef()); }); +} + +} // namespace class_sh_basic +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_class_sh_basic.py b/wrap/pybind11/tests/test_class_sh_basic.py new file mode 100644 index 000000000..f07253a5d --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_basic.py @@ -0,0 +1,246 @@ +# Importing re before pytest after observing a PyPy CI flake when importing pytest first. +from __future__ import annotations + +import re + +import pytest + +from pybind11_tests import class_sh_basic as m + + +def test_atyp_constructors(): + obj = m.atyp() + assert obj.__class__.__name__ == "atyp" + obj = m.atyp("") + assert obj.__class__.__name__ == "atyp" + obj = m.atyp("txtm") + assert obj.__class__.__name__ == "atyp" + + +@pytest.mark.parametrize( + ("rtrn_f", "expected"), + [ + (m.rtrn_valu, "rtrn_valu(_MvCtor)*_MvCtor"), + (m.rtrn_rref, "rtrn_rref(_MvCtor)*_MvCtor"), + (m.rtrn_cref, "rtrn_cref(_MvCtor)*_CpCtor"), + (m.rtrn_mref, "rtrn_mref(_MvCtor)*_CpCtor"), + (m.rtrn_cptr, "rtrn_cptr"), + (m.rtrn_mptr, "rtrn_mptr"), + (m.rtrn_shmp, "rtrn_shmp"), + (m.rtrn_shcp, "rtrn_shcp"), + (m.rtrn_uqmp, "rtrn_uqmp"), + (m.rtrn_uqcp, "rtrn_uqcp"), + (m.rtrn_udmp, "rtrn_udmp"), + (m.rtrn_udcp, "rtrn_udcp"), + ], +) +def test_cast(rtrn_f, expected): + assert re.match(expected, m.get_mtxt(rtrn_f())) + + +@pytest.mark.parametrize( + ("pass_f", "mtxt", "expected"), + [ + (m.pass_valu, "Valu", "pass_valu:Valu(_MvCtor)*_CpCtor"), + (m.pass_cref, "Cref", "pass_cref:Cref(_MvCtor)*_MvCtor"), + (m.pass_mref, "Mref", "pass_mref:Mref(_MvCtor)*_MvCtor"), + (m.pass_cptr, "Cptr", "pass_cptr:Cptr(_MvCtor)*_MvCtor"), + (m.pass_mptr, "Mptr", "pass_mptr:Mptr(_MvCtor)*_MvCtor"), + (m.pass_shmp, "Shmp", "pass_shmp:Shmp(_MvCtor)*_MvCtor"), + (m.pass_shcp, "Shcp", "pass_shcp:Shcp(_MvCtor)*_MvCtor"), + (m.pass_uqmp, "Uqmp", "pass_uqmp:Uqmp(_MvCtor)*_MvCtor"), + (m.pass_uqcp, "Uqcp", "pass_uqcp:Uqcp(_MvCtor)*_MvCtor"), + ], +) +def test_load_with_mtxt(pass_f, mtxt, expected): + assert re.match(expected, pass_f(m.atyp(mtxt))) + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "expected"), + [ + (m.pass_udmp, m.rtrn_udmp, "pass_udmp:rtrn_udmp"), + (m.pass_udcp, m.rtrn_udcp, "pass_udcp:rtrn_udcp"), + ], +) +def test_load_with_rtrn_f(pass_f, rtrn_f, expected): + assert pass_f(rtrn_f()) == expected + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "regex_expected"), + [ + ( + m.pass_udmp_del, + m.rtrn_udmp_del, + "pass_udmp_del:rtrn_udmp_del,udmp_deleter(_MvCtorTo)*_MvCtorTo", + ), + ( + m.pass_udcp_del, + m.rtrn_udcp_del, + "pass_udcp_del:rtrn_udcp_del,udcp_deleter(_MvCtorTo)*_MvCtorTo", + ), + ( + m.pass_udmp_del_nd, + m.rtrn_udmp_del_nd, + "pass_udmp_del_nd:rtrn_udmp_del_nd,udmp_deleter_nd(_MvCtorTo)*_MvCtorTo", + ), + ( + m.pass_udcp_del_nd, + m.rtrn_udcp_del_nd, + "pass_udcp_del_nd:rtrn_udcp_del_nd,udcp_deleter_nd(_MvCtorTo)*_MvCtorTo", + ), + ], +) +def test_deleter_roundtrip(pass_f, rtrn_f, regex_expected): + assert re.match(regex_expected, pass_f(rtrn_f())) + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "expected"), + [ + (m.pass_uqmp, m.rtrn_uqmp, "pass_uqmp:rtrn_uqmp"), + (m.pass_uqcp, m.rtrn_uqcp, "pass_uqcp:rtrn_uqcp"), + (m.pass_udmp, m.rtrn_udmp, "pass_udmp:rtrn_udmp"), + (m.pass_udcp, m.rtrn_udcp, "pass_udcp:rtrn_udcp"), + ], +) +def test_pass_unique_ptr_disowns(pass_f, rtrn_f, expected): + obj = rtrn_f() + assert pass_f(obj) == expected + with pytest.raises(ValueError) as exc_info: + pass_f(obj) + assert str(exc_info.value) == ( + "Missing value for wrapped C++ type" + + " `pybind11_tests::class_sh_basic::atyp`:" + + " Python instance was disowned." + ) + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f"), + [ + (m.pass_uqmp, m.rtrn_uqmp), + (m.pass_uqcp, m.rtrn_uqcp), + (m.pass_udmp, m.rtrn_udmp), + (m.pass_udcp, m.rtrn_udcp), + ], +) +def test_cannot_disown_use_count_ne_1(pass_f, rtrn_f): + obj = rtrn_f() + stash = m.SharedPtrStash() + stash.Add(obj) + with pytest.raises(ValueError) as exc_info: + pass_f(obj) + assert str(exc_info.value) == ("Cannot disown use_count != 1 (load_as_unique_ptr).") + + +def test_unique_ptr_roundtrip(num_round_trips=1000): + # Multiple roundtrips to stress-test instance registration/deregistration. + recycled = m.atyp("passenger") + for _ in range(num_round_trips): + id_orig = id(recycled) + recycled = m.unique_ptr_roundtrip(recycled) + assert re.match("passenger(_MvCtor)*_MvCtor", m.get_mtxt(recycled)) + id_rtrn = id(recycled) + # Ensure the returned object is a different Python instance. + assert id_rtrn != id_orig + id_orig = id_rtrn + + +def test_pass_unique_ptr_cref(): + obj = m.atyp("ctor_arg") + assert re.match("ctor_arg(_MvCtor)*_MvCtor", m.get_mtxt(obj)) + assert re.match("ctor_arg(_MvCtor)*_MvCtor", m.pass_unique_ptr_cref(obj)) + assert re.match("ctor_arg(_MvCtor)*_MvCtor", m.get_mtxt(obj)) + + +def test_rtrn_unique_ptr_cref(): + obj0 = m.rtrn_unique_ptr_cref("") + assert m.get_mtxt(obj0) == "static_ctor_arg" + obj1 = m.rtrn_unique_ptr_cref("passed_mtxt_1") + assert m.get_mtxt(obj1) == "passed_mtxt_1" + assert m.get_mtxt(obj0) == "passed_mtxt_1" + assert obj0 is obj1 + + +def test_unique_ptr_cref_roundtrip(num_round_trips=1000): + # Multiple roundtrips to stress-test implementation. + orig = m.atyp("passenger") + mtxt_orig = m.get_mtxt(orig) + recycled = orig + for _ in range(num_round_trips): + recycled = m.unique_ptr_cref_roundtrip(recycled) + assert recycled is orig + assert m.get_mtxt(recycled) == mtxt_orig + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "moved_out", "moved_in"), + [ + (m.uconsumer.pass_valu, m.uconsumer.rtrn_valu, True, True), + (m.uconsumer.pass_rref, m.uconsumer.rtrn_valu, True, True), + (m.uconsumer.pass_valu, m.uconsumer.rtrn_lref, True, False), + (m.uconsumer.pass_valu, m.uconsumer.rtrn_cref, True, False), + ], +) +def test_unique_ptr_consumer_roundtrip(pass_f, rtrn_f, moved_out, moved_in): + c = m.uconsumer() + assert not c.valid() + recycled = m.atyp("passenger") + mtxt_orig = m.get_mtxt(recycled) + assert re.match("passenger_(MvCtor){1,2}", mtxt_orig) + + pass_f(c, recycled) + if moved_out: + with pytest.raises(ValueError) as excinfo: + m.get_mtxt(recycled) + assert "Python instance was disowned" in str(excinfo.value) + + recycled = rtrn_f(c) + assert c.valid() != moved_in + assert m.get_mtxt(recycled) == mtxt_orig + + +def test_py_type_handle_of_atyp(): + obj = m.py_type_handle_of_atyp() + assert obj.__class__.__name__ == "pybind11_type" + + +def test_function_signatures(doc): + assert ( + doc(m.args_shared_ptr) + == "args_shared_ptr(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + assert ( + doc(m.args_shared_ptr_const) + == "args_shared_ptr_const(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + assert ( + doc(m.args_unique_ptr) + == "args_unique_ptr(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + assert ( + doc(m.args_unique_ptr_const) + == "args_unique_ptr_const(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + + +def test_unique_ptr_return_value_policy_automatic_reference(): + assert m.get_mtxt(m.rtrn_uq_automatic_reference()) == "rtrn_uq_automatic_reference" + + +def test_pass_shared_ptr_ptr(): + obj = m.atyp() + with pytest.raises(RuntimeError) as excinfo: + m.pass_shared_ptr_ptr(obj) + assert str(excinfo.value) == ( + "Passing `std::shared_ptr *` from Python to C++ is not supported" + " (inherently unsafe)." + ) + + +def test_unusual_op_ref(): + # Merely to test that this still exists and built successfully. + assert m.CallCastUnusualOpRefConstRef().__class__.__name__ == "LocalUnusualOpRef" + assert m.CallCastUnusualOpRefMovable().__class__.__name__ == "LocalUnusualOpRef" diff --git a/wrap/pybind11/tests/test_class_sh_disowning.cpp b/wrap/pybind11/tests/test_class_sh_disowning.cpp new file mode 100644 index 000000000..490e6d59f --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_disowning.cpp @@ -0,0 +1,41 @@ +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_disowning { + +template // Using int as a trick to easily generate a series of types. +struct Atype { + int val = 0; + explicit Atype(int val_) : val{val_} {} + int get() const { return val * 10 + SerNo; } +}; + +int same_twice(std::unique_ptr> at1a, std::unique_ptr> at1b) { + return at1a->get() * 100 + at1b->get() * 10; +} + +int mixed(std::unique_ptr> at1, std::unique_ptr> at2) { + return at1->get() * 200 + at2->get() * 20; +} + +int overloaded(std::unique_ptr> at1, int i) { return at1->get() * 30 + i; } +int overloaded(std::unique_ptr> at2, int i) { return at2->get() * 40 + i; } + +} // namespace class_sh_disowning +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_disowning, m) { + using namespace pybind11_tests::class_sh_disowning; + + py::classh>(m, "Atype1").def(py::init()).def("get", &Atype<1>::get); + py::classh>(m, "Atype2").def(py::init()).def("get", &Atype<2>::get); + + m.def("same_twice", same_twice); + + m.def("mixed", mixed); + + m.def("overloaded", (int (*)(std::unique_ptr>, int)) &overloaded); + m.def("overloaded", (int (*)(std::unique_ptr>, int)) &overloaded); +} diff --git a/wrap/pybind11/tests/test_class_sh_disowning.py b/wrap/pybind11/tests/test_class_sh_disowning.py new file mode 100644 index 000000000..b9e648999 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_disowning.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_disowning as m + + +def is_disowned(obj): + try: + obj.get() + except ValueError: + return True + return False + + +def test_same_twice(): + while True: + obj1a = m.Atype1(57) + obj1b = m.Atype1(62) + assert m.same_twice(obj1a, obj1b) == (57 * 10 + 1) * 100 + (62 * 10 + 1) * 10 + assert is_disowned(obj1a) + assert is_disowned(obj1b) + obj1c = m.Atype1(0) + with pytest.raises(ValueError): + # Disowning works for one argument, but not both. + m.same_twice(obj1c, obj1c) + assert is_disowned(obj1c) + return # Comment out for manual leak checking (use `top` command). + + +def test_mixed(): + first_pass = True + while True: + obj1a = m.Atype1(90) + obj2a = m.Atype2(25) + assert m.mixed(obj1a, obj2a) == (90 * 10 + 1) * 200 + (25 * 10 + 2) * 20 + assert is_disowned(obj1a) + assert is_disowned(obj2a) + + # The C++ order of evaluation of function arguments is (unfortunately) unspecified: + # https://en.cppreference.com/w/cpp/language/eval_order + # Read on. + obj1b = m.Atype1(0) + with pytest.raises(ValueError): + # If the 1st argument is evaluated first, obj1b is disowned before the conversion for + # the already disowned obj2a fails as expected. + m.mixed(obj1b, obj2a) + obj2b = m.Atype2(0) + with pytest.raises(ValueError): + # If the 2nd argument is evaluated first, obj2b is disowned before the conversion for + # the already disowned obj1a fails as expected. + m.mixed(obj1a, obj2b) + + # Either obj1b or obj2b was disowned in the expected failed m.mixed() calls above, but not + # both. + is_disowned_results = (is_disowned(obj1b), is_disowned(obj2b)) + assert is_disowned_results.count(True) == 1 + if first_pass: + first_pass = False + ix = is_disowned_results.index(True) + 1 + print(f"\nC++ function argument {ix} is evaluated first.") + + return # Comment out for manual leak checking (use `top` command). + + +def test_overloaded(): + while True: + obj1 = m.Atype1(81) + obj2 = m.Atype2(60) + with pytest.raises(TypeError): + m.overloaded(obj1, "NotInt") + assert obj1.get() == 81 * 10 + 1 # Not disowned. + assert m.overloaded(obj1, 3) == (81 * 10 + 1) * 30 + 3 + with pytest.raises(TypeError): + m.overloaded(obj2, "NotInt") + assert obj2.get() == 60 * 10 + 2 # Not disowned. + assert m.overloaded(obj2, 2) == (60 * 10 + 2) * 40 + 2 + return # Comment out for manual leak checking (use `top` command). diff --git a/wrap/pybind11/tests/test_class_sh_disowning_mi.cpp b/wrap/pybind11/tests/test_class_sh_disowning_mi.cpp new file mode 100644 index 000000000..d0ffd45ec --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_disowning_mi.cpp @@ -0,0 +1,85 @@ +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_disowning_mi { + +// Diamond inheritance (copied from test_multiple_inheritance.cpp). +struct B { + int val_b = 10; + B() = default; + B(const B &) = default; + virtual ~B() = default; +}; + +struct C0 : public virtual B { + int val_c0 = 20; +}; + +struct C1 : public virtual B { + int val_c1 = 21; +}; + +struct D : public C0, public C1 { + int val_d = 30; +}; + +void disown_b(std::unique_ptr) {} + +// test_multiple_inheritance_python +struct Base1 { + explicit Base1(int i) : i(i) {} + int foo() const { return i; } + int i; +}; + +struct Base2 { + explicit Base2(int j) : j(j) {} + int bar() const { return j; } + int j; +}; + +int disown_base1(std::unique_ptr b1) { return b1->i * 2000 + 1; } +int disown_base2(std::unique_ptr b2) { return b2->j * 2000 + 2; } + +} // namespace class_sh_disowning_mi +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_disowning_mi, m) { + using namespace pybind11_tests::class_sh_disowning_mi; + + py::classh(m, "B") + .def(py::init<>()) + .def_readonly("val_b", &D::val_b) + .def("b", [](B *self) { return self; }) + .def("get", [](const B &self) { return self.val_b; }); + + py::classh(m, "C0") + .def(py::init<>()) + .def_readonly("val_c0", &D::val_c0) + .def("c0", [](C0 *self) { return self; }) + .def("get", [](const C0 &self) { return self.val_b * 100 + self.val_c0; }); + + py::classh(m, "C1") + .def(py::init<>()) + .def_readonly("val_c1", &D::val_c1) + .def("c1", [](C1 *self) { return self; }) + .def("get", [](const C1 &self) { return self.val_b * 100 + self.val_c1; }); + + py::classh(m, "D") + .def(py::init<>()) + .def_readonly("val_d", &D::val_d) + .def("d", [](D *self) { return self; }) + .def("get", [](const D &self) { + return self.val_b * 1000000 + self.val_c0 * 10000 + self.val_c1 * 100 + self.val_d; + }); + + m.def("disown_b", disown_b); + + // test_multiple_inheritance_python + py::classh(m, "Base1").def(py::init()).def("foo", &Base1::foo); + py::classh(m, "Base2").def(py::init()).def("bar", &Base2::bar); + m.def("disown_base1", disown_base1); + m.def("disown_base2", disown_base2); +} diff --git a/wrap/pybind11/tests/test_class_sh_disowning_mi.py b/wrap/pybind11/tests/test_class_sh_disowning_mi.py new file mode 100644 index 000000000..4a4beecce --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_disowning_mi.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import pytest + +import env # noqa: F401 +from pybind11_tests import class_sh_disowning_mi as m + + +def test_diamond_inheritance(): + # Very similar to test_multiple_inheritance.py:test_diamond_inheritance. + d = m.D() + assert d is d.d() + assert d is d.c0() + assert d is d.c1() + assert d is d.b() + assert d is d.c0().b() + assert d is d.c1().b() + assert d is d.c0().c1().b().c0().b() + + +def is_disowned(callable_method): + try: + callable_method() + except ValueError as e: + assert "Python instance was disowned" in str(e) # noqa: PT017 + return True + return False + + +def test_disown_b(): + b = m.B() + assert b.get() == 10 + m.disown_b(b) + assert is_disowned(b.get) + + +@pytest.mark.parametrize("var_to_disown", ["c0", "b"]) +def test_disown_c0(var_to_disown): + c0 = m.C0() + assert c0.get() == 1020 + b = c0.b() + m.disown_b(locals()[var_to_disown]) + assert is_disowned(c0.get) + assert is_disowned(b.get) + + +@pytest.mark.parametrize("var_to_disown", ["c1", "b"]) +def test_disown_c1(var_to_disown): + c1 = m.C1() + assert c1.get() == 1021 + b = c1.b() + m.disown_b(locals()[var_to_disown]) + assert is_disowned(c1.get) + assert is_disowned(b.get) + + +@pytest.mark.parametrize("var_to_disown", ["d", "c1", "c0", "b"]) +def test_disown_d(var_to_disown): + d = m.D() + assert d.get() == 10202130 + b = d.b() + c0 = d.c0() + c1 = d.c1() + m.disown_b(locals()[var_to_disown]) + assert is_disowned(d.get) + assert is_disowned(c1.get) + assert is_disowned(c0.get) + assert is_disowned(b.get) + + +# Based on test_multiple_inheritance.py:test_multiple_inheritance_python. +class MI1(m.Base1, m.Base2): + def __init__(self, i, j): + m.Base1.__init__(self, i) + m.Base2.__init__(self, j) + + +class B1: + def v(self): + return 1 + + +class MI2(B1, m.Base1, m.Base2): + def __init__(self, i, j): + B1.__init__(self) + m.Base1.__init__(self, i) + m.Base2.__init__(self, j) + + +class MI3(MI2): + def __init__(self, i, j): + MI2.__init__(self, i, j) + + +class MI4(MI3, m.Base2): + def __init__(self, i, j): + MI3.__init__(self, i, j) + # This should be ignored (Base2 is already initialized via MI2): + m.Base2.__init__(self, i + 100) + + +class MI5(m.Base2, B1, m.Base1): + def __init__(self, i, j): + B1.__init__(self) + m.Base1.__init__(self, i) + m.Base2.__init__(self, j) + + +class MI6(m.Base2, B1): + def __init__(self, i): + m.Base2.__init__(self, i) + B1.__init__(self) + + +class B2(B1): + def v(self): + return 2 + + +class B3: + def v(self): + return 3 + + +class B4(B3, B2): + def v(self): + return 4 + + +class MI7(B4, MI6): + def __init__(self, i): + B4.__init__(self) + MI6.__init__(self, i) + + +class MI8(MI6, B3): + def __init__(self, i): + MI6.__init__(self, i) + B3.__init__(self) + + +class MI8b(B3, MI6): + def __init__(self, i): + B3.__init__(self) + MI6.__init__(self, i) + + +@pytest.mark.xfail("env.PYPY") +def test_multiple_inheritance_python(): + # Based on test_multiple_inheritance.py:test_multiple_inheritance_python. + # Exercises values_and_holders with 2 value_and_holder instances. + + mi1 = MI1(1, 2) + assert mi1.foo() == 1 + assert mi1.bar() == 2 + + mi2 = MI2(3, 4) + assert mi2.v() == 1 + assert mi2.foo() == 3 + assert mi2.bar() == 4 + + mi3 = MI3(5, 6) + assert mi3.v() == 1 + assert mi3.foo() == 5 + assert mi3.bar() == 6 + + mi4 = MI4(7, 8) + assert mi4.v() == 1 + assert mi4.foo() == 7 + assert mi4.bar() == 8 + + mi5 = MI5(10, 11) + assert mi5.v() == 1 + assert mi5.foo() == 10 + assert mi5.bar() == 11 + + mi6 = MI6(12) + assert mi6.v() == 1 + assert mi6.bar() == 12 + + mi7 = MI7(13) + assert mi7.v() == 4 + assert mi7.bar() == 13 + + mi8 = MI8(14) + assert mi8.v() == 1 + assert mi8.bar() == 14 + + mi8b = MI8b(15) + assert mi8b.v() == 3 + assert mi8b.bar() == 15 + + +DISOWN_CLS_I_J_V_LIST = [ + (MI1, 1, 2, None), + (MI2, 3, 4, 1), + (MI3, 5, 6, 1), + (MI4, 7, 8, 1), + (MI5, 10, 11, 1), +] + + +@pytest.mark.xfail("env.PYPY", strict=False) +@pytest.mark.parametrize(("cls", "i", "j", "v"), DISOWN_CLS_I_J_V_LIST) +def test_disown_base1_first(cls, i, j, v): + obj = cls(i, j) + assert obj.foo() == i + assert m.disown_base1(obj) == 2000 * i + 1 + assert is_disowned(obj.foo) + assert obj.bar() == j + assert m.disown_base2(obj) == 2000 * j + 2 + assert is_disowned(obj.bar) + if v is not None: + assert obj.v() == v + + +@pytest.mark.xfail("env.PYPY", strict=False) +@pytest.mark.parametrize(("cls", "i", "j", "v"), DISOWN_CLS_I_J_V_LIST) +def test_disown_base2_first(cls, i, j, v): + obj = cls(i, j) + assert obj.bar() == j + assert m.disown_base2(obj) == 2000 * j + 2 + assert is_disowned(obj.bar) + assert obj.foo() == i + assert m.disown_base1(obj) == 2000 * i + 1 + assert is_disowned(obj.foo) + if v is not None: + assert obj.v() == v + + +@pytest.mark.xfail("env.PYPY", strict=False) +@pytest.mark.parametrize( + ("cls", "j", "v"), + [ + (MI6, 12, 1), + (MI7, 13, 4), + (MI8, 14, 1), + (MI8b, 15, 3), + ], +) +def test_disown_base2(cls, j, v): + obj = cls(j) + assert obj.bar() == j + assert m.disown_base2(obj) == 2000 * j + 2 + assert is_disowned(obj.bar) + assert obj.v() == v diff --git a/wrap/pybind11/tests/test_class_sh_factory_constructors.cpp b/wrap/pybind11/tests/test_class_sh_factory_constructors.cpp new file mode 100644 index 000000000..b3a8daea5 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_factory_constructors.cpp @@ -0,0 +1,164 @@ +#include "pybind11_tests.h" + +#include +#include + +namespace pybind11_tests { +namespace class_sh_factory_constructors { + +template // Using int as a trick to easily generate a series of types. +struct atyp { // Short for "any type". + std::string mtxt; +}; + +template +std::string get_mtxt(const T &obj) { + return obj.mtxt; +} + +using atyp_valu = atyp<0x0>; +using atyp_rref = atyp<0x1>; +using atyp_cref = atyp<0x2>; +using atyp_mref = atyp<0x3>; +using atyp_cptr = atyp<0x4>; +using atyp_mptr = atyp<0x5>; +using atyp_shmp = atyp<0x6>; +using atyp_shcp = atyp<0x7>; +using atyp_uqmp = atyp<0x8>; +using atyp_uqcp = atyp<0x9>; +using atyp_udmp = atyp<0xA>; +using atyp_udcp = atyp<0xB>; + +// clang-format off + +atyp_valu rtrn_valu() { atyp_valu obj{"Valu"}; return obj; } +atyp_rref&& rtrn_rref() { static atyp_rref obj; obj.mtxt = "Rref"; return std::move(obj); } +atyp_cref const& rtrn_cref() { static atyp_cref obj; obj.mtxt = "Cref"; return obj; } +atyp_mref& rtrn_mref() { static atyp_mref obj; obj.mtxt = "Mref"; return obj; } +atyp_cptr const* rtrn_cptr() { return new atyp_cptr{"Cptr"}; } +atyp_mptr* rtrn_mptr() { return new atyp_mptr{"Mptr"}; } + +std::shared_ptr rtrn_shmp() { return std::make_shared(atyp_shmp{"Shmp"}); } +std::shared_ptr rtrn_shcp() { return std::shared_ptr(new atyp_shcp{"Shcp"}); } + +std::unique_ptr rtrn_uqmp() { return std::unique_ptr(new atyp_uqmp{"Uqmp"}); } +std::unique_ptr rtrn_uqcp() { return std::unique_ptr(new atyp_uqcp{"Uqcp"}); } + +struct sddm : std::default_delete {}; +struct sddc : std::default_delete {}; + +std::unique_ptr rtrn_udmp() { return std::unique_ptr(new atyp_udmp{"Udmp"}); } +std::unique_ptr rtrn_udcp() { return std::unique_ptr(new atyp_udcp{"Udcp"}); } + +// clang-format on + +// Minimalistic approach to achieve full coverage of construct() overloads for constructing +// smart_holder from unique_ptr and shared_ptr returns. +struct with_alias { + int val = 0; + virtual ~with_alias() = default; + // Some compilers complain about implicitly defined versions of some of the following: + with_alias() = default; + with_alias(const with_alias &) = default; + with_alias(with_alias &&) = default; + with_alias &operator=(const with_alias &) = default; + with_alias &operator=(with_alias &&) = default; +}; +struct with_alias_alias : with_alias {}; +struct sddwaa : std::default_delete {}; + +} // namespace class_sh_factory_constructors +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_factory_constructors, m) { + using namespace pybind11_tests::class_sh_factory_constructors; + + py::classh(m, "atyp_valu") + .def(py::init(&rtrn_valu)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_rref") + .def(py::init(&rtrn_rref)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_cref") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_cref)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_mref") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_mref)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_cptr") + // class_: ... must return a compatible ... + // classh: ... must return a compatible ... + // .def(py::init(&rtrn_cptr)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_mptr") + .def(py::init(&rtrn_mptr)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_shmp") + .def(py::init(&rtrn_shmp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_shcp") + // py::class_>(m, "atyp_shcp") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_shcp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_uqmp") + .def(py::init(&rtrn_uqmp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_uqcp") + // class_: ... cannot pass object of non-trivial type ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_uqcp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_udmp") + .def(py::init(&rtrn_udmp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_udcp") + // py::class_>(m, "atyp_udcp") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_udcp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "with_alias") + .def_readonly("val", &with_alias::val) + .def(py::init([](int i) { + auto p = std::unique_ptr(new with_alias_alias); + p->val = i * 100; + return p; + })) + .def(py::init([](int i, int j) { + auto p = std::unique_ptr(new with_alias_alias); + p->val = i * 100 + j * 10; + return p; + })) + .def(py::init([](int i, int j, int k) { + auto p = std::make_shared(); + p->val = i * 100 + j * 10 + k; + return p; + })) + .def(py::init( + [](int, int, int, int) { return std::unique_ptr(new with_alias); }, + [](int, int, int, int) { + return std::unique_ptr(new with_alias); // Invalid alias factory. + })) + .def(py::init([](int, int, int, int, int) { return std::make_shared(); }, + [](int, int, int, int, int) { + return std::make_shared(); // Invalid alias factory. + })); +} diff --git a/wrap/pybind11/tests/test_class_sh_factory_constructors.py b/wrap/pybind11/tests/test_class_sh_factory_constructors.py new file mode 100644 index 000000000..5d45db6fd --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_factory_constructors.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_factory_constructors as m + + +def test_atyp_factories(): + assert m.atyp_valu().get_mtxt() == "Valu" + assert m.atyp_rref().get_mtxt() == "Rref" + # sert m.atyp_cref().get_mtxt() == "Cref" + # sert m.atyp_mref().get_mtxt() == "Mref" + # sert m.atyp_cptr().get_mtxt() == "Cptr" + assert m.atyp_mptr().get_mtxt() == "Mptr" + assert m.atyp_shmp().get_mtxt() == "Shmp" + # sert m.atyp_shcp().get_mtxt() == "Shcp" + assert m.atyp_uqmp().get_mtxt() == "Uqmp" + # sert m.atyp_uqcp().get_mtxt() == "Uqcp" + assert m.atyp_udmp().get_mtxt() == "Udmp" + # sert m.atyp_udcp().get_mtxt() == "Udcp" + + +@pytest.mark.parametrize( + ("init_args", "expected"), + [ + ((3,), 300), + ((5, 7), 570), + ((9, 11, 13), 1023), + ], +) +def test_with_alias_success(init_args, expected): + assert m.with_alias(*init_args).val == expected + + +@pytest.mark.parametrize( + ("num_init_args", "smart_ptr"), + [ + (4, "std::unique_ptr"), + (5, "std::shared_ptr"), + ], +) +def test_with_alias_invalid(num_init_args, smart_ptr): + class PyDrvdWithAlias(m.with_alias): + pass + + with pytest.raises(TypeError) as excinfo: + PyDrvdWithAlias(*((0,) * num_init_args)) + assert ( + str(excinfo.value) + == "pybind11::init(): construction failed: returned " + + smart_ptr + + " pointee is not an alias instance" + ) diff --git a/wrap/pybind11/tests/test_class_sh_inheritance.cpp b/wrap/pybind11/tests/test_class_sh_inheritance.cpp new file mode 100644 index 000000000..8bdd0a7f8 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_inheritance.cpp @@ -0,0 +1,90 @@ +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_inheritance { + +template +struct base_template { + base_template() : base_id(Id) {} + virtual ~base_template() = default; + virtual int id() const { return base_id; } + int base_id; + + // Some compilers complain about implicitly defined versions of some of the following: + base_template(const base_template &) = default; + base_template(base_template &&) noexcept = default; + base_template &operator=(const base_template &) = default; + base_template &operator=(base_template &&) noexcept = default; +}; + +using base = base_template<100>; + +struct drvd : base { + int id() const override { return 2 * base_id; } +}; + +// clang-format off +inline drvd *rtrn_mptr_drvd() { return new drvd; } +inline base *rtrn_mptr_drvd_up_cast() { return new drvd; } + +inline int pass_cptr_base(base const *b) { return b->id() + 11; } +inline int pass_cptr_drvd(drvd const *d) { return d->id() + 12; } + +inline std::shared_ptr rtrn_shmp_drvd() { return std::make_shared(); } +inline std::shared_ptr rtrn_shmp_drvd_up_cast() { return std::make_shared(); } + +inline int pass_shcp_base(const std::shared_ptr& b) { return b->id() + 21; } +inline int pass_shcp_drvd(const std::shared_ptr& d) { return d->id() + 22; } +// clang-format on + +using base1 = base_template<110>; +using base2 = base_template<120>; + +// Not reusing base here because it would interfere with the single-inheritance test. +struct drvd2 : base1, base2 { + int id() const override { return 3 * base1::base_id + 4 * base2::base_id; } +}; + +// clang-format off +inline drvd2 *rtrn_mptr_drvd2() { return new drvd2; } +inline base1 *rtrn_mptr_drvd2_up_cast1() { return new drvd2; } +inline base2 *rtrn_mptr_drvd2_up_cast2() { return new drvd2; } + +inline int pass_cptr_base1(base1 const *b) { return b->id() + 21; } +inline int pass_cptr_base2(base2 const *b) { return b->id() + 22; } +inline int pass_cptr_drvd2(drvd2 const *d) { return d->id() + 23; } +// clang-format on + +TEST_SUBMODULE(class_sh_inheritance, m) { + py::classh(m, "base"); + py::classh(m, "drvd"); + + auto rvto = py::return_value_policy::take_ownership; + + m.def("rtrn_mptr_drvd", rtrn_mptr_drvd, rvto); + m.def("rtrn_mptr_drvd_up_cast", rtrn_mptr_drvd_up_cast, rvto); + m.def("pass_cptr_base", pass_cptr_base); + m.def("pass_cptr_drvd", pass_cptr_drvd); + + m.def("rtrn_shmp_drvd", rtrn_shmp_drvd); + m.def("rtrn_shmp_drvd_up_cast", rtrn_shmp_drvd_up_cast); + m.def("pass_shcp_base", pass_shcp_base); + m.def("pass_shcp_drvd", pass_shcp_drvd); + + // __init__ needed for Python inheritance. + py::classh(m, "base1").def(py::init<>()); + py::classh(m, "base2").def(py::init<>()); + py::classh(m, "drvd2"); + + m.def("rtrn_mptr_drvd2", rtrn_mptr_drvd2, rvto); + m.def("rtrn_mptr_drvd2_up_cast1", rtrn_mptr_drvd2_up_cast1, rvto); + m.def("rtrn_mptr_drvd2_up_cast2", rtrn_mptr_drvd2_up_cast2, rvto); + m.def("pass_cptr_base1", pass_cptr_base1); + m.def("pass_cptr_base2", pass_cptr_base2); + m.def("pass_cptr_drvd2", pass_cptr_drvd2); +} + +} // namespace class_sh_inheritance +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_class_sh_inheritance.py b/wrap/pybind11/tests/test_class_sh_inheritance.py new file mode 100644 index 000000000..cd9d6f47e --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_inheritance.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from pybind11_tests import class_sh_inheritance as m + + +def test_rtrn_mptr_drvd_pass_cptr_base(): + d = m.rtrn_mptr_drvd() + i = m.pass_cptr_base(d) # load_impl Case 2a + assert i == 2 * 100 + 11 + + +def test_rtrn_shmp_drvd_pass_shcp_base(): + d = m.rtrn_shmp_drvd() + i = m.pass_shcp_base(d) # load_impl Case 2a + assert i == 2 * 100 + 21 + + +def test_rtrn_mptr_drvd_up_cast_pass_cptr_drvd(): + b = m.rtrn_mptr_drvd_up_cast() + # the base return is down-cast immediately. + assert b.__class__.__name__ == "drvd" + i = m.pass_cptr_drvd(b) + assert i == 2 * 100 + 12 + + +def test_rtrn_shmp_drvd_up_cast_pass_shcp_drvd(): + b = m.rtrn_shmp_drvd_up_cast() + # the base return is down-cast immediately. + assert b.__class__.__name__ == "drvd" + i = m.pass_shcp_drvd(b) + assert i == 2 * 100 + 22 + + +def test_rtrn_mptr_drvd2_pass_cptr_bases(): + d = m.rtrn_mptr_drvd2() + i1 = m.pass_cptr_base1(d) # load_impl Case 2c + assert i1 == 3 * 110 + 4 * 120 + 21 + i2 = m.pass_cptr_base2(d) + assert i2 == 3 * 110 + 4 * 120 + 22 + + +def test_rtrn_mptr_drvd2_up_casts_pass_cptr_drvd2(): + b1 = m.rtrn_mptr_drvd2_up_cast1() + assert b1.__class__.__name__ == "drvd2" + i1 = m.pass_cptr_drvd2(b1) + assert i1 == 3 * 110 + 4 * 120 + 23 + b2 = m.rtrn_mptr_drvd2_up_cast2() + assert b2.__class__.__name__ == "drvd2" + i2 = m.pass_cptr_drvd2(b2) + assert i2 == 3 * 110 + 4 * 120 + 23 + + +def test_python_drvd2(): + class Drvd2(m.base1, m.base2): + def __init__(self): + m.base1.__init__(self) + m.base2.__init__(self) + + d = Drvd2() + i1 = m.pass_cptr_base1(d) # load_impl Case 2b + assert i1 == 110 + 21 + i2 = m.pass_cptr_base2(d) + assert i2 == 120 + 22 diff --git a/wrap/pybind11/tests/test_class_sh_mi_thunks.cpp b/wrap/pybind11/tests/test_class_sh_mi_thunks.cpp new file mode 100644 index 000000000..d8548ec5c --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_mi_thunks.cpp @@ -0,0 +1,93 @@ +#include "pybind11_tests.h" + +#include +#include +#include + +namespace test_class_sh_mi_thunks { + +// For general background: https://shaharmike.com/cpp/vtable-part2/ +// C++ vtables - Part 2 - Multiple Inheritance +// ... the compiler creates a 'thunk' method that corrects `this` ... + +struct Base0 { + virtual ~Base0() = default; + Base0() = default; + Base0(const Base0 &) = delete; +}; + +struct Base1 { + virtual ~Base1() = default; + // Using `vector` here because it is known to make this test very sensitive to bugs. + std::vector vec = {1, 2, 3, 4, 5}; + Base1() = default; + Base1(const Base1 &) = delete; +}; + +struct Derived : Base1, Base0 { + ~Derived() override = default; + Derived() = default; + Derived(const Derived &) = delete; +}; + +} // namespace test_class_sh_mi_thunks + +TEST_SUBMODULE(class_sh_mi_thunks, m) { + using namespace test_class_sh_mi_thunks; + + m.def("ptrdiff_drvd_base0", []() { + auto drvd = std::unique_ptr(new Derived); + auto *base0 = dynamic_cast(drvd.get()); + return std::ptrdiff_t(reinterpret_cast(drvd.get()) + - reinterpret_cast(base0)); + }); + + py::classh(m, "Base0"); + py::classh(m, "Base1"); + py::classh(m, "Derived"); + + m.def( + "get_drvd_as_base0_raw_ptr", + []() { + auto *drvd = new Derived; + auto *base0 = dynamic_cast(drvd); + return base0; + }, + py::return_value_policy::take_ownership); + + m.def("get_drvd_as_base0_shared_ptr", []() { + auto drvd = std::make_shared(); + auto base0 = std::dynamic_pointer_cast(drvd); + return base0; + }); + + m.def("get_drvd_as_base0_unique_ptr", []() { + auto drvd = std::unique_ptr(new Derived); + auto base0 = std::unique_ptr(std::move(drvd)); + return base0; + }); + + m.def("vec_size_base0_raw_ptr", [](const Base0 *obj) { + const auto *obj_der = dynamic_cast(obj); + if (obj_der == nullptr) { + return std::size_t(0); + } + return obj_der->vec.size(); + }); + + m.def("vec_size_base0_shared_ptr", [](const std::shared_ptr &obj) -> std::size_t { + const auto obj_der = std::dynamic_pointer_cast(obj); + if (!obj_der) { + return std::size_t(0); + } + return obj_der->vec.size(); + }); + + m.def("vec_size_base0_unique_ptr", [](std::unique_ptr obj) -> std::size_t { + const auto *obj_der = dynamic_cast(obj.get()); + if (obj_der == nullptr) { + return std::size_t(0); + } + return obj_der->vec.size(); + }); +} diff --git a/wrap/pybind11/tests/test_class_sh_mi_thunks.py b/wrap/pybind11/tests/test_class_sh_mi_thunks.py new file mode 100644 index 000000000..32bf47554 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_mi_thunks.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_mi_thunks as m + + +def test_ptrdiff_drvd_base0(): + ptrdiff = m.ptrdiff_drvd_base0() + # A failure here does not (necessarily) mean that there is a bug, but that + # test_class_sh_mi_thunks is not exercising what it is supposed to. + # If this ever fails on some platforms: use pytest.skip() + # If this ever fails on all platforms: don't know, seems extremely unlikely. + assert ptrdiff != 0 + + +@pytest.mark.parametrize( + "vec_size_fn", + [ + m.vec_size_base0_raw_ptr, + m.vec_size_base0_shared_ptr, + ], +) +@pytest.mark.parametrize( + "get_fn", + [ + m.get_drvd_as_base0_raw_ptr, + m.get_drvd_as_base0_shared_ptr, + m.get_drvd_as_base0_unique_ptr, + ], +) +def test_get_vec_size_raw_shared(get_fn, vec_size_fn): + obj = get_fn() + assert vec_size_fn(obj) == 5 + + +@pytest.mark.parametrize( + "get_fn", [m.get_drvd_as_base0_raw_ptr, m.get_drvd_as_base0_unique_ptr] +) +def test_get_vec_size_unique(get_fn): + obj = get_fn() + assert m.vec_size_base0_unique_ptr(obj) == 5 + with pytest.raises(ValueError, match="Python instance was disowned"): + m.vec_size_base0_unique_ptr(obj) + + +def test_get_shared_vec_size_unique(): + obj = m.get_drvd_as_base0_shared_ptr() + with pytest.raises(ValueError) as exc_info: + m.vec_size_base0_unique_ptr(obj) + assert ( + str(exc_info.value) == "Cannot disown external shared_ptr (load_as_unique_ptr)." + ) diff --git a/wrap/pybind11/tests/test_class_sh_property.cpp b/wrap/pybind11/tests/test_class_sh_property.cpp new file mode 100644 index 000000000..8863ad7d7 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_property.cpp @@ -0,0 +1,94 @@ +// The compact 4-character naming matches that in test_class_sh_basic.cpp +// Variable names are intentionally terse, to not distract from the more important C++ type names: +// valu(e), ref(erence), ptr or p (pointer), r = rvalue, m = mutable, c = const, +// sh = shared_ptr, uq = unique_ptr. + +#include "pybind11_tests.h" + +#include + +namespace test_class_sh_property { + +struct ClassicField { + int num = -88; +}; + +struct ClassicOuter { + ClassicField *m_mptr = nullptr; + const ClassicField *m_cptr = nullptr; +}; + +struct Field { + int num = -99; +}; + +struct Outer { + Field m_valu; + Field *m_mptr = nullptr; + const Field *m_cptr = nullptr; + std::unique_ptr m_uqmp; + std::unique_ptr m_uqcp; + std::shared_ptr m_shmp; + std::shared_ptr m_shcp; +}; + +inline void DisownOuter(std::unique_ptr) {} + +struct WithCharArrayMember { + WithCharArrayMember() { std::memcpy(char6_member, "Char6", 6); } + char char6_member[6]; +}; + +struct WithConstCharPtrMember { + const char *const_char_ptr_member = "ConstChar*"; +}; + +} // namespace test_class_sh_property + +TEST_SUBMODULE(class_sh_property, m) { + using namespace test_class_sh_property; + + py::class_>(m, "ClassicField") + .def(py::init<>()) + .def_readwrite("num", &ClassicField::num); + + py::class_>(m, "ClassicOuter") + .def(py::init<>()) + .def_readonly("m_mptr_readonly", &ClassicOuter::m_mptr) + .def_readwrite("m_mptr_readwrite", &ClassicOuter::m_mptr) + .def_readwrite("m_cptr_readonly", &ClassicOuter::m_cptr) + .def_readwrite("m_cptr_readwrite", &ClassicOuter::m_cptr); + + py::classh(m, "Field").def(py::init<>()).def_readwrite("num", &Field::num); + + py::classh(m, "Outer") + .def(py::init<>()) + + .def_readonly("m_valu_readonly", &Outer::m_valu) + .def_readwrite("m_valu_readwrite", &Outer::m_valu) + + .def_readonly("m_mptr_readonly", &Outer::m_mptr) + .def_readwrite("m_mptr_readwrite", &Outer::m_mptr) + .def_readonly("m_cptr_readonly", &Outer::m_cptr) + .def_readwrite("m_cptr_readwrite", &Outer::m_cptr) + + // .def_readonly("m_uqmp_readonly", &Outer::m_uqmp) // Custom compilation Error. + .def_readwrite("m_uqmp_readwrite", &Outer::m_uqmp) + // .def_readonly("m_uqcp_readonly", &Outer::m_uqcp) // Custom compilation Error. + .def_readwrite("m_uqcp_readwrite", &Outer::m_uqcp) + + .def_readwrite("m_shmp_readonly", &Outer::m_shmp) + .def_readwrite("m_shmp_readwrite", &Outer::m_shmp) + .def_readwrite("m_shcp_readonly", &Outer::m_shcp) + .def_readwrite("m_shcp_readwrite", &Outer::m_shcp); + + m.def("DisownOuter", DisownOuter); + + py::classh(m, "WithCharArrayMember") + .def(py::init<>()) + .def_readonly("char6_member", &WithCharArrayMember::char6_member); + + py::classh(m, "WithConstCharPtrMember") + .def(py::init<>()) + .def_readonly("const_char_ptr_member", &WithConstCharPtrMember::const_char_ptr_member); +} diff --git a/wrap/pybind11/tests/test_class_sh_property.py b/wrap/pybind11/tests/test_class_sh_property.py new file mode 100644 index 000000000..0250a7f78 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_property.py @@ -0,0 +1,166 @@ +# The compact 4-character naming scheme (e.g. mptr, cptr, shcp) is explained at the top of +# test_class_sh_property.cpp. +from __future__ import annotations + +import pytest + +import env # noqa: F401 +from pybind11_tests import class_sh_property as m + + +@pytest.mark.skipif( + "env.PYPY or env.GRAALPY", reason="gc after `del field` is apparently deferred" +) +@pytest.mark.parametrize("m_attr", ["m_valu_readonly", "m_valu_readwrite"]) +def test_valu_getter(m_attr): + # Reduced from PyCLIF test: + # https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/testing/python/nested_fields_test.py#L56 + outer = m.Outer() + field = getattr(outer, m_attr) + assert field.num == -99 + with pytest.raises(ValueError) as excinfo: + m.DisownOuter(outer) + assert str(excinfo.value) == "Cannot disown use_count != 1 (load_as_unique_ptr)." + del field + m.DisownOuter(outer) + with pytest.raises(ValueError, match="Python instance was disowned") as excinfo: + getattr(outer, m_attr) + + +def test_valu_setter(): + outer = m.Outer() + assert outer.m_valu_readonly.num == -99 + assert outer.m_valu_readwrite.num == -99 + field = m.Field() + field.num = 35 + outer.m_valu_readwrite = field + assert outer.m_valu_readonly.num == 35 + assert outer.m_valu_readwrite.num == 35 + + +@pytest.mark.parametrize("m_attr", ["m_shmp", "m_shcp"]) +def test_shp(m_attr): + m_attr_readonly = m_attr + "_readonly" + m_attr_readwrite = m_attr + "_readwrite" + outer = m.Outer() + assert getattr(outer, m_attr_readonly) is None + assert getattr(outer, m_attr_readwrite) is None + field = m.Field() + field.num = 43 + setattr(outer, m_attr_readwrite, field) + assert getattr(outer, m_attr_readonly).num == 43 + assert getattr(outer, m_attr_readwrite).num == 43 + getattr(outer, m_attr_readonly).num = 57 + getattr(outer, m_attr_readwrite).num = 57 + assert field.num == 57 + del field + assert getattr(outer, m_attr_readonly).num == 57 + assert getattr(outer, m_attr_readwrite).num == 57 + + +@pytest.mark.parametrize( + ("field_type", "num_default", "outer_type"), + [ + (m.ClassicField, -88, m.ClassicOuter), + (m.Field, -99, m.Outer), + ], +) +@pytest.mark.parametrize("m_attr", ["m_mptr", "m_cptr"]) +@pytest.mark.parametrize("r_kind", ["_readonly", "_readwrite"]) +def test_ptr(field_type, num_default, outer_type, m_attr, r_kind): + m_attr_r_kind = m_attr + r_kind + outer = outer_type() + assert getattr(outer, m_attr_r_kind) is None + field = field_type() + assert field.num == num_default + setattr(outer, m_attr + "_readwrite", field) + assert getattr(outer, m_attr_r_kind).num == num_default + field.num = 76 + assert getattr(outer, m_attr_r_kind).num == 76 + # Change to -88 or -99 to demonstrate Undefined Behavior (dangling pointer). + if num_default == 88 and m_attr == "m_mptr": + del field + assert getattr(outer, m_attr_r_kind).num == 76 + + +@pytest.mark.parametrize("m_attr_readwrite", ["m_uqmp_readwrite", "m_uqcp_readwrite"]) +def test_uqp(m_attr_readwrite): + outer = m.Outer() + assert getattr(outer, m_attr_readwrite) is None + field_orig = m.Field() + field_orig.num = 39 + setattr(outer, m_attr_readwrite, field_orig) + with pytest.raises(ValueError, match="Python instance was disowned"): + _ = field_orig.num + field_retr1 = getattr(outer, m_attr_readwrite) + assert getattr(outer, m_attr_readwrite) is None + assert field_retr1.num == 39 + field_retr1.num = 93 + setattr(outer, m_attr_readwrite, field_retr1) + with pytest.raises(ValueError): + _ = field_retr1.num + field_retr2 = getattr(outer, m_attr_readwrite) + assert field_retr2.num == 93 + + +# Proof-of-concept (POC) for safe & intuitive Python access to unique_ptr members. +# The C++ member unique_ptr is disowned to a temporary Python object for accessing +# an attribute of the member. After the attribute was accessed, the Python object +# is disowned back to the C++ member unique_ptr. +# Productizing this POC is left for a future separate PR, as needed. +class unique_ptr_field_proxy_poc: + def __init__(self, obj, field_name): + object.__setattr__(self, "__obj", obj) + object.__setattr__(self, "__field_name", field_name) + + def __getattr__(self, *args, **kwargs): + return _proxy_dereference(self, getattr, *args, **kwargs) + + def __setattr__(self, *args, **kwargs): + return _proxy_dereference(self, setattr, *args, **kwargs) + + def __delattr__(self, *args, **kwargs): + return _proxy_dereference(self, delattr, *args, **kwargs) + + +def _proxy_dereference(proxy, xxxattr, *args, **kwargs): + obj = object.__getattribute__(proxy, "__obj") + field_name = object.__getattribute__(proxy, "__field_name") + field = getattr(obj, field_name) # Disowns the C++ unique_ptr member. + assert field is not None + try: + return xxxattr(field, *args, **kwargs) + finally: + setattr(obj, field_name, field) # Disowns the temporary Python object (field). + + +@pytest.mark.parametrize("m_attr", ["m_uqmp", "m_uqcp"]) +def test_unique_ptr_field_proxy_poc(m_attr): + m_attr_readwrite = m_attr + "_readwrite" + outer = m.Outer() + field_orig = m.Field() + field_orig.num = 45 + setattr(outer, m_attr_readwrite, field_orig) + field_proxy = unique_ptr_field_proxy_poc(outer, m_attr_readwrite) + assert field_proxy.num == 45 + assert field_proxy.num == 45 + with pytest.raises(AttributeError): + _ = field_proxy.xyz + assert field_proxy.num == 45 + field_proxy.num = 82 + assert field_proxy.num == 82 + field_proxy = unique_ptr_field_proxy_poc(outer, m_attr_readwrite) + assert field_proxy.num == 82 + with pytest.raises(AttributeError): + del field_proxy.num + assert field_proxy.num == 82 + + +def test_readonly_char6_member(): + obj = m.WithCharArrayMember() + assert obj.char6_member == "Char6" + + +def test_readonly_const_char_ptr_member(): + obj = m.WithConstCharPtrMember() + assert obj.const_char_ptr_member == "ConstChar*" diff --git a/wrap/pybind11/tests/test_class_sh_property_non_owning.cpp b/wrap/pybind11/tests/test_class_sh_property_non_owning.cpp new file mode 100644 index 000000000..45fe7c7be --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_property_non_owning.cpp @@ -0,0 +1,63 @@ +#include "pybind11_tests.h" + +#include +#include + +namespace test_class_sh_property_non_owning { + +struct CoreField { + explicit CoreField(int int_value = -99) : int_value{int_value} {} + int int_value; +}; + +struct DataField { + DataField(int i_value, int i_shared, int i_unique) + : core_fld_value{i_value}, core_fld_shared_ptr{new CoreField{i_shared}}, + core_fld_raw_ptr{core_fld_shared_ptr.get()}, + core_fld_unique_ptr{new CoreField{i_unique}} {} + CoreField core_fld_value; + std::shared_ptr core_fld_shared_ptr; + CoreField *core_fld_raw_ptr; + std::unique_ptr core_fld_unique_ptr; +}; + +struct DataFieldsHolder { +private: + std::vector vec; + +public: + explicit DataFieldsHolder(std::size_t vec_size) { + for (std::size_t i = 0; i < vec_size; i++) { + int i11 = static_cast(i) * 11; + vec.emplace_back(13 + i11, 14 + i11, 15 + i11); + } + } + + DataField *vec_at(std::size_t index) { + if (index >= vec.size()) { + return nullptr; + } + return &vec[index]; + } +}; + +} // namespace test_class_sh_property_non_owning + +using namespace test_class_sh_property_non_owning; + +TEST_SUBMODULE(class_sh_property_non_owning, m) { + py::classh(m, "CoreField").def_readwrite("int_value", &CoreField::int_value); + + py::classh(m, "DataField") + .def_readonly("core_fld_value_ro", &DataField::core_fld_value) + .def_readwrite("core_fld_value_rw", &DataField::core_fld_value) + .def_readonly("core_fld_shared_ptr_ro", &DataField::core_fld_shared_ptr) + .def_readwrite("core_fld_shared_ptr_rw", &DataField::core_fld_shared_ptr) + .def_readonly("core_fld_raw_ptr_ro", &DataField::core_fld_raw_ptr) + .def_readwrite("core_fld_raw_ptr_rw", &DataField::core_fld_raw_ptr) + .def_readwrite("core_fld_unique_ptr_rw", &DataField::core_fld_unique_ptr); + + py::classh(m, "DataFieldsHolder") + .def(py::init()) + .def("vec_at", &DataFieldsHolder::vec_at, py::return_value_policy::reference_internal); +} diff --git a/wrap/pybind11/tests/test_class_sh_property_non_owning.py b/wrap/pybind11/tests/test_class_sh_property_non_owning.py new file mode 100644 index 000000000..33a9d4503 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_property_non_owning.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_property_non_owning as m + + +@pytest.mark.parametrize("persistent_holder", [True, False]) +@pytest.mark.parametrize( + ("core_fld", "expected"), + [ + ("core_fld_value_ro", (13, 24)), + ("core_fld_value_rw", (13, 24)), + ("core_fld_shared_ptr_ro", (14, 25)), + ("core_fld_shared_ptr_rw", (14, 25)), + ("core_fld_raw_ptr_ro", (14, 25)), + ("core_fld_raw_ptr_rw", (14, 25)), + ("core_fld_unique_ptr_rw", (15, 26)), + ], +) +def test_core_fld_common(core_fld, expected, persistent_holder): + if persistent_holder: + h = m.DataFieldsHolder(2) + for i, exp in enumerate(expected): + c = getattr(h.vec_at(i), core_fld) + assert c.int_value == exp + else: + for i, exp in enumerate(expected): + c = getattr(m.DataFieldsHolder(2).vec_at(i), core_fld) + assert c.int_value == exp diff --git a/wrap/pybind11/tests/test_class_sh_shared_ptr_copy_move.cpp b/wrap/pybind11/tests/test_class_sh_shared_ptr_copy_move.cpp new file mode 100644 index 000000000..889425a0b --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_shared_ptr_copy_move.cpp @@ -0,0 +1,103 @@ +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace { + +const std::string fooNames[] = {"ShPtr_", "SmHld_"}; + +template +struct Foo { + std::string history; + explicit Foo(const std::string &history_) : history(history_) {} + Foo(const Foo &other) : history(other.history + "_CpCtor") {} + Foo(Foo &&other) noexcept : history(other.history + "_MvCtor") {} + Foo &operator=(const Foo &other) { + history = other.history + "_OpEqLv"; + return *this; + } + Foo &operator=(Foo &&other) noexcept { + history = other.history + "_OpEqRv"; + return *this; + } + std::string get_history() const { return "Foo" + fooNames[SerNo] + history; } +}; + +using FooShPtr = Foo<0>; +using FooSmHld = Foo<1>; + +struct Outer { + std::shared_ptr ShPtr; + std::shared_ptr SmHld; + Outer() + : ShPtr(std::make_shared("Outer")), SmHld(std::make_shared("Outer")) {} + std::shared_ptr getShPtr() const { return ShPtr; } + std::shared_ptr getSmHld() const { return SmHld; } +}; + +} // namespace + +TEST_SUBMODULE(class_sh_shared_ptr_copy_move, m) { + namespace py = pybind11; + + py::class_>(m, "FooShPtr") + .def("get_history", &FooShPtr::get_history); + py::classh(m, "FooSmHld").def("get_history", &FooSmHld::get_history); + + auto outer = py::class_(m, "Outer").def(py::init()); +#define MAKE_PROP(PropTyp) \ + MAKE_PROP_FOO(ShPtr, PropTyp) \ + MAKE_PROP_FOO(SmHld, PropTyp) + +#define MAKE_PROP_FOO(FooTyp, PropTyp) \ + .def_##PropTyp(#FooTyp "_" #PropTyp "_default", &Outer::FooTyp) \ + .def_##PropTyp( \ + #FooTyp "_" #PropTyp "_copy", &Outer::FooTyp, py::return_value_policy::copy) \ + .def_##PropTyp( \ + #FooTyp "_" #PropTyp "_move", &Outer::FooTyp, py::return_value_policy::move) + outer MAKE_PROP(readonly) MAKE_PROP(readwrite); +#undef MAKE_PROP_FOO + +#define MAKE_PROP_FOO(FooTyp, PropTyp) \ + .def_##PropTyp(#FooTyp "_property_" #PropTyp "_default", &Outer::FooTyp) \ + .def_property_##PropTyp(#FooTyp "_property_" #PropTyp "_copy", \ + &Outer::get##FooTyp, \ + py::return_value_policy::copy) \ + .def_property_##PropTyp(#FooTyp "_property_" #PropTyp "_move", \ + &Outer::get##FooTyp, \ + py::return_value_policy::move) + outer MAKE_PROP(readonly); +#undef MAKE_PROP_FOO +#undef MAKE_PROP + + m.def("test_ShPtr_copy", []() { + auto o = std::make_shared("copy"); + auto l = py::list(); + l.append(o); + return l; + }); + m.def("test_SmHld_copy", []() { + auto o = std::make_shared("copy"); + auto l = py::list(); + l.append(o); + return l; + }); + + m.def("test_ShPtr_move", []() { + auto o = std::make_shared("move"); + auto l = py::list(); + l.append(std::move(o)); + return l; + }); + m.def("test_SmHld_move", []() { + auto o = std::make_shared("move"); + auto l = py::list(); + l.append(std::move(o)); + return l; + }); +} + +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_class_sh_shared_ptr_copy_move.py b/wrap/pybind11/tests/test_class_sh_shared_ptr_copy_move.py new file mode 100644 index 000000000..067bb47d2 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_shared_ptr_copy_move.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pybind11_tests import class_sh_shared_ptr_copy_move as m + + +def test_shptr_copy(): + txt = m.test_ShPtr_copy()[0].get_history() + assert txt == "FooShPtr_copy" + + +def test_smhld_copy(): + txt = m.test_SmHld_copy()[0].get_history() + assert txt == "FooSmHld_copy" + + +def test_shptr_move(): + txt = m.test_ShPtr_move()[0].get_history() + assert txt == "FooShPtr_move" + + +def test_smhld_move(): + txt = m.test_SmHld_move()[0].get_history() + assert txt == "FooSmHld_move" + + +def _check_property(foo_typ, prop_typ, policy): + o = m.Outer() + name = f"{foo_typ}_{prop_typ}_{policy}" + history = f"Foo{foo_typ}_Outer" + f = getattr(o, name) + assert f.get_history() == history + # and try again to check that o did not get changed + f = getattr(o, name) + assert f.get_history() == history + + +def test_properties(): + for prop_typ in ("readonly", "readwrite", "property_readonly"): + for foo_typ in ("ShPtr", "SmHld"): + for policy in ("default", "copy", "move"): + _check_property(foo_typ, prop_typ, policy) diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_basic.cpp b/wrap/pybind11/tests/test_class_sh_trampoline_basic.cpp new file mode 100644 index 000000000..0f42dcc71 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_basic.cpp @@ -0,0 +1,82 @@ +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_basic { + +template // Using int as a trick to easily generate a series of types. +struct Abase { + int val = 0; + virtual ~Abase() = default; + explicit Abase(int val_) : val{val_} {} + int Get() const { return val * 10 + 3; } + virtual int Add(int other_val) const = 0; + + // Some compilers complain about implicitly defined versions of some of the following: + Abase(const Abase &) = default; + Abase(Abase &&) noexcept = default; + Abase &operator=(const Abase &) = default; + Abase &operator=(Abase &&) noexcept = default; +}; + +template +struct AbaseAlias : Abase { + using Abase::Abase; + + int Add(int other_val) const override { + PYBIND11_OVERRIDE_PURE(int, /* Return type */ + Abase, /* Parent class */ + Add, /* Name of function in C++ (must match Python name) */ + other_val); + } +}; + +template <> +struct AbaseAlias<1> : Abase<1>, py::trampoline_self_life_support { + using Abase<1>::Abase; + + int Add(int other_val) const override { + PYBIND11_OVERRIDE_PURE(int, /* Return type */ + Abase<1>, /* Parent class */ + Add, /* Name of function in C++ (must match Python name) */ + other_val); + } +}; + +template +int AddInCppRawPtr(const Abase *obj, int other_val) { + return obj->Add(other_val) * 10 + 7; +} + +template +int AddInCppSharedPtr(std::shared_ptr> obj, int other_val) { + return obj->Add(other_val) * 100 + 11; +} + +template +int AddInCppUniquePtr(std::unique_ptr> obj, int other_val) { + return obj->Add(other_val) * 100 + 13; +} + +template +void wrap(py::module_ m, const char *py_class_name) { + py::classh, AbaseAlias>(m, py_class_name) + .def(py::init(), py::arg("val")) + .def("Get", &Abase::Get) + .def("Add", &Abase::Add, py::arg("other_val")); + + m.def("AddInCppRawPtr", AddInCppRawPtr, py::arg("obj"), py::arg("other_val")); + m.def("AddInCppSharedPtr", AddInCppSharedPtr, py::arg("obj"), py::arg("other_val")); + m.def("AddInCppUniquePtr", AddInCppUniquePtr, py::arg("obj"), py::arg("other_val")); +} + +} // namespace class_sh_trampoline_basic +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_basic; + +TEST_SUBMODULE(class_sh_trampoline_basic, m) { + wrap<0>(m, "Abase0"); + wrap<1>(m, "Abase1"); +} diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_basic.py b/wrap/pybind11/tests/test_class_sh_trampoline_basic.py new file mode 100644 index 000000000..eab82121f --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_basic.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_trampoline_basic as m + + +class PyDrvd0(m.Abase0): + def __init__(self, val): + super().__init__(val) + + def Add(self, other_val): + return self.Get() * 100 + other_val + + +class PyDrvd1(m.Abase1): + def __init__(self, val): + super().__init__(val) + + def Add(self, other_val): + return self.Get() * 200 + other_val + + +def test_drvd0_add(): + drvd = PyDrvd0(74) + assert drvd.Add(38) == (74 * 10 + 3) * 100 + 38 + + +def test_drvd0_add_in_cpp_raw_ptr(): + drvd = PyDrvd0(52) + assert m.AddInCppRawPtr(drvd, 27) == ((52 * 10 + 3) * 100 + 27) * 10 + 7 + + +def test_drvd0_add_in_cpp_shared_ptr(): + while True: + drvd = PyDrvd0(36) + assert m.AddInCppSharedPtr(drvd, 56) == ((36 * 10 + 3) * 100 + 56) * 100 + 11 + return # Comment out for manual leak checking (use `top` command). + + +def test_drvd0_add_in_cpp_unique_ptr(): + while True: + drvd = PyDrvd0(0) + with pytest.raises(ValueError) as exc_info: + m.AddInCppUniquePtr(drvd, 0) + assert ( + str(exc_info.value) + == "Alias class (also known as trampoline) does not inherit from" + " py::trampoline_self_life_support, therefore the ownership of this" + " instance cannot safely be transferred to C++." + ) + return # Comment out for manual leak checking (use `top` command). + + +def test_drvd1_add_in_cpp_unique_ptr(): + while True: + drvd = PyDrvd1(25) + assert m.AddInCppUniquePtr(drvd, 83) == ((25 * 10 + 3) * 200 + 83) * 100 + 13 + return # Comment out for manual leak checking (use `top` command). diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_self_life_support.cpp b/wrap/pybind11/tests/test_class_sh_trampoline_self_life_support.cpp new file mode 100644 index 000000000..22b728e28 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_self_life_support.cpp @@ -0,0 +1,86 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11/trampoline_self_life_support.h" +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_self_life_support { + +struct Big5 { // Also known as "rule of five". + std::string history; + + explicit Big5(std::string history_start) : history{std::move(history_start)} {} + + Big5(const Big5 &other) { history = other.history + "_CpCtor"; } + + Big5(Big5 &&other) noexcept { history = other.history + "_MvCtor"; } + + Big5 &operator=(const Big5 &other) { + history = other.history + "_OpEqLv"; + return *this; + } + + Big5 &operator=(Big5 &&other) noexcept { + history = other.history + "_OpEqRv"; + return *this; + } + + virtual ~Big5() = default; + +protected: + Big5() : history{"DefaultConstructor"} {} +}; + +struct Big5Trampoline : Big5, py::trampoline_self_life_support { + using Big5::Big5; +}; + +} // namespace class_sh_trampoline_self_life_support +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_self_life_support; + +TEST_SUBMODULE(class_sh_trampoline_self_life_support, m) { + py::classh(m, "Big5") + .def(py::init()) + .def_readonly("history", &Big5::history); + + m.def("action", [](std::unique_ptr obj, int action_id) { + py::object o2 = py::none(); + // This is very unusual, but needed to directly exercise the trampoline_self_life_support + // CpCtor, MvCtor, operator= lvalue, operator= rvalue. + auto *obj_trampoline = dynamic_cast(obj.get()); + if (obj_trampoline != nullptr) { + switch (action_id) { + case 0: { // CpCtor + std::unique_ptr cp(new Big5Trampoline(*obj_trampoline)); + o2 = py::cast(std::move(cp)); + } break; + case 1: { // MvCtor + std::unique_ptr mv(new Big5Trampoline(std::move(*obj_trampoline))); + o2 = py::cast(std::move(mv)); + } break; + case 2: { // operator= lvalue + std::unique_ptr lv(new Big5Trampoline); + *lv = *obj_trampoline; // NOLINT clang-tidy cppcoreguidelines-slicing + o2 = py::cast(std::move(lv)); + } break; + case 3: { // operator= rvalue + std::unique_ptr rv(new Big5Trampoline); + *rv = std::move(*obj_trampoline); + o2 = py::cast(std::move(rv)); + } break; + default: + break; + } + } + py::object o1 = py::cast(std::move(obj)); + return py::make_tuple(o1, o2); + }); +} diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_self_life_support.py b/wrap/pybind11/tests/test_class_sh_trampoline_self_life_support.py new file mode 100644 index 000000000..d4af2ab99 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_self_life_support.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import pytest + +import pybind11_tests.class_sh_trampoline_self_life_support as m + + +class PyBig5(m.Big5): + pass + + +def test_m_big5(): + obj = m.Big5("Seed") + assert obj.history == "Seed" + o1, o2 = m.action(obj, 0) + assert o1 is not obj + assert o1.history == "Seed" + with pytest.raises(ValueError) as excinfo: + _ = obj.history + assert "Python instance was disowned" in str(excinfo.value) + assert o2 is None + + +@pytest.mark.parametrize( + ("action_id", "expected_history"), + [ + (0, "Seed_CpCtor"), + (1, "Seed_MvCtor"), + (2, "Seed_OpEqLv"), + (3, "Seed_OpEqRv"), + ], +) +def test_py_big5(action_id, expected_history): + obj = PyBig5("Seed") + assert obj.history == "Seed" + o1, o2 = m.action(obj, action_id) + assert o1 is obj + assert o2.history == expected_history diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_shared_from_this.cpp b/wrap/pybind11/tests/test_class_sh_trampoline_shared_from_this.cpp new file mode 100644 index 000000000..dc6bf1c72 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_shared_from_this.cpp @@ -0,0 +1,137 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11_tests.h" + +#include +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_shared_from_this { + +struct Sft : std::enable_shared_from_this { + std::string history; + explicit Sft(const std::string &history_seed) : history{history_seed} {} + virtual ~Sft() = default; + +#if defined(__clang__) + // "Group of 4" begin. + // This group is not meant to be used, but will leave a trace in the + // history in case something goes wrong. + // However, compilers other than clang have a variety of issues. It is not + // worth the trouble covering all platforms. + Sft(const Sft &other) : enable_shared_from_this(other) { history = other.history + "_CpCtor"; } + + Sft(Sft &&other) noexcept { history = other.history + "_MvCtor"; } + + Sft &operator=(const Sft &other) { + history = other.history + "_OpEqLv"; + return *this; + } + + Sft &operator=(Sft &&other) noexcept { + history = other.history + "_OpEqRv"; + return *this; + } + // "Group of 4" end. +#endif +}; + +struct SftSharedPtrStash { + int ser_no; + std::vector> stash; + explicit SftSharedPtrStash(int ser_no) : ser_no{ser_no} {} + void Clear() { stash.clear(); } + void Add(const std::shared_ptr &obj) { + if (!obj->history.empty()) { + obj->history += "_Stash" + std::to_string(ser_no) + "Add"; + } + stash.push_back(obj); + } + void AddSharedFromThis(Sft *obj) { + auto sft = obj->shared_from_this(); + if (!sft->history.empty()) { + sft->history += "_Stash" + std::to_string(ser_no) + "AddSharedFromThis"; + } + stash.push_back(sft); + } + std::string history(unsigned i) { + if (i < stash.size()) { + return stash[i]->history; + } + return "OutOfRange"; + } + long use_count(unsigned i) { + if (i < stash.size()) { + return stash[i].use_count(); + } + return -1; + } +}; + +struct SftTrampoline : Sft, py::trampoline_self_life_support { + using Sft::Sft; +}; + +long use_count(const std::shared_ptr &obj) { return obj.use_count(); } + +long pass_shared_ptr(const std::shared_ptr &obj) { + auto sft = obj->shared_from_this(); + if (!sft->history.empty()) { + sft->history += "_PassSharedPtr"; + } + return sft.use_count(); +} + +std::string pass_unique_ptr_cref(const std::unique_ptr &obj) { + return obj ? obj->history : ""; +} +void pass_unique_ptr_rref(std::unique_ptr &&) { + throw std::runtime_error("Expected to not be reached."); +} + +Sft *make_pure_cpp_sft_raw_ptr(const std::string &history_seed) { return new Sft{history_seed}; } + +std::unique_ptr make_pure_cpp_sft_unq_ptr(const std::string &history_seed) { + return std::unique_ptr(new Sft{history_seed}); +} + +std::shared_ptr make_pure_cpp_sft_shd_ptr(const std::string &history_seed) { + return std::make_shared(history_seed); +} + +std::shared_ptr pass_through_shd_ptr(const std::shared_ptr &obj) { return obj; } + +} // namespace class_sh_trampoline_shared_from_this +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_shared_from_this; + +TEST_SUBMODULE(class_sh_trampoline_shared_from_this, m) { + py::classh(m, "Sft") + .def(py::init()) + .def(py::init([](const std::string &history, int) { + return std::make_shared(history); + })) + .def_readonly("history", &Sft::history) + // This leads to multiple entries in registered_instances: + .def(py::init([](const std::shared_ptr &existing) { return existing; })); + + py::classh(m, "SftSharedPtrStash") + .def(py::init()) + .def("Clear", &SftSharedPtrStash::Clear) + .def("Add", &SftSharedPtrStash::Add) + .def("AddSharedFromThis", &SftSharedPtrStash::AddSharedFromThis) + .def("history", &SftSharedPtrStash::history) + .def("use_count", &SftSharedPtrStash::use_count); + + m.def("use_count", use_count); + m.def("pass_shared_ptr", pass_shared_ptr); + m.def("pass_unique_ptr_cref", pass_unique_ptr_cref); + m.def("pass_unique_ptr_rref", pass_unique_ptr_rref); + m.def("make_pure_cpp_sft_raw_ptr", make_pure_cpp_sft_raw_ptr); + m.def("make_pure_cpp_sft_unq_ptr", make_pure_cpp_sft_unq_ptr); + m.def("make_pure_cpp_sft_shd_ptr", make_pure_cpp_sft_shd_ptr); + m.def("pass_through_shd_ptr", pass_through_shd_ptr); +} diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_shared_from_this.py b/wrap/pybind11/tests/test_class_sh_trampoline_shared_from_this.py new file mode 100644 index 000000000..fbe31387a --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_shared_from_this.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import sys +import weakref + +import pytest + +import env +import pybind11_tests.class_sh_trampoline_shared_from_this as m + + +class PySft(m.Sft): + pass + + +def test_release_and_shared_from_this(): + # Exercises the most direct path from building a shared_from_this-visible + # shared_ptr to calling shared_from_this. + obj = PySft("PySft") + assert obj.history == "PySft" + assert m.use_count(obj) == 1 + assert m.pass_shared_ptr(obj) == 2 + assert obj.history == "PySft_PassSharedPtr" + assert m.use_count(obj) == 1 + assert m.pass_shared_ptr(obj) == 2 + assert obj.history == "PySft_PassSharedPtr_PassSharedPtr" + assert m.use_count(obj) == 1 + + +def test_release_and_shared_from_this_leak(): + obj = PySft("") + while True: + m.pass_shared_ptr(obj) + assert not obj.history + assert m.use_count(obj) == 1 + break # Comment out for manual leak checking (use `top` command). + + +def test_release_and_stash(): + # Exercises correct functioning of guarded_delete weak_ptr. + obj = PySft("PySft") + stash1 = m.SftSharedPtrStash(1) + stash1.Add(obj) + exp_hist = "PySft_Stash1Add" + assert obj.history == exp_hist + assert m.use_count(obj) == 2 + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 1 + assert m.pass_shared_ptr(obj) == 3 + exp_hist += "_PassSharedPtr" + assert obj.history == exp_hist + assert m.use_count(obj) == 2 + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 1 + stash2 = m.SftSharedPtrStash(2) + stash2.Add(obj) + exp_hist += "_Stash2Add" + assert obj.history == exp_hist + assert m.use_count(obj) == 3 + assert stash2.history(0) == exp_hist + assert stash2.use_count(0) == 2 + stash2.Add(obj) + exp_hist += "_Stash2Add" + assert obj.history == exp_hist + assert m.use_count(obj) == 4 + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 3 + assert stash2.history(0) == exp_hist + assert stash2.use_count(0) == 3 + assert stash2.history(1) == exp_hist + assert stash2.use_count(1) == 3 + del obj + assert stash2.history(0) == exp_hist + assert stash2.use_count(0) == 3 + assert stash2.history(1) == exp_hist + assert stash2.use_count(1) == 3 + stash2.Clear() + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 1 + + +def test_release_and_stash_leak(): + obj = PySft("") + while True: + stash1 = m.SftSharedPtrStash(1) + stash1.Add(obj) + assert not obj.history + assert m.use_count(obj) == 2 + assert stash1.use_count(0) == 1 + stash1.Add(obj) + assert not obj.history + assert m.use_count(obj) == 3 + assert stash1.use_count(0) == 2 + assert stash1.use_count(1) == 2 + break # Comment out for manual leak checking (use `top` command). + + +def test_release_and_stash_via_shared_from_this(): + # Exercises that the smart_holder vptr is invisible to the shared_from_this mechanism. + obj = PySft("PySft") + stash1 = m.SftSharedPtrStash(1) + with pytest.raises(RuntimeError) as exc_info: + stash1.AddSharedFromThis(obj) + assert str(exc_info.value) == "bad_weak_ptr" + stash1.Add(obj) + assert obj.history == "PySft_Stash1Add" + assert stash1.use_count(0) == 1 + stash1.AddSharedFromThis(obj) + assert obj.history == "PySft_Stash1Add_Stash1AddSharedFromThis" + assert stash1.use_count(0) == 2 + assert stash1.use_count(1) == 2 + + +def test_release_and_stash_via_shared_from_this_leak(): + obj = PySft("") + while True: + stash1 = m.SftSharedPtrStash(1) + with pytest.raises(RuntimeError) as exc_info: + stash1.AddSharedFromThis(obj) + assert str(exc_info.value) == "bad_weak_ptr" + stash1.Add(obj) + assert not obj.history + assert stash1.use_count(0) == 1 + stash1.AddSharedFromThis(obj) + assert not obj.history + assert stash1.use_count(0) == 2 + assert stash1.use_count(1) == 2 + break # Comment out for manual leak checking (use `top` command). + + +def test_pass_released_shared_ptr_as_unique_ptr(): + # Exercises that returning a unique_ptr fails while a shared_from_this + # visible shared_ptr exists. + obj = PySft("PySft") + stash1 = m.SftSharedPtrStash(1) + stash1.Add(obj) # Releases shared_ptr to C++. + assert m.pass_unique_ptr_cref(obj) == "PySft_Stash1Add" + assert obj.history == "PySft_Stash1Add" + with pytest.raises(ValueError) as exc_info: + m.pass_unique_ptr_rref(obj) + assert str(exc_info.value) == ( + "Python instance is currently owned by a std::shared_ptr." + ) + assert obj.history == "PySft_Stash1Add" + + +@pytest.mark.parametrize( + "make_f", + [ + m.make_pure_cpp_sft_raw_ptr, + m.make_pure_cpp_sft_unq_ptr, + m.make_pure_cpp_sft_shd_ptr, + ], +) +def test_pure_cpp_sft_raw_ptr(make_f): + # Exercises void_cast_raw_ptr logic for different situations. + obj = make_f("PureCppSft") + assert m.pass_shared_ptr(obj) == 3 + assert obj.history == "PureCppSft_PassSharedPtr" + obj = make_f("PureCppSft") + stash1 = m.SftSharedPtrStash(1) + stash1.AddSharedFromThis(obj) + assert obj.history == "PureCppSft_Stash1AddSharedFromThis" + + +def test_multiple_registered_instances_for_same_pointee(): + obj0 = PySft("PySft") + obj0.attachment_in_dict = "Obj0" + assert m.pass_through_shd_ptr(obj0) is obj0 + while True: + obj = m.Sft(obj0) + assert obj is not obj0 + obj_pt = m.pass_through_shd_ptr(obj) + # Unpredictable! Because registered_instances is as std::unordered_multimap. + assert obj_pt is obj0 or obj_pt is obj + # Multiple registered_instances for the same pointee can lead to unpredictable results: + if obj_pt is obj0: + assert obj_pt.attachment_in_dict == "Obj0" + else: + assert not hasattr(obj_pt, "attachment_in_dict") + assert obj0.history == "PySft" + break # Comment out for manual leak checking (use `top` command). + + +def test_multiple_registered_instances_for_same_pointee_leak(): + obj0 = PySft("") + while True: + stash1 = m.SftSharedPtrStash(1) + stash1.Add(m.Sft(obj0)) + assert stash1.use_count(0) == 1 + stash1.Add(m.Sft(obj0)) + assert stash1.use_count(0) == 1 + assert stash1.use_count(1) == 1 + assert not obj0.history + break # Comment out for manual leak checking (use `top` command). + + +def test_multiple_registered_instances_for_same_pointee_recursive(): + while True: + obj0 = PySft("PySft") + if not env.PYPY: + obj0_wr = weakref.ref(obj0) + obj = obj0 + # This loop creates a chain of instances linked by shared_ptrs. + for _ in range(10): + obj_next = m.Sft(obj) + assert obj_next is not obj + obj = obj_next + del obj_next + assert obj.history == "PySft" + del obj0 + if not env.PYPY and not env.GRAALPY: + assert obj0_wr() is not None + del obj # This releases the chain recursively. + if not env.PYPY and not env.GRAALPY: + assert obj0_wr() is None + break # Comment out for manual leak checking (use `top` command). + + +# As of 2021-07-10 the pybind11 GitHub Actions valgrind build uses Python 3.9. +WORKAROUND_ENABLING_ROLLBACK_OF_PR3068 = env.LINUX and sys.version_info == (3, 9) + + +def test_std_make_shared_factory(): + class PySftMakeShared(m.Sft): + def __init__(self, history): + super().__init__(history, 0) + + obj = PySftMakeShared("PySftMakeShared") + assert obj.history == "PySftMakeShared" + if WORKAROUND_ENABLING_ROLLBACK_OF_PR3068: + try: + m.pass_through_shd_ptr(obj) + except RuntimeError as e: + str_exc_info_value = str(e) + else: + str_exc_info_value = "RuntimeError NOT RAISED" + else: + with pytest.raises(RuntimeError) as exc_info: + m.pass_through_shd_ptr(obj) + str_exc_info_value = str(exc_info.value) + assert ( + str_exc_info_value + == "smart_holder_type_casters load_as_shared_ptr failure: not implemented:" + " trampoline-self-life-support for external shared_ptr to type inheriting" + " from std::enable_shared_from_this." + ) diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp b/wrap/pybind11/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp new file mode 100644 index 000000000..49e1ac885 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_shared_ptr_cpp_arg { + +// For testing whether a python subclass of a C++ object dies when the +// last python reference is lost +struct SpBase { + // returns true if the base virtual function is called + virtual bool is_base_used() { return true; } + + // returns true if there's an associated python instance + bool has_python_instance() { + auto *tinfo = py::detail::get_type_info(typeid(SpBase)); + return (bool) py::detail::get_object_handle(this, tinfo); + } + + SpBase() = default; + SpBase(const SpBase &) = delete; + virtual ~SpBase() = default; +}; + +std::shared_ptr pass_through_shd_ptr(const std::shared_ptr &obj) { return obj; } + +struct PySpBase : SpBase { + using SpBase::SpBase; + bool is_base_used() override { PYBIND11_OVERRIDE(bool, SpBase, is_base_used); } +}; + +struct SpBaseTester { + std::shared_ptr get_object() const { return m_obj; } + void set_object(std::shared_ptr obj) { m_obj = std::move(obj); } + bool is_base_used() { return m_obj->is_base_used(); } + bool has_instance() { return (bool) m_obj; } + bool has_python_instance() { return m_obj && m_obj->has_python_instance(); } + void set_nonpython_instance() { m_obj = std::make_shared(); } + std::shared_ptr m_obj; +}; + +// For testing that a C++ class without an alias does not retain the python +// portion of the object +struct SpGoAway {}; + +struct SpGoAwayTester { + std::shared_ptr m_obj; +}; + +} // namespace class_sh_trampoline_shared_ptr_cpp_arg +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_shared_ptr_cpp_arg; + +TEST_SUBMODULE(class_sh_trampoline_shared_ptr_cpp_arg, m) { + // For testing whether a python subclass of a C++ object dies when the + // last python reference is lost + + py::classh(m, "SpBase") + .def(py::init<>()) + .def(py::init([](int) { return std::make_shared(); })) + .def("is_base_used", &SpBase::is_base_used) + .def("has_python_instance", &SpBase::has_python_instance); + + m.def("pass_through_shd_ptr", pass_through_shd_ptr); + m.def("pass_through_shd_ptr_release_gil", + pass_through_shd_ptr, + py::call_guard()); // PR #4196 + + py::classh(m, "SpBaseTester") + .def(py::init<>()) + .def("get_object", &SpBaseTester::get_object) + .def("set_object", &SpBaseTester::set_object) + .def("is_base_used", &SpBaseTester::is_base_used) + .def("has_instance", &SpBaseTester::has_instance) + .def("has_python_instance", &SpBaseTester::has_python_instance) + .def("set_nonpython_instance", &SpBaseTester::set_nonpython_instance) + .def_readwrite("obj", &SpBaseTester::m_obj); + + // For testing that a C++ class without an alias does not retain the python + // portion of the object + + py::classh(m, "SpGoAway").def(py::init<>()); + + py::classh(m, "SpGoAwayTester") + .def(py::init<>()) + .def_readwrite("obj", &SpGoAwayTester::m_obj); +} diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.py b/wrap/pybind11/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.py new file mode 100644 index 000000000..a693621e3 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import pytest + +import env # noqa: F401 +import pybind11_tests.class_sh_trampoline_shared_ptr_cpp_arg as m + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_cpp_arg(): + import weakref + + class PyChild(m.SpBase): + def is_base_used(self): + return False + + tester = m.SpBaseTester() + + obj = PyChild() + objref = weakref.ref(obj) + + # Pass the last python reference to the C++ function + tester.set_object(obj) + del obj + pytest.gc_collect() + + # python reference is still around since C++ has it now + assert objref() is not None + assert tester.is_base_used() is False + assert tester.obj.is_base_used() is False + assert tester.get_object() is objref() + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_cpp_prop(): + class PyChild(m.SpBase): + def is_base_used(self): + return False + + tester = m.SpBaseTester() + + # Set the last python reference as a property of the C++ object + tester.obj = PyChild() + pytest.gc_collect() + + # python reference is still around since C++ has it now + assert tester.is_base_used() is False + assert tester.has_python_instance() is True + assert tester.obj.is_base_used() is False + assert tester.obj.has_python_instance() is True + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_arg_identity(): + import weakref + + tester = m.SpBaseTester() + + obj = m.SpBase() + objref = weakref.ref(obj) + + tester.set_object(obj) + del obj + pytest.gc_collect() + + # NOTE: the behavior below is DIFFERENT from PR #2839 + # python reference is gone because it is not an Alias instance + assert objref() is None + assert tester.has_python_instance() is False + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_alias_nonpython(): + tester = m.SpBaseTester() + + # C++ creates the object, a python instance shouldn't exist + tester.set_nonpython_instance() + assert tester.is_base_used() is True + assert tester.has_instance() is True + assert tester.has_python_instance() is False + + # Now a python instance exists + cobj = tester.get_object() + assert cobj.has_python_instance() + assert tester.has_instance() is True + assert tester.has_python_instance() is True + + # Now it's gone + del cobj + pytest.gc_collect() + assert tester.has_instance() is True + assert tester.has_python_instance() is False + + # When we pass it as an arg to a new tester the python instance should + # disappear because it wasn't created with an alias + new_tester = m.SpBaseTester() + + cobj = tester.get_object() + assert cobj.has_python_instance() + + new_tester.set_object(cobj) + assert tester.has_python_instance() is True + assert new_tester.has_python_instance() is True + + del cobj + pytest.gc_collect() + + # Gone! + assert tester.has_instance() is True + assert tester.has_python_instance() is False + assert new_tester.has_instance() is True + assert new_tester.has_python_instance() is False + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_goaway(): + import weakref + + tester = m.SpGoAwayTester() + + obj = m.SpGoAway() + objref = weakref.ref(obj) + + assert tester.obj is None + + tester.obj = obj + del obj + pytest.gc_collect() + + # python reference is no longer around + assert objref() is None + # C++ reference is still around + assert tester.obj is not None + + +def test_infinite(): + tester = m.SpBaseTester() + while True: + tester.set_object(m.SpBase()) + break # Comment out for manual leak checking (use `top` command). + + +@pytest.mark.parametrize( + "pass_through_func", [m.pass_through_shd_ptr, m.pass_through_shd_ptr_release_gil] +) +def test_std_make_shared_factory(pass_through_func): + class PyChild(m.SpBase): + def __init__(self): + super().__init__(0) + + obj = PyChild() + while True: + assert pass_through_func(obj) is obj + break # Comment out for manual leak checking (use `top` command). diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_unique_ptr.cpp b/wrap/pybind11/tests/test_class_sh_trampoline_unique_ptr.cpp new file mode 100644 index 000000000..debe3324e --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_unique_ptr.cpp @@ -0,0 +1,63 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11/trampoline_self_life_support.h" +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_unique_ptr { + +class Class { +public: + virtual ~Class() = default; + + void setVal(std::uint64_t val) { val_ = val; } + std::uint64_t getVal() const { return val_; } + + virtual std::unique_ptr clone() const = 0; + virtual int foo() const = 0; + +protected: + Class() = default; + + // Some compilers complain about implicitly defined versions of some of the following: + Class(const Class &) = default; + +private: + std::uint64_t val_ = 0; +}; + +} // namespace class_sh_trampoline_unique_ptr +} // namespace pybind11_tests + +namespace pybind11_tests { +namespace class_sh_trampoline_unique_ptr { + +class PyClass : public Class, public py::trampoline_self_life_support { +public: + std::unique_ptr clone() const override { + PYBIND11_OVERRIDE_PURE(std::unique_ptr, Class, clone); + } + + int foo() const override { PYBIND11_OVERRIDE_PURE(int, Class, foo); } +}; + +} // namespace class_sh_trampoline_unique_ptr +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_trampoline_unique_ptr, m) { + using namespace pybind11_tests::class_sh_trampoline_unique_ptr; + + py::classh(m, "Class") + .def(py::init<>()) + .def("set_val", &Class::setVal) + .def("get_val", &Class::getVal) + .def("clone", &Class::clone) + .def("foo", &Class::foo); + + m.def("clone", [](const Class &obj) { return obj.clone(); }); + m.def("clone_and_foo", [](const Class &obj) { return obj.clone()->foo(); }); +} diff --git a/wrap/pybind11/tests/test_class_sh_trampoline_unique_ptr.py b/wrap/pybind11/tests/test_class_sh_trampoline_unique_ptr.py new file mode 100644 index 000000000..7799df6d6 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_trampoline_unique_ptr.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pybind11_tests.class_sh_trampoline_unique_ptr as m + + +class MyClass(m.Class): + def foo(self): + return 10 + self.get_val() + + def clone(self): + cloned = MyClass() + cloned.set_val(self.get_val() + 3) + return cloned + + +def test_m_clone(): + obj = MyClass() + while True: + obj.set_val(5) + obj = m.clone(obj) + assert obj.get_val() == 5 + 3 + assert obj.foo() == 10 + 5 + 3 + return # Comment out for manual leak checking (use `top` command). + + +def test_m_clone_and_foo(): + obj = MyClass() + obj.set_val(7) + while True: + assert m.clone_and_foo(obj) == 10 + 7 + 3 + return # Comment out for manual leak checking (use `top` command). diff --git a/wrap/pybind11/tests/test_class_sh_unique_ptr_custom_deleter.cpp b/wrap/pybind11/tests/test_class_sh_unique_ptr_custom_deleter.cpp new file mode 100644 index 000000000..adaa2e47d --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_unique_ptr_custom_deleter.cpp @@ -0,0 +1,30 @@ +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_unique_ptr_custom_deleter { + +// Reduced from a PyCLIF use case in the wild by @wangxf123456. +class Pet { +public: + using Ptr = std::unique_ptr>; + + std::string name; + + static Ptr New(const std::string &name) { + return Ptr(new Pet(name), std::default_delete()); + } + +private: + explicit Pet(const std::string &name) : name(name) {} +}; + +TEST_SUBMODULE(class_sh_unique_ptr_custom_deleter, m) { + py::classh(m, "Pet").def_readwrite("name", &Pet::name); + + m.def("create", &Pet::New); +} + +} // namespace class_sh_unique_ptr_custom_deleter +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_class_sh_unique_ptr_custom_deleter.py b/wrap/pybind11/tests/test_class_sh_unique_ptr_custom_deleter.py new file mode 100644 index 000000000..34aa52068 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_unique_ptr_custom_deleter.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from pybind11_tests import class_sh_unique_ptr_custom_deleter as m + + +def test_create(): + pet = m.create("abc") + assert pet.name == "abc" diff --git a/wrap/pybind11/tests/test_class_sh_unique_ptr_member.cpp b/wrap/pybind11/tests/test_class_sh_unique_ptr_member.cpp new file mode 100644 index 000000000..50c505a65 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_unique_ptr_member.cpp @@ -0,0 +1,50 @@ +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_unique_ptr_member { + +class pointee { // NOT copyable. +public: + pointee() = default; + + int get_int() const { return 213; } + + pointee(const pointee &) = delete; + pointee(pointee &&) = delete; + pointee &operator=(const pointee &) = delete; + pointee &operator=(pointee &&) = delete; +}; + +inline std::unique_ptr make_unique_pointee() { + return std::unique_ptr(new pointee); +} + +class ptr_owner { +public: + explicit ptr_owner(std::unique_ptr ptr) : ptr_(std::move(ptr)) {} + + bool is_owner() const { return bool(ptr_); } + + std::unique_ptr give_up_ownership_via_unique_ptr() { return std::move(ptr_); } + std::shared_ptr give_up_ownership_via_shared_ptr() { return std::move(ptr_); } + +private: + std::unique_ptr ptr_; +}; + +TEST_SUBMODULE(class_sh_unique_ptr_member, m) { + py::classh(m, "pointee").def(py::init<>()).def("get_int", &pointee::get_int); + + m.def("make_unique_pointee", make_unique_pointee); + + py::class_(m, "ptr_owner") + .def(py::init>(), py::arg("ptr")) + .def("is_owner", &ptr_owner::is_owner) + .def("give_up_ownership_via_unique_ptr", &ptr_owner::give_up_ownership_via_unique_ptr) + .def("give_up_ownership_via_shared_ptr", &ptr_owner::give_up_ownership_via_shared_ptr); +} + +} // namespace class_sh_unique_ptr_member +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_class_sh_unique_ptr_member.py b/wrap/pybind11/tests/test_class_sh_unique_ptr_member.py new file mode 100644 index 000000000..a5d2ccd23 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_unique_ptr_member.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_unique_ptr_member as m + + +def test_make_unique_pointee(): + obj = m.make_unique_pointee() + assert obj.get_int() == 213 + + +@pytest.mark.parametrize( + "give_up_ownership_via", + ["give_up_ownership_via_unique_ptr", "give_up_ownership_via_shared_ptr"], +) +def test_pointee_and_ptr_owner(give_up_ownership_via): + obj = m.pointee() + assert obj.get_int() == 213 + owner = m.ptr_owner(obj) + with pytest.raises(ValueError, match="Python instance was disowned"): + obj.get_int() + assert owner.is_owner() + reclaimed = getattr(owner, give_up_ownership_via)() + assert not owner.is_owner() + assert reclaimed.get_int() == 213 diff --git a/wrap/pybind11/tests/test_class_sh_virtual_py_cpp_mix.cpp b/wrap/pybind11/tests/test_class_sh_virtual_py_cpp_mix.cpp new file mode 100644 index 000000000..df8af19e4 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_virtual_py_cpp_mix.cpp @@ -0,0 +1,58 @@ +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_virtual_py_cpp_mix { + +class Base { +public: + virtual ~Base() = default; + virtual int get() const { return 101; } + + // Some compilers complain about implicitly defined versions of some of the following: + Base() = default; + Base(const Base &) = default; +}; + +class CppDerivedPlain : public Base { +public: + int get() const override { return 202; } +}; + +class CppDerived : public Base { +public: + int get() const override { return 212; } +}; + +int get_from_cpp_plainc_ptr(const Base *b) { return b->get() + 4000; } + +int get_from_cpp_unique_ptr(std::unique_ptr b) { return b->get() + 5000; } + +struct BaseVirtualOverrider : Base, py::trampoline_self_life_support { + using Base::Base; + + int get() const override { PYBIND11_OVERRIDE(int, Base, get); } +}; + +struct CppDerivedVirtualOverrider : CppDerived, py::trampoline_self_life_support { + using CppDerived::CppDerived; + + int get() const override { PYBIND11_OVERRIDE(int, CppDerived, get); } +}; + +} // namespace class_sh_virtual_py_cpp_mix +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_virtual_py_cpp_mix; + +TEST_SUBMODULE(class_sh_virtual_py_cpp_mix, m) { + py::classh(m, "Base").def(py::init<>()).def("get", &Base::get); + + py::classh(m, "CppDerivedPlain").def(py::init<>()); + + py::classh(m, "CppDerived").def(py::init<>()); + + m.def("get_from_cpp_plainc_ptr", get_from_cpp_plainc_ptr, py::arg("b")); + m.def("get_from_cpp_unique_ptr", get_from_cpp_unique_ptr, py::arg("b")); +} diff --git a/wrap/pybind11/tests/test_class_sh_virtual_py_cpp_mix.py b/wrap/pybind11/tests/test_class_sh_virtual_py_cpp_mix.py new file mode 100644 index 000000000..33133eb88 --- /dev/null +++ b/wrap/pybind11/tests/test_class_sh_virtual_py_cpp_mix.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_virtual_py_cpp_mix as m + + +class PyBase(m.Base): # Avoiding name PyDerived, for more systematic naming. + def __init__(self): + m.Base.__init__(self) + + def get(self): + return 323 + + +class PyCppDerived(m.CppDerived): + def __init__(self): + m.CppDerived.__init__(self) + + def get(self): + return 434 + + +@pytest.mark.parametrize( + ("ctor", "expected"), + [ + (m.Base, 101), + (PyBase, 323), + (m.CppDerivedPlain, 202), + (m.CppDerived, 212), + (PyCppDerived, 434), + ], +) +def test_base_get(ctor, expected): + obj = ctor() + assert obj.get() == expected + + +@pytest.mark.parametrize( + ("ctor", "expected"), + [ + (m.Base, 4101), + (PyBase, 4323), + (m.CppDerivedPlain, 4202), + (m.CppDerived, 4212), + (PyCppDerived, 4434), + ], +) +def test_get_from_cpp_plainc_ptr(ctor, expected): + obj = ctor() + assert m.get_from_cpp_plainc_ptr(obj) == expected + + +@pytest.mark.parametrize( + ("ctor", "expected"), + [ + (m.Base, 5101), + (PyBase, 5323), + (m.CppDerivedPlain, 5202), + (m.CppDerived, 5212), + (PyCppDerived, 5434), + ], +) +def test_get_from_cpp_unique_ptr(ctor, expected): + obj = ctor() + assert m.get_from_cpp_unique_ptr(obj) == expected diff --git a/wrap/pybind11/tests/test_cmake_build/CMakeLists.txt b/wrap/pybind11/tests/test_cmake_build/CMakeLists.txt index f28bde08e..ec365c0f6 100644 --- a/wrap/pybind11/tests/test_cmake_build/CMakeLists.txt +++ b/wrap/pybind11/tests/test_cmake_build/CMakeLists.txt @@ -55,8 +55,10 @@ possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID) pybind11_add_build_test(subdirectory_function) pybind11_add_build_test(subdirectory_target) -if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy") - message(STATUS "Skipping embed test on PyPy") +if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" + OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") + message(STATUS "Skipping embed test on PyPy or GraalPy") else() pybind11_add_build_test(subdirectory_embed) endif() @@ -66,10 +68,14 @@ if(PYBIND11_INSTALL) mock_install ${CMAKE_COMMAND} "-DCMAKE_INSTALL_PREFIX=${pybind11_BINARY_DIR}/mock_install" -P "${pybind11_BINARY_DIR}/cmake_install.cmake") - pybind11_add_build_test(installed_function INSTALL) + if(NOT "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") + pybind11_add_build_test(installed_function INSTALL) + endif() pybind11_add_build_test(installed_target INSTALL) - if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" - )) + if(NOT + ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" + OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy")) pybind11_add_build_test(installed_embed INSTALL) endif() endif() diff --git a/wrap/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt b/wrap/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt index 2be0aa659..5c6267c72 100644 --- a/wrap/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/wrap/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,13 +1,4 @@ -cmake_minimum_required(VERSION 3.5) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) project(test_installed_embed CXX) diff --git a/wrap/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt b/wrap/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt index fa7795e1e..2945b3d2e 100644 --- a/wrap/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/wrap/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,14 +1,4 @@ -cmake_minimum_required(VERSION 3.5) -project(test_installed_module CXX) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) project(test_installed_function CXX) diff --git a/wrap/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt b/wrap/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt index 7e73f4243..344c8bc69 100644 --- a/wrap/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/wrap/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,13 +1,4 @@ -cmake_minimum_required(VERSION 3.5) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) project(test_installed_target CXX) diff --git a/wrap/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/wrap/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index 33a366472..d0ae0798e 100644 --- a/wrap/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/wrap/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,13 +1,4 @@ -cmake_minimum_required(VERSION 3.5) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) project(test_subdirectory_embed CXX) diff --git a/wrap/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/wrap/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index 76418a71f..a521f33df 100644 --- a/wrap/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/wrap/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,13 +1,4 @@ -cmake_minimum_required(VERSION 3.5) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) project(test_subdirectory_function CXX) diff --git a/wrap/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/wrap/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index 28e903187..4a5a7f2be 100644 --- a/wrap/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/wrap/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,13 +1,4 @@ -cmake_minimum_required(VERSION 3.5) - -# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.29) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.29) -endif() +cmake_minimum_required(VERSION 3.15...3.30) project(test_subdirectory_target CXX) diff --git a/wrap/pybind11/tests/test_copy_move.py b/wrap/pybind11/tests/test_copy_move.py index ee046305f..3a3f29341 100644 --- a/wrap/pybind11/tests/test_copy_move.py +++ b/wrap/pybind11/tests/test_copy_move.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest +import env # noqa: F401 from pybind11_tests import copy_move_policies as m @@ -17,6 +18,7 @@ def test_lacking_move_ctor(): assert "is neither movable nor copyable!" in str(excinfo.value) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_and_copy_casts(): """Cast some values in C++ via custom type casters and count the number of moves/copies.""" @@ -44,6 +46,7 @@ def test_move_and_copy_casts(): assert c_m.alive() + c_mc.alive() + c_c.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_and_copy_loads(): """Call some functions that load arguments via custom type casters and count the number of moves/copies.""" @@ -77,6 +80,7 @@ def test_move_and_copy_loads(): @pytest.mark.skipif(not m.has_optional, reason="no ") +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_and_copy_load_optional(): """Tests move/copy loads of std::optional arguments""" diff --git a/wrap/pybind11/tests/test_cpp_conduit.cpp b/wrap/pybind11/tests/test_cpp_conduit.cpp new file mode 100644 index 000000000..4ee4f0690 --- /dev/null +++ b/wrap/pybind11/tests/test_cpp_conduit.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2024 The pybind Community. + +#include "pybind11_tests.h" +#include "test_cpp_conduit_traveler_bindings.h" + +#include + +namespace pybind11_tests { +namespace test_cpp_conduit { + +TEST_SUBMODULE(cpp_conduit, m) { + m.attr("PYBIND11_PLATFORM_ABI_ID") = py::bytes(PYBIND11_PLATFORM_ABI_ID); + m.attr("cpp_type_info_capsule_Traveler") + = py::capsule(&typeid(Traveler), typeid(std::type_info).name()); + m.attr("cpp_type_info_capsule_int") = py::capsule(&typeid(int), typeid(std::type_info).name()); + + wrap_traveler(m); + wrap_lonely_traveler(m); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_cpp_conduit.py b/wrap/pybind11/tests/test_cpp_conduit.py new file mode 100644 index 000000000..eb300587f --- /dev/null +++ b/wrap/pybind11/tests/test_cpp_conduit.py @@ -0,0 +1,166 @@ +# Copyright (c) 2024 The pybind Community. + +from __future__ import annotations + +import exo_planet_c_api +import exo_planet_pybind11 +import home_planet_very_lonely_traveler +import pytest + +import env +from pybind11_tests import cpp_conduit as home_planet + + +def test_traveler_getattr_actually_exists(): + t_h = home_planet.Traveler("home") + assert t_h.any_name == "Traveler GetAttr: any_name luggage: home" + + +def test_premium_traveler_getattr_actually_exists(): + t_h = home_planet.PremiumTraveler("home", 7) + assert t_h.secret_name == "PremiumTraveler GetAttr: secret_name points: 7" + + +def test_call_cpp_conduit_success(): + t_h = home_planet.Traveler("home") + cap = t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID, + home_planet.cpp_type_info_capsule_Traveler, + b"raw_pointer_ephemeral", + ) + assert cap.__class__.__name__ == "PyCapsule" or ( + # Note: this will become unnecessary in the next GraalPy release + env.GRAALPY and cap.__class__.__name__ == "capsule" + ) + + +def test_call_cpp_conduit_platform_abi_id_mismatch(): + t_h = home_planet.Traveler("home") + cap = t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID + b"MISMATCH", + home_planet.cpp_type_info_capsule_Traveler, + b"raw_pointer_ephemeral", + ) + assert cap is None + + +def test_call_cpp_conduit_cpp_type_info_capsule_mismatch(): + t_h = home_planet.Traveler("home") + cap = t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID, + home_planet.cpp_type_info_capsule_int, + b"raw_pointer_ephemeral", + ) + assert cap is None + + +def test_call_cpp_conduit_pointer_kind_invalid(): + t_h = home_planet.Traveler("home") + with pytest.raises( + RuntimeError, match='^Invalid pointer_kind: "raw_pointer_ephemreal"$' + ): + t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID, + home_planet.cpp_type_info_capsule_Traveler, + b"raw_pointer_ephemreal", + ) + + +def test_home_only_basic(): + t_h = home_planet.Traveler("home") + assert t_h.luggage == "home" + assert home_planet.get_luggage(t_h) == "home" + + +def test_home_only_premium(): + p_h = home_planet.PremiumTraveler("home", 2) + assert p_h.luggage == "home" + assert home_planet.get_luggage(p_h) == "home" + assert home_planet.get_points(p_h) == 2 + + +def test_exo_only_basic(): + t_e = exo_planet_pybind11.Traveler("exo") + assert t_e.luggage == "exo" + assert exo_planet_pybind11.get_luggage(t_e) == "exo" + + +def test_exo_only_premium(): + p_e = exo_planet_pybind11.PremiumTraveler("exo", 3) + assert p_e.luggage == "exo" + assert exo_planet_pybind11.get_luggage(p_e) == "exo" + assert exo_planet_pybind11.get_points(p_e) == 3 + + +def test_home_passed_to_exo_basic(): + t_h = home_planet.Traveler("home") + assert exo_planet_pybind11.get_luggage(t_h) == "home" + + +def test_exo_passed_to_home_basic(): + t_e = exo_planet_pybind11.Traveler("exo") + assert home_planet.get_luggage(t_e) == "exo" + + +def test_home_passed_to_exo_premium(): + p_h = home_planet.PremiumTraveler("home", 2) + assert exo_planet_pybind11.get_luggage(p_h) == "home" + assert exo_planet_pybind11.get_points(p_h) == 2 + + +def test_exo_passed_to_home_premium(): + p_e = exo_planet_pybind11.PremiumTraveler("exo", 3) + assert home_planet.get_luggage(p_e) == "exo" + assert home_planet.get_points(p_e) == 3 + + +@pytest.mark.parametrize( + "traveler_type", [home_planet.Traveler, exo_planet_pybind11.Traveler] +) +def test_exo_planet_c_api_traveler(traveler_type): + t = traveler_type("socks") + assert exo_planet_c_api.GetLuggage(t) == "socks" + + +@pytest.mark.parametrize( + "premium_traveler_type", + [home_planet.PremiumTraveler, exo_planet_pybind11.PremiumTraveler], +) +def test_exo_planet_c_api_premium_traveler(premium_traveler_type): + pt = premium_traveler_type("gucci", 5) + assert exo_planet_c_api.GetLuggage(pt) == "gucci" + assert exo_planet_c_api.GetPoints(pt) == 5 + + +def test_home_planet_wrap_very_lonely_traveler(): + # This does not exercise the cpp_conduit feature, but is here to + # demonstrate that the cpp_conduit feature does not solve all + # cross-extension interoperability issues. + # Here is the proof that the following works for extensions with + # matching `PYBIND11_INTERNALS_ID`s: + # test_cpp_conduit.cpp: + # py::class_ + # home_planet_very_lonely_traveler.cpp: + # py::class_ + # See test_exo_planet_pybind11_wrap_very_lonely_traveler() for the negative + # test. + assert home_planet.LonelyTraveler is not None # Verify that the base class exists. + home_planet_very_lonely_traveler.wrap_very_lonely_traveler() + # Ensure that the derived class exists. + assert home_planet_very_lonely_traveler.VeryLonelyTraveler is not None + + +def test_exo_planet_pybind11_wrap_very_lonely_traveler(): + # See comment under test_home_planet_wrap_very_lonely_traveler() first. + # Here the `PYBIND11_INTERNALS_ID`s don't match between: + # test_cpp_conduit.cpp: + # py::class_ + # exo_planet_pybind11.cpp: + # py::class_ + assert home_planet.LonelyTraveler is not None # Verify that the base class exists. + with pytest.raises( + RuntimeError, + match='^generic_type: type "VeryLonelyTraveler" referenced unknown base type ' + '"pybind11_tests::test_cpp_conduit::LonelyTraveler"$', + ): + exo_planet_pybind11.wrap_very_lonely_traveler() diff --git a/wrap/pybind11/tests/test_cpp_conduit_traveler_bindings.h b/wrap/pybind11/tests/test_cpp_conduit_traveler_bindings.h new file mode 100644 index 000000000..4e52c90c1 --- /dev/null +++ b/wrap/pybind11/tests/test_cpp_conduit_traveler_bindings.h @@ -0,0 +1,47 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include + +#include "test_cpp_conduit_traveler_types.h" + +#include + +namespace pybind11_tests { +namespace test_cpp_conduit { + +namespace py = pybind11; + +inline void wrap_traveler(py::module_ m) { + py::class_(m, "Traveler") + .def(py::init()) + .def_readwrite("luggage", &Traveler::luggage) + // See issue #3788: + .def("__getattr__", [](const Traveler &self, const std::string &key) { + return "Traveler GetAttr: " + key + " luggage: " + self.luggage; + }); + + m.def("get_luggage", [](const Traveler &person) { return person.luggage; }); + + py::class_(m, "PremiumTraveler") + .def(py::init()) + .def_readwrite("points", &PremiumTraveler::points) + // See issue #3788: + .def("__getattr__", [](const PremiumTraveler &self, const std::string &key) { + return "PremiumTraveler GetAttr: " + key + " points: " + std::to_string(self.points); + }); + + m.def("get_points", [](const PremiumTraveler &person) { return person.points; }); +} + +inline void wrap_lonely_traveler(py::module_ m) { + py::class_(std::move(m), "LonelyTraveler"); +} + +inline void wrap_very_lonely_traveler(py::module_ m) { + py::class_(std::move(m), "VeryLonelyTraveler"); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_cpp_conduit_traveler_types.h b/wrap/pybind11/tests/test_cpp_conduit_traveler_types.h new file mode 100644 index 000000000..b8e6a5a77 --- /dev/null +++ b/wrap/pybind11/tests/test_cpp_conduit_traveler_types.h @@ -0,0 +1,25 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include + +namespace pybind11_tests { +namespace test_cpp_conduit { + +struct Traveler { + explicit Traveler(const std::string &luggage) : luggage(luggage) {} + std::string luggage; +}; + +struct PremiumTraveler : Traveler { + explicit PremiumTraveler(const std::string &luggage, int points) + : Traveler(luggage), points(points) {} + int points; +}; + +struct LonelyTraveler {}; +struct VeryLonelyTraveler : LonelyTraveler {}; + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/wrap/pybind11/tests/test_custom_type_casters.py b/wrap/pybind11/tests/test_custom_type_casters.py index 689b1e9cb..dcf3ca734 100644 --- a/wrap/pybind11/tests/test_custom_type_casters.py +++ b/wrap/pybind11/tests/test_custom_type_casters.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest +import env # noqa: F401 from pybind11_tests import custom_type_casters as m @@ -94,6 +95,7 @@ def test_noconvert_args(msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_custom_caster_destruction(): """Tests that returning a pointer to a type that gets converted with a custom type caster gets destroyed when the function has py::return_value_policy::take_ownership policy applied. diff --git a/wrap/pybind11/tests/test_custom_type_setup.py b/wrap/pybind11/tests/test_custom_type_setup.py index 56922c6dd..bb2865cad 100644 --- a/wrap/pybind11/tests/test_custom_type_setup.py +++ b/wrap/pybind11/tests/test_custom_type_setup.py @@ -9,7 +9,7 @@ import env # noqa: F401 from pybind11_tests import custom_type_setup as m -@pytest.fixture() +@pytest.fixture def gc_tester(): """Tests that an object is garbage collected. @@ -34,7 +34,7 @@ def gc_tester(): # PyPy does not seem to reliably garbage collect. -@pytest.mark.skipif("env.PYPY") +@pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_self_cycle(gc_tester): obj = m.OwnsPythonObjects() obj.value = obj @@ -42,7 +42,7 @@ def test_self_cycle(gc_tester): # PyPy does not seem to reliably garbage collect. -@pytest.mark.skipif("env.PYPY") +@pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_indirect_cycle(gc_tester): obj = m.OwnsPythonObjects() obj_list = [obj] diff --git a/wrap/pybind11/tests/test_docs_advanced_cast_custom.cpp b/wrap/pybind11/tests/test_docs_advanced_cast_custom.cpp new file mode 100644 index 000000000..0ec1b17ac --- /dev/null +++ b/wrap/pybind11/tests/test_docs_advanced_cast_custom.cpp @@ -0,0 +1,68 @@ +// ######################################################################### +// PLEASE UPDATE docs/advanced/cast/custom.rst IF ANY CHANGES ARE MADE HERE. +// ######################################################################### + +#include "pybind11_tests.h" + +namespace user_space { + +struct Point2D { + double x; + double y; +}; + +Point2D negate(const Point2D &point) { return Point2D{-point.x, -point.y}; } + +} // namespace user_space + +namespace pybind11 { +namespace detail { + +template <> +struct type_caster { + // This macro inserts a lot of boilerplate code and sets the type hint. + // `io_name` is used to specify different type hints for arguments and return values. + // The signature of our negate function would then look like: + // `negate(Sequence[float]) -> tuple[float, float]` + PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]")); + + // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments + // are used to indicate the return value policy and parent object (for + // return_value_policy::reference_internal) and are often ignored by custom casters. + // The return value should reflect the type hint specified by the second argument of `io_name`. + static handle + cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) { + return py::make_tuple(number.x, number.y).release(); + } + + // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The + // second argument indicates whether implicit conversions should be allowed. + // The accepted types should reflect the type hint specified by the first argument of + // `io_name`. + bool load(handle src, bool /*convert*/) { + // Check if handle is a Sequence + if (!py::isinstance(src)) { + return false; + } + auto seq = py::reinterpret_borrow(src); + // Check if exactly two values are in the Sequence + if (seq.size() != 2) { + return false; + } + // Check if each element is either a float or an int + for (auto item : seq) { + if (!py::isinstance(item) && !py::isinstance(item)) { + return false; + } + } + value.x = seq[0].cast(); + value.y = seq[1].cast(); + return true; + } +}; + +} // namespace detail +} // namespace pybind11 + +// Bind the negate function +TEST_SUBMODULE(docs_advanced_cast_custom, m) { m.def("negate", user_space::negate); } diff --git a/wrap/pybind11/tests/test_docs_advanced_cast_custom.py b/wrap/pybind11/tests/test_docs_advanced_cast_custom.py new file mode 100644 index 000000000..8018b8f57 --- /dev/null +++ b/wrap/pybind11/tests/test_docs_advanced_cast_custom.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence + +if TYPE_CHECKING: + from conftest import SanitizedString + +from pybind11_tests import docs_advanced_cast_custom as m + + +def assert_negate_function( + input_sequence: Sequence[float], + target: tuple[float, float], +) -> None: + output = m.negate(input_sequence) + assert isinstance(output, tuple) + assert len(output) == 2 + assert isinstance(output[0], float) + assert isinstance(output[1], float) + assert output == target + + +def test_negate(doc: SanitizedString) -> None: + assert doc(m.negate) == "negate(arg0: Sequence[float]) -> tuple[float, float]" + assert_negate_function([1.0, -1.0], (-1.0, 1.0)) + assert_negate_function((1.0, -1.0), (-1.0, 1.0)) + assert_negate_function([1, -1], (-1.0, 1.0)) + assert_negate_function((1, -1), (-1.0, 1.0)) + + +def test_docs() -> None: + ########################################################################### + # PLEASE UPDATE docs/advanced/cast/custom.rst IF ANY CHANGES ARE MADE HERE. + ########################################################################### + point1 = [1.0, -1.0] + point2 = m.negate(point1) + assert point2 == (-1.0, 1.0) diff --git a/wrap/pybind11/tests/test_eigen_matrix.cpp b/wrap/pybind11/tests/test_eigen_matrix.cpp index 21261bfc2..4e6689a79 100644 --- a/wrap/pybind11/tests/test_eigen_matrix.cpp +++ b/wrap/pybind11/tests/test_eigen_matrix.cpp @@ -55,7 +55,7 @@ void reset_refs() { } // Returns element 2,1 from a matrix (used to test copy/nocopy) -double get_elem(const Eigen::Ref &m) { return m(2, 1); }; +double get_elem(const Eigen::Ref &m) { return m(2, 1); } // Returns a matrix with 10*r + 100*c added to each matrix element (to help test that the matrix // reference is referencing rows/columns correctly). @@ -76,7 +76,7 @@ struct CustomOperatorNew { Eigen::Matrix4d a = Eigen::Matrix4d::Zero(); Eigen::Matrix4d b = Eigen::Matrix4d::Identity(); - EIGEN_MAKE_ALIGNED_OPERATOR_NEW; + EIGEN_MAKE_ALIGNED_OPERATOR_NEW }; TEST_SUBMODULE(eigen_matrix, m) { @@ -440,4 +440,8 @@ TEST_SUBMODULE(eigen_matrix, m) { py::module_::import("numpy").attr("ones")(10); return v[0](5); }); + m.def("round_trip_vector", [](const Eigen::VectorXf &x) -> Eigen::VectorXf { return x; }); + m.def("round_trip_dense", [](const DenseMatrixR &m) -> DenseMatrixR { return m; }); + m.def("round_trip_dense_ref", + [](const Eigen::Ref &m) -> Eigen::Ref { return m; }); } diff --git a/wrap/pybind11/tests/test_eigen_matrix.py b/wrap/pybind11/tests/test_eigen_matrix.py index e1d7433f1..9324c2a7d 100644 --- a/wrap/pybind11/tests/test_eigen_matrix.py +++ b/wrap/pybind11/tests/test_eigen_matrix.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest +import env # noqa: F401 from pybind11_tests import ConstructorStats np = pytest.importorskip("numpy") @@ -94,19 +95,20 @@ def test_mutator_descriptors(): with pytest.raises(TypeError) as excinfo: m.fixed_mutator_r(zc) assert ( - "(arg0: numpy.ndarray[numpy.float32[5, 6]," - " flags.writeable, flags.c_contiguous]) -> None" in str(excinfo.value) + '(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[5, 6]",' + ' "flags.writeable", "flags.c_contiguous"]) -> None' in str(excinfo.value) ) with pytest.raises(TypeError) as excinfo: m.fixed_mutator_c(zr) assert ( - "(arg0: numpy.ndarray[numpy.float32[5, 6]," - " flags.writeable, flags.f_contiguous]) -> None" in str(excinfo.value) + '(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[5, 6]",' + ' "flags.writeable", "flags.f_contiguous"]) -> None' in str(excinfo.value) ) with pytest.raises(TypeError) as excinfo: m.fixed_mutator_a(np.array([[1, 2], [3, 4]], dtype="float32")) - assert "(arg0: numpy.ndarray[numpy.float32[5, 6], flags.writeable]) -> None" in str( - excinfo.value + assert ( + '(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[5, 6]", "flags.writeable"]) -> None' + in str(excinfo.value) ) zr.flags.writeable = False with pytest.raises(TypeError): @@ -200,7 +202,7 @@ def test_negative_stride_from_python(msg): msg(excinfo.value) == """ double_threer(): incompatible function arguments. The following argument types are supported: - 1. (arg0: numpy.ndarray[numpy.float32[1, 3], flags.writeable]) -> None + 1. (arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[1, 3]", "flags.writeable"]) -> None Invoked with: """ + repr(np.array([5.0, 4.0, 3.0], dtype="float32")) @@ -212,7 +214,7 @@ def test_negative_stride_from_python(msg): msg(excinfo.value) == """ double_threec(): incompatible function arguments. The following argument types are supported: - 1. (arg0: numpy.ndarray[numpy.float32[3, 1], flags.writeable]) -> None + 1. (arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[3, 1]", "flags.writeable"]) -> None Invoked with: """ + repr(np.array([7.0, 4.0, 1.0], dtype="float32")) @@ -243,9 +245,9 @@ def test_eigen_ref_to_python(): chols = [m.cholesky1, m.cholesky2, m.cholesky3, m.cholesky4] for i, chol in enumerate(chols, start=1): mymat = chol(np.array([[1.0, 2, 4], [2, 13, 23], [4, 23, 77]])) - assert np.all( - mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]]) - ), f"cholesky{i}" + assert np.all(mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]])), ( + f"cholesky{i}" + ) def assign_both(a1, a2, r, c, v): @@ -394,6 +396,7 @@ def test_eigen_return_references(): np.testing.assert_array_equal(a_copy5, c5want) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def assert_keeps_alive(cl, method, *args): cstats = ConstructorStats.get(cl) start_with = cstats.alive() @@ -409,6 +412,7 @@ def assert_keeps_alive(cl, method, *args): assert cstats.alive() == start_with +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_eigen_keepalive(): a = m.ReturnTester() cstats = ConstructorStats.get(m.ReturnTester) @@ -631,16 +635,16 @@ def test_nocopy_wrapper(): with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_colmajor) assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) - assert ", flags.f_contiguous" in str(excinfo.value) + assert ', "flags.f_contiguous"' in str(excinfo.value) assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_rowmajor) assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) - assert ", flags.f_contiguous" in str(excinfo.value) + assert ', "flags.f_contiguous"' in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(dbl_matrix_rowmajor) assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) - assert ", flags.f_contiguous" in str(excinfo.value) + assert ', "flags.f_contiguous"' in str(excinfo.value) # For the row-major test, we take a long matrix in row-major, so only the third is allowed: with pytest.raises(TypeError) as excinfo: @@ -648,20 +652,20 @@ def test_nocopy_wrapper(): assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value ) - assert ", flags.c_contiguous" in str(excinfo.value) + assert ', "flags.c_contiguous"' in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_colmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value ) - assert ", flags.c_contiguous" in str(excinfo.value) + assert ', "flags.c_contiguous"' in str(excinfo.value) assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_rowmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value ) - assert ", flags.c_contiguous" in str(excinfo.value) + assert ', "flags.c_contiguous"' in str(excinfo.value) def test_eigen_ref_life_support(): @@ -697,25 +701,25 @@ def test_dense_signature(doc): assert ( doc(m.double_col) == """ - double_col(arg0: numpy.ndarray[numpy.float32[m, 1]]) -> numpy.ndarray[numpy.float32[m, 1]] + double_col(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, 1]"]) -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, 1]"] """ ) assert ( doc(m.double_row) == """ - double_row(arg0: numpy.ndarray[numpy.float32[1, n]]) -> numpy.ndarray[numpy.float32[1, n]] + double_row(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[1, n]"]) -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[1, n]"] """ ) assert doc(m.double_complex) == ( """ - double_complex(arg0: numpy.ndarray[numpy.complex64[m, 1]])""" - """ -> numpy.ndarray[numpy.complex64[m, 1]] + double_complex(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.complex64, "[m, 1]"])""" + """ -> typing.Annotated[numpy.typing.NDArray[numpy.complex64], "[m, 1]"] """ ) assert doc(m.double_mat_rm) == ( """ - double_mat_rm(arg0: numpy.ndarray[numpy.float32[m, n]])""" - """ -> numpy.ndarray[numpy.float32[m, n]] + double_mat_rm(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, n]"])""" + """ -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]"] """ ) @@ -814,3 +818,22 @@ def test_custom_operator_new(): o = m.CustomOperatorNew() np.testing.assert_allclose(o.a, 0.0) np.testing.assert_allclose(o.b.diagonal(), 1.0) + + +def test_arraylike_signature(doc): + assert doc(m.round_trip_vector) == ( + 'round_trip_vector(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, 1]"])' + ' -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, 1]"]' + ) + assert doc(m.round_trip_dense) == ( + 'round_trip_dense(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, n]"])' + ' -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]"]' + ) + assert doc(m.round_trip_dense_ref) == ( + 'round_trip_dense_ref(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]", "flags.writeable", "flags.c_contiguous"])' + ' -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]", "flags.writeable", "flags.c_contiguous"]' + ) + m.round_trip_vector([1.0, 2.0]) + m.round_trip_dense([[1.0, 2.0], [3.0, 4.0]]) + with pytest.raises(TypeError, match="incompatible function arguments"): + m.round_trip_dense_ref([[1.0, 2.0], [3.0, 4.0]]) diff --git a/wrap/pybind11/tests/test_eigen_tensor.inl b/wrap/pybind11/tests/test_eigen_tensor.inl index 25cf29f15..3b3641e8d 100644 --- a/wrap/pybind11/tests/test_eigen_tensor.inl +++ b/wrap/pybind11/tests/test_eigen_tensor.inl @@ -147,33 +147,39 @@ void init_tensor_module(pybind11::module &m) { m.def( "take_fixed_tensor", - []() { Eigen::aligned_allocator< Eigen::TensorFixedSize, Options>> allocator; - return new (allocator.allocate(1)) + static auto *obj = new (allocator.allocate(1)) Eigen::TensorFixedSize, Options>( get_fixed_tensor()); + return obj; // take_ownership will fail. }, py::return_value_policy::take_ownership); m.def( "take_tensor", - []() { return new Eigen::Tensor(get_tensor()); }, + []() { + static auto *obj = new Eigen::Tensor(get_tensor()); + return obj; // take_ownership will fail. + }, py::return_value_policy::take_ownership); m.def( "take_const_tensor", []() -> const Eigen::Tensor * { - return new Eigen::Tensor(get_tensor()); + static auto *obj = new Eigen::Tensor(get_tensor()); + return obj; // take_ownership will fail. }, py::return_value_policy::take_ownership); m.def( "take_view_tensor", []() -> const Eigen::TensorMap> * { - return new Eigen::TensorMap>(get_tensor()); + static auto *obj + = new Eigen::TensorMap>(get_tensor()); + return obj; // take_ownership will fail. }, py::return_value_policy::take_ownership); diff --git a/wrap/pybind11/tests/test_eigen_tensor.py b/wrap/pybind11/tests/test_eigen_tensor.py index a2b99d9d7..4b018551b 100644 --- a/wrap/pybind11/tests/test_eigen_tensor.py +++ b/wrap/pybind11/tests/test_eigen_tensor.py @@ -4,6 +4,8 @@ import sys import pytest +import env # noqa: F401 + np = pytest.importorskip("numpy") eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") submodules = [eigen_tensor.c_style, eigen_tensor.f_style] @@ -61,6 +63,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) +@pytest.mark.skipif("env.GRAALPY", reason="Different refcounting mechanism") def test_reference_internal(m, member_name): if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") @@ -268,23 +271,46 @@ def test_round_trip_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_doc_string(m, doc): assert ( - doc(m.copy_tensor) == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" + doc(m.copy_tensor) + == 'copy_tensor() -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]' ) assert ( doc(m.copy_fixed_tensor) - == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2]]" + == 'copy_fixed_tensor() -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[3, 5, 2]"]' ) assert ( doc(m.reference_const_tensor) - == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" + == 'reference_const_tensor() -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]' ) - order_flag = f"flags.{m.needed_options.lower()}_contiguous" + order_flag = f'"flags.{m.needed_options.lower()}_contiguous"' assert doc(m.round_trip_view_tensor) == ( - f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])" - f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" + f'round_trip_view_tensor(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}])' + f' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}]' ) assert doc(m.round_trip_const_view_tensor) == ( - f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])" - " -> numpy.ndarray[numpy.float64[?, ?, ?]]" + f'round_trip_const_view_tensor(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", {order_flag}])' + ' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]' ) + + +@pytest.mark.parametrize("m", submodules) +def test_arraylike_signature(m, doc): + order_flag = f'"flags.{m.needed_options.lower()}_contiguous"' + assert doc(m.round_trip_tensor) == ( + 'round_trip_tensor(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float64, "[?, ?, ?]"])' + ' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]' + ) + assert doc(m.round_trip_tensor_noconvert) == ( + 'round_trip_tensor_noconvert(tensor: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"])' + ' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]' + ) + assert doc(m.round_trip_view_tensor) == ( + f'round_trip_view_tensor(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}])' + f' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}]' + ) + m.round_trip_tensor(tensor_ref.tolist()) + with pytest.raises(TypeError, match="incompatible function arguments"): + m.round_trip_tensor_noconvert(tensor_ref.tolist()) + with pytest.raises(TypeError, match="incompatible function arguments"): + m.round_trip_view_tensor(tensor_ref.tolist()) diff --git a/wrap/pybind11/tests/test_embed/CMakeLists.txt b/wrap/pybind11/tests/test_embed/CMakeLists.txt index 9b539cd42..f646458d1 100644 --- a/wrap/pybind11/tests/test_embed/CMakeLists.txt +++ b/wrap/pybind11/tests/test_embed/CMakeLists.txt @@ -1,8 +1,10 @@ possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID) -if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy") - message(STATUS "Skipping embed test on PyPy") - add_custom_target(cpptest) # Dummy target on PyPy. Embedding is not supported. +if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" + OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") + message(STATUS "Skipping embed test on PyPy or GraalPy") + add_custom_target(cpptest) # Dummy target on PyPy or GraalPy. Embedding is not supported. set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}") return() endif() diff --git a/wrap/pybind11/tests/test_enum.py b/wrap/pybind11/tests/test_enum.py index 9914b9001..044ef1803 100644 --- a/wrap/pybind11/tests/test_enum.py +++ b/wrap/pybind11/tests/test_enum.py @@ -1,11 +1,15 @@ # ruff: noqa: SIM201 SIM300 SIM202 from __future__ import annotations +import re + import pytest +import env # noqa: F401 from pybind11_tests import enums as m +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_unscoped_enum(): assert str(m.UnscopedEnum.EOne) == "UnscopedEnum.EOne" assert str(m.UnscopedEnum.ETwo) == "UnscopedEnum.ETwo" @@ -193,6 +197,7 @@ def test_implicit_conversion(): assert repr(x) == "{: 3, : 4}" +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_binary_operators(): assert int(m.Flags.Read) == 4 assert int(m.Flags.Write) == 2 @@ -268,3 +273,61 @@ def test_docstring_signatures(): def test_str_signature(): for enum_type in [m.ScopedEnum, m.UnscopedEnum]: assert enum_type.__str__.__doc__.startswith("__str__") + + +def test_generated_dunder_methods_pos_only(): + for enum_type in [m.ScopedEnum, m.UnscopedEnum]: + for binary_op in [ + "__eq__", + "__ne__", + "__ge__", + "__gt__", + "__lt__", + "__le__", + "__and__", + "__rand__", + # "__or__", # fail with some compilers (__doc__ = "Return self|value.") + # "__ror__", # fail with some compilers (__doc__ = "Return value|self.") + "__xor__", + "__rxor__", + "__rxor__", + ]: + method = getattr(enum_type, binary_op, None) + if method is not None: + assert ( + re.match( + rf"^{binary_op}\(self: [\w\.]+, other: [\w\.]+, /\)", + method.__doc__, + ) + is not None + ) + for unary_op in [ + "__int__", + "__index__", + "__hash__", + "__str__", + "__repr__", + ]: + method = getattr(enum_type, unary_op, None) + if method is not None: + assert ( + re.match( + rf"^{unary_op}\(self: [\w\.]+, /\)", + method.__doc__, + ) + is not None + ) + assert ( + re.match( + r"^__getstate__\(self: [\w\.]+, /\)", + enum_type.__getstate__.__doc__, + ) + is not None + ) + assert ( + re.match( + r"^__setstate__\(self: [\w\.]+, state: [\w\.]+, /\)", + enum_type.__setstate__.__doc__, + ) + is not None + ) diff --git a/wrap/pybind11/tests/test_eval.py b/wrap/pybind11/tests/test_eval.py index 45b68ece7..8ac1907c7 100644 --- a/wrap/pybind11/tests/test_eval.py +++ b/wrap/pybind11/tests/test_eval.py @@ -19,7 +19,7 @@ def test_evals(capture): assert m.test_eval_failure() -@pytest.mark.xfail("env.PYPY", raises=RuntimeError) +@pytest.mark.xfail("env.PYPY or env.GRAALPY", raises=RuntimeError) def test_eval_file(): filename = os.path.join(os.path.dirname(__file__), "test_eval_call.py") assert m.test_eval_file(filename) diff --git a/wrap/pybind11/tests/test_exceptions.cpp b/wrap/pybind11/tests/test_exceptions.cpp index c1d05bb24..0a970065b 100644 --- a/wrap/pybind11/tests/test_exceptions.cpp +++ b/wrap/pybind11/tests/test_exceptions.cpp @@ -111,6 +111,16 @@ struct PythonAlreadySetInDestructor { py::str s; }; +struct CustomData { + explicit CustomData(const std::string &a) : a(a) {} + std::string a; +}; + +struct MyException7 { + explicit MyException7(const CustomData &message) : message(message) {} + CustomData message; +}; + TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); @@ -385,4 +395,33 @@ TEST_SUBMODULE(exceptions, m) { // m.def("pass_exception_void", [](const py::exception&) {}); // Does not compile. m.def("return_exception_void", []() { return py::exception(); }); + + m.def("throws7", []() { + auto data = CustomData("abc"); + throw MyException7(data); + }); + + py::class_(m, "CustomData", py::module_local()) + .def(py::init()) + .def_readwrite("a", &CustomData::a); + + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store + PythonMyException7_storage; + PythonMyException7_storage.call_once_and_store_result([&]() { + auto mod = py::module_::import("custom_exceptions"); + py::object obj = mod.attr("PythonMyException7"); + return obj; + }); + + py::register_local_exception_translator([](std::exception_ptr p) { + try { + if (p) { + std::rethrow_exception(p); + } + } catch (const MyException7 &e) { + auto exc_type = PythonMyException7_storage.get_stored(); + py::object exc_inst = exc_type(e.message); + PyErr_SetObject(PyExc_Exception, exc_inst.ptr()); + } + }); } diff --git a/wrap/pybind11/tests/test_exceptions.py b/wrap/pybind11/tests/test_exceptions.py index b33997eee..47214a702 100644 --- a/wrap/pybind11/tests/test_exceptions.py +++ b/wrap/pybind11/tests/test_exceptions.py @@ -3,6 +3,7 @@ from __future__ import annotations import sys import pytest +from custom_exceptions import PythonMyException7 import env import pybind11_cross_module_tests as cm @@ -75,7 +76,7 @@ def test_cross_module_exceptions(msg): # TODO: FIXME @pytest.mark.xfail( - "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang'))", + "env.MACOS and env.PYPY", raises=RuntimeError, reason="See Issue #2847, PR #2999, PR #4324", ) @@ -103,28 +104,24 @@ def ignore_pytest_unraisable_warning(f): @pytest.mark.xfail(env.PYPY, reason="Failure on PyPy 3.8 (7.3.7)", strict=False) @ignore_pytest_unraisable_warning def test_python_alreadyset_in_destructor(monkeypatch, capsys): - hooked = False triggered = False - if hasattr(sys, "unraisablehook"): # Python 3.8+ - hooked = True - # Don't take `sys.unraisablehook`, as that's overwritten by pytest - default_hook = sys.__unraisablehook__ + # Don't take `sys.unraisablehook`, as that's overwritten by pytest + default_hook = sys.__unraisablehook__ - def hook(unraisable_hook_args): - exc_type, exc_value, exc_tb, err_msg, obj = unraisable_hook_args - if obj == "already_set demo": - nonlocal triggered - triggered = True - default_hook(unraisable_hook_args) - return + def hook(unraisable_hook_args): + exc_type, exc_value, exc_tb, err_msg, obj = unraisable_hook_args + if obj == "already_set demo": + nonlocal triggered + triggered = True + default_hook(unraisable_hook_args) + return - # Use monkeypatch so pytest can apply and remove the patch as appropriate - monkeypatch.setattr(sys, "unraisablehook", hook) + # Use monkeypatch so pytest can apply and remove the patch as appropriate + monkeypatch.setattr(sys, "unraisablehook", hook) assert m.python_alreadyset_in_destructor("already_set demo") is True - if hooked: - assert triggered is True + assert triggered is True _, captured_stderr = capsys.readouterr() assert captured_stderr.startswith("Exception ignored in: 'already_set demo'") @@ -199,7 +196,12 @@ def test_custom(msg): raise RuntimeError("Exception error: caught child from parent") from err assert msg(excinfo.value) == "this is a helper-defined translated exception" + with pytest.raises(PythonMyException7) as excinfo: + m.throws7() + assert msg(excinfo.value) == "[PythonMyException7]: abc" + +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_nested_throws(capture): """Tests nested (e.g. C++ -> Python -> C++) exception handling""" @@ -368,6 +370,7 @@ def _test_flaky_exception_failure_point_init_py_3_12(): "env.PYPY and sys.version_info[:2] < (3, 12)", reason="PyErr_NormalizeException Segmentation fault", ) +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_flaky_exception_failure_point_init(): if sys.version_info[:2] < (3, 12): _test_flaky_exception_failure_point_init_before_py_3_12() @@ -375,6 +378,7 @@ def test_flaky_exception_failure_point_init(): _test_flaky_exception_failure_point_init_py_3_12() +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_flaky_exception_failure_point_str(): what, py_err_set_after_what = m.error_already_set_what( FlakyException, ("failure_point_str",) diff --git a/wrap/pybind11/tests/test_factory_constructors.py b/wrap/pybind11/tests/test_factory_constructors.py index 0ddad5e32..1d3a9bcdd 100644 --- a/wrap/pybind11/tests/test_factory_constructors.py +++ b/wrap/pybind11/tests/test_factory_constructors.py @@ -4,11 +4,13 @@ import re import pytest +import env # noqa: F401 from pybind11_tests import ConstructorStats from pybind11_tests import factory_constructors as m from pybind11_tests.factory_constructors import tag +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_basic(): """Tests py::init_factory() wrapper around various ways of returning the object""" @@ -102,6 +104,7 @@ def test_init_factory_signature(msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_casting(): """Tests py::init_factory() wrapper with various upcasting and downcasting returns""" @@ -150,6 +153,7 @@ def test_init_factory_casting(): ] +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_alias(): """Tests py::init_factory() wrapper with value conversions and alias types""" @@ -220,6 +224,7 @@ def test_init_factory_alias(): ] +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_dual(): """Tests init factory functions with dual main/alias factory functions""" from pybind11_tests.factory_constructors import TestFactory7 @@ -302,6 +307,7 @@ def test_init_factory_dual(): ] +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_no_placement_new(capture): """Prior to 2.2, `py::init<...>` relied on the type supporting placement new; this tests a class without placement new support.""" @@ -350,6 +356,7 @@ def strip_comments(s): return re.sub(r"\s+#.*", "", s) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_a(capture, msg): """When the constructor is overloaded, previous overloads can require a preallocated value. This test makes sure that such preallocated values only happen when they might be necessary, @@ -372,6 +379,7 @@ def test_reallocation_a(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_b(capture, msg): with capture: create_and_destroy(1.5) @@ -388,6 +396,7 @@ def test_reallocation_b(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_c(capture, msg): with capture: create_and_destroy(2, 3) @@ -402,6 +411,7 @@ def test_reallocation_c(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_d(capture, msg): with capture: create_and_destroy(2.5, 3) @@ -417,6 +427,7 @@ def test_reallocation_d(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_e(capture, msg): with capture: create_and_destroy(3.5, 4.5) @@ -432,6 +443,7 @@ def test_reallocation_e(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_f(capture, msg): with capture: create_and_destroy(4, 0.5) @@ -448,6 +460,7 @@ def test_reallocation_f(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_g(capture, msg): with capture: create_and_destroy(5, "hi") diff --git a/wrap/pybind11/tests/test_gil_scoped.py b/wrap/pybind11/tests/test_gil_scoped.py index a18387684..2db771a50 100644 --- a/wrap/pybind11/tests/test_gil_scoped.py +++ b/wrap/pybind11/tests/test_gil_scoped.py @@ -71,24 +71,28 @@ def test_cross_module_gil_inner_pybind11_acquired(): m.test_cross_module_gil_inner_pybind11_acquired() +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_cross_module_gil_nested_custom_released(): """Makes sure that the GIL can be nested acquired/released by another module from a GIL-released state using custom locking logic.""" m.test_cross_module_gil_nested_custom_released() +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_cross_module_gil_nested_custom_acquired(): """Makes sure that the GIL can be nested acquired/acquired by another module from a GIL-acquired state using custom locking logic.""" m.test_cross_module_gil_nested_custom_acquired() +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_cross_module_gil_nested_pybind11_released(): """Makes sure that the GIL can be nested acquired/released by another module from a GIL-released state using pybind11 locking logic.""" m.test_cross_module_gil_nested_pybind11_released() +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_cross_module_gil_nested_pybind11_acquired(): """Makes sure that the GIL can be nested acquired/acquired by another module from a GIL-acquired state using pybind11 locking logic.""" @@ -103,6 +107,11 @@ def test_nested_acquire(): assert m.test_nested_acquire(0xAB) == "171" +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") +@pytest.mark.skipif( + env.GRAALPY and sys.platform == "darwin", + reason="Transiently crashes on GraalPy on OS X", +) def test_multi_acquire_release_cross_module(): for bits in range(16 * 8): internals_ids = m.test_multi_acquire_release_cross_module(bits) @@ -204,8 +213,12 @@ def _run_in_threads(test_fn, num_threads, parallel): thread.join() -# TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9 +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_one_thread(test_fn): """Makes sure there is no GIL deadlock when running in a thread. @@ -214,8 +227,12 @@ def test_run_in_process_one_thread(test_fn): assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0 -# TODO: FIXME on macOS Python 3.9 +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_multiple_threads_parallel(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. @@ -224,8 +241,12 @@ def test_run_in_process_multiple_threads_parallel(test_fn): assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0 -# TODO: FIXME on macOS Python 3.9 +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_multiple_threads_sequential(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. @@ -234,8 +255,12 @@ def test_run_in_process_multiple_threads_sequential(test_fn): assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0 -# TODO: FIXME on macOS Python 3.9 +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_direct(test_fn): """Makes sure there is no GIL deadlock when using processes. diff --git a/wrap/pybind11/tests/test_iostream.py b/wrap/pybind11/tests/test_iostream.py index f7eeff502..c3d987787 100644 --- a/wrap/pybind11/tests/test_iostream.py +++ b/wrap/pybind11/tests/test_iostream.py @@ -1,8 +1,11 @@ from __future__ import annotations +import sys from contextlib import redirect_stderr, redirect_stdout from io import StringIO +import pytest + from pybind11_tests import iostream as m @@ -270,6 +273,7 @@ def test_redirect_both(capfd): assert stream2.getvalue() == msg2 +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_threading(): with m.ostream_redirect(stdout=True, stderr=False): # start some threads diff --git a/wrap/pybind11/tests/test_kwargs_and_defaults.cpp b/wrap/pybind11/tests/test_kwargs_and_defaults.cpp index bc76ec7c2..831947f16 100644 --- a/wrap/pybind11/tests/test_kwargs_and_defaults.cpp +++ b/wrap/pybind11/tests/test_kwargs_and_defaults.cpp @@ -322,4 +322,10 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { py::pos_only{}, py::arg("i"), py::arg("j")); + + // Test support for args and kwargs subclasses + m.def("args_kwargs_subclass_function", + [](const py::Args &args, const py::KWArgs &kwargs) { + return py::make_tuple(args, kwargs); + }); } diff --git a/wrap/pybind11/tests/test_kwargs_and_defaults.py b/wrap/pybind11/tests/test_kwargs_and_defaults.py index b9b1a7ea8..e558d8ad2 100644 --- a/wrap/pybind11/tests/test_kwargs_and_defaults.py +++ b/wrap/pybind11/tests/test_kwargs_and_defaults.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest +import env # noqa: F401 from pybind11_tests import kwargs_and_defaults as m @@ -17,6 +18,10 @@ def test_function_signatures(doc): assert ( doc(m.args_kwargs_function) == "args_kwargs_function(*args, **kwargs) -> tuple" ) + assert ( + doc(m.args_kwargs_subclass_function) + == "args_kwargs_subclass_function(*args: str, **kwargs: str) -> tuple" + ) assert ( doc(m.KWClass.foo0) == "foo0(self: m.kwargs_and_defaults.KWClass, arg0: int, arg1: float) -> None" @@ -98,6 +103,7 @@ def test_arg_and_kwargs(): args = "a1", "a2" kwargs = {"arg3": "a3", "arg4": 4} assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs) + assert m.args_kwargs_subclass_function(*args, **kwargs) == (args, kwargs) def test_mixed_args_and_kwargs(msg): @@ -378,6 +384,7 @@ def test_signatures(): ) +@pytest.mark.skipif("env.GRAALPY", reason="Different refcounting mechanism") def test_args_refcount(): """Issue/PR #1216 - py::args elements get double-inc_ref()ed when combined with regular arguments""" @@ -413,6 +420,12 @@ def test_args_refcount(): ) assert refcount(myval) == expected + assert m.args_kwargs_subclass_function(7, 8, myval, a=1, b=myval) == ( + (7, 8, myval), + {"a": 1, "b": myval}, + ) + assert refcount(myval) == expected + exp3 = refcount(myval, myval, myval) assert m.args_refcount(myval, myval, myval) == (exp3, exp3, exp3) assert refcount(myval) == expected diff --git a/wrap/pybind11/tests/test_methods_and_attributes.cpp b/wrap/pybind11/tests/test_methods_and_attributes.cpp index f433847c7..e324c8bdd 100644 --- a/wrap/pybind11/tests/test_methods_and_attributes.cpp +++ b/wrap/pybind11/tests/test_methods_and_attributes.cpp @@ -294,7 +294,7 @@ TEST_SUBMODULE(methods_and_attributes, m) { static_cast( &ExampleMandA::overloaded)); }) - .def("__str__", &ExampleMandA::toString) + .def("__str__", &ExampleMandA::toString, py::pos_only()) .def_readwrite("value", &ExampleMandA::value); // test_copy_method diff --git a/wrap/pybind11/tests/test_methods_and_attributes.py b/wrap/pybind11/tests/test_methods_and_attributes.py index dfa31f546..cecc18464 100644 --- a/wrap/pybind11/tests/test_methods_and_attributes.py +++ b/wrap/pybind11/tests/test_methods_and_attributes.py @@ -4,7 +4,7 @@ import sys import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats from pybind11_tests import methods_and_attributes as m @@ -19,6 +19,13 @@ NO_DELETER_MSG = ( ) +def test_self_only_pos_only(): + assert ( + m.ExampleMandA.__str__.__doc__ + == "__str__(self: pybind11_tests.methods_and_attributes.ExampleMandA, /) -> str\n" + ) + + def test_methods_and_attributes(): instance1 = m.ExampleMandA() instance2 = m.ExampleMandA(32) @@ -68,6 +75,9 @@ def test_methods_and_attributes(): instance1.value = 100 assert str(instance1) == "ExampleMandA[value=100]" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.ExampleMandA) assert cstats.alive() == 2 del instance1, instance2 @@ -316,6 +326,8 @@ def test_dynamic_attributes(): instance.__dict__ = [] assert str(excinfo.value) == "__dict__ must be set to a dictionary, not a 'list'" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") cstats = ConstructorStats.get(m.DynamicClass) assert cstats.alive() == 1 del instance @@ -337,6 +349,7 @@ def test_dynamic_attributes(): # https://foss.heptapod.net/pypy/pypy/-/issues/2447 @pytest.mark.xfail("env.PYPY") +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_cyclic_gc(): # One object references itself instance = m.DynamicClass() diff --git a/wrap/pybind11/tests/test_modules.cpp b/wrap/pybind11/tests/test_modules.cpp index 18a7ec74c..7f01687c7 100644 --- a/wrap/pybind11/tests/test_modules.cpp +++ b/wrap/pybind11/tests/test_modules.cpp @@ -90,32 +90,32 @@ TEST_SUBMODULE(modules, m) { try { py::class_(dm, "Dupe1"); failures.append("Dupe1 class"); - } catch (std::runtime_error &) { + } catch (std::runtime_error &) { // NOLINT(bugprone-empty-catch) } try { dm.def("Dupe1", []() { return Dupe1(); }); failures.append("Dupe1 function"); - } catch (std::runtime_error &) { + } catch (std::runtime_error &) { // NOLINT(bugprone-empty-catch) } try { py::class_(dm, "dupe1_factory"); failures.append("dupe1_factory"); - } catch (std::runtime_error &) { + } catch (std::runtime_error &) { // NOLINT(bugprone-empty-catch) } try { py::exception(dm, "Dupe2"); failures.append("Dupe2"); - } catch (std::runtime_error &) { + } catch (std::runtime_error &) { // NOLINT(bugprone-empty-catch) } try { dm.def("DupeException", []() { return 30; }); failures.append("DupeException1"); - } catch (std::runtime_error &) { + } catch (std::runtime_error &) { // NOLINT(bugprone-empty-catch) } try { py::class_(dm, "DupeException"); failures.append("DupeException2"); - } catch (std::runtime_error &) { + } catch (std::runtime_error &) { // NOLINT(bugprone-empty-catch) } return failures; diff --git a/wrap/pybind11/tests/test_modules.py b/wrap/pybind11/tests/test_modules.py index 95835e14e..ad898be89 100644 --- a/wrap/pybind11/tests/test_modules.py +++ b/wrap/pybind11/tests/test_modules.py @@ -39,6 +39,9 @@ def test_reference_internal(): assert str(b.get_a2()) == "A[43]" assert str(b.a2) == "A[43]" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + astats, bstats = ConstructorStats.get(ms.A), ConstructorStats.get(ms.B) assert astats.alive() == 2 assert bstats.alive() == 1 @@ -78,6 +81,13 @@ def test_pydoc(): assert pydoc.text.docmodule(pybind11_tests) +def test_module_handle_type_name(): + assert ( + m.def_submodule.__doc__ + == "def_submodule(arg0: types.ModuleType, arg1: str) -> types.ModuleType\n" + ) + + def test_duplicate_registration(): """Registering two things with the same name""" @@ -97,9 +107,9 @@ def test_def_submodule_failures(): sm = m.def_submodule(m, b"ScratchSubModuleName") # Using bytes to show it works. assert sm.__name__ == m.__name__ + "." + "ScratchSubModuleName" malformed_utf8 = b"\x80" - if env.PYPY: + if env.PYPY or env.GRAALPY: # It is not worth the effort finding a trigger for a failure when running with PyPy. - pytest.skip("Sufficiently exercised on platforms other than PyPy.") + pytest.skip("Sufficiently exercised on platforms other than PyPy/GraalPy.") else: # Meant to trigger PyModule_GetName() failure: sm_name_orig = sm.__name__ diff --git a/wrap/pybind11/tests/test_multiple_inheritance.py b/wrap/pybind11/tests/test_multiple_inheritance.py index d445824b5..6f5a656f5 100644 --- a/wrap/pybind11/tests/test_multiple_inheritance.py +++ b/wrap/pybind11/tests/test_multiple_inheritance.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats from pybind11_tests import multiple_inheritance as m @@ -279,8 +279,9 @@ def test_mi_unaligned_base(): c = m.I801C() d = m.I801D() - # + 4 below because we have the two instances, and each instance has offset base I801B2 - assert ConstructorStats.detail_reg_inst() == n_inst + 4 + if not env.GRAALPY: + # + 4 below because we have the two instances, and each instance has offset base I801B2 + assert ConstructorStats.detail_reg_inst() == n_inst + 4 b1c = m.i801b1_c(c) assert b1c is c b2c = m.i801b2_c(c) @@ -290,6 +291,9 @@ def test_mi_unaligned_base(): b2d = m.i801b2_d(d) assert b2d is d + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + assert ConstructorStats.detail_reg_inst() == n_inst + 4 # no extra instances del c, b1c, b2c assert ConstructorStats.detail_reg_inst() == n_inst + 2 @@ -312,7 +316,8 @@ def test_mi_base_return(): assert d1.a == 1 assert d1.b == 2 - assert ConstructorStats.detail_reg_inst() == n_inst + 4 + if not env.GRAALPY: + assert ConstructorStats.detail_reg_inst() == n_inst + 4 c2 = m.i801c_b2() assert type(c2) is m.I801C @@ -324,12 +329,13 @@ def test_mi_base_return(): assert d2.a == 1 assert d2.b == 2 - assert ConstructorStats.detail_reg_inst() == n_inst + 8 + if not env.GRAALPY: + assert ConstructorStats.detail_reg_inst() == n_inst + 8 - del c2 - assert ConstructorStats.detail_reg_inst() == n_inst + 6 - del c1, d1, d2 - assert ConstructorStats.detail_reg_inst() == n_inst + del c2 + assert ConstructorStats.detail_reg_inst() == n_inst + 6 + del c1, d1, d2 + assert ConstructorStats.detail_reg_inst() == n_inst # Returning an unregistered derived type with a registered base; we won't # pick up the derived type, obviously, but should still work (as an object diff --git a/wrap/pybind11/tests/test_numpy_array.cpp b/wrap/pybind11/tests/test_numpy_array.cpp index c2f754208..1bfca33bb 100644 --- a/wrap/pybind11/tests/test_numpy_array.cpp +++ b/wrap/pybind11/tests/test_numpy_array.cpp @@ -156,6 +156,55 @@ py::handle auxiliaries(T &&r, T2 &&r2) { return l.release(); } +template +PyObjectType convert_to_pyobjecttype(py::object obj); + +template <> +PyObject *convert_to_pyobjecttype(py::object obj) { + return obj.release().ptr(); +} + +template <> +py::handle convert_to_pyobjecttype(py::object obj) { + return obj.release(); +} + +template <> +py::object convert_to_pyobjecttype(py::object obj) { + return obj; +} + +template +std::string pass_array_return_sum_str_values(const py::array_t &objs) { + std::string sum_str_values; + for (const auto &obj : objs) { + sum_str_values += py::str(obj.attr("value")); + } + return sum_str_values; +} + +template +py::list pass_array_return_as_list(const py::array_t &objs) { + return objs; +} + +template +py::array_t return_array_cpp_loop(const py::list &objs) { + py::size_t arr_size = py::len(objs); + py::array_t arr_from_list(static_cast(arr_size)); + PyObjectType *data = arr_from_list.mutable_data(); + for (py::size_t i = 0; i < arr_size; i++) { + assert(!data[i]); + data[i] = convert_to_pyobjecttype(objs[i].attr("value")); + } + return arr_from_list; +} + +template +py::array_t return_array_from_list(const py::list &objs) { + return objs; +} + // note: declaration at local scope would create a dangling reference! static int data_i = 42; @@ -520,28 +569,30 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("round_trip_float", [](double d) { return d; }); sm.def("pass_array_pyobject_ptr_return_sum_str_values", - [](const py::array_t &objs) { - std::string sum_str_values; - for (const auto &obj : objs) { - sum_str_values += py::str(obj.attr("value")); - } - return sum_str_values; - }); + pass_array_return_sum_str_values); + sm.def("pass_array_handle_return_sum_str_values", + pass_array_return_sum_str_values); + sm.def("pass_array_object_return_sum_str_values", + pass_array_return_sum_str_values); - sm.def("pass_array_pyobject_ptr_return_as_list", - [](const py::array_t &objs) -> py::list { return objs; }); + sm.def("pass_array_pyobject_ptr_return_as_list", pass_array_return_as_list); + sm.def("pass_array_handle_return_as_list", pass_array_return_as_list); + sm.def("pass_array_object_return_as_list", pass_array_return_as_list); - sm.def("return_array_pyobject_ptr_cpp_loop", [](const py::list &objs) { - py::size_t arr_size = py::len(objs); - py::array_t arr_from_list(static_cast(arr_size)); - PyObject **data = arr_from_list.mutable_data(); - for (py::size_t i = 0; i < arr_size; i++) { - assert(data[i] == nullptr); - data[i] = py::cast(objs[i].attr("value")); - } - return arr_from_list; - }); + sm.def("return_array_pyobject_ptr_cpp_loop", return_array_cpp_loop); + sm.def("return_array_handle_cpp_loop", return_array_cpp_loop); + sm.def("return_array_object_cpp_loop", return_array_cpp_loop); - sm.def("return_array_pyobject_ptr_from_list", - [](const py::list &objs) -> py::array_t { return objs; }); + sm.def("return_array_pyobject_ptr_from_list", return_array_from_list); + sm.def("return_array_handle_from_list", return_array_from_list); + sm.def("return_array_object_from_list", return_array_from_list); + + sm.def( + "round_trip_array_t", + [](const py::array_t &x) -> py::array_t { return x; }, + py::arg("x")); + sm.def( + "round_trip_array_t_noconvert", + [](const py::array_t &x) -> py::array_t { return x; }, + py::arg("x").noconvert()); } diff --git a/wrap/pybind11/tests/test_numpy_array.py b/wrap/pybind11/tests/test_numpy_array.py index 4726a8e73..3a3f22a64 100644 --- a/wrap/pybind11/tests/test_numpy_array.py +++ b/wrap/pybind11/tests/test_numpy_array.py @@ -24,7 +24,7 @@ def test_dtypes(): ) -@pytest.fixture() +@pytest.fixture def arr(): return np.array([[1, 2, 3], [4, 5, 6]], "=u2") @@ -242,6 +242,7 @@ def test_wrap(): assert_references(a1m, a2, a1) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_numpy_view(capture): with capture: ac = m.ArrayClass() @@ -320,13 +321,13 @@ def test_overload_resolution(msg): msg(excinfo.value) == """ overloaded(): incompatible function arguments. The following argument types are supported: - 1. (arg0: numpy.ndarray[numpy.float64]) -> str - 2. (arg0: numpy.ndarray[numpy.float32]) -> str - 3. (arg0: numpy.ndarray[numpy.int32]) -> str - 4. (arg0: numpy.ndarray[numpy.uint16]) -> str - 5. (arg0: numpy.ndarray[numpy.int64]) -> str - 6. (arg0: numpy.ndarray[numpy.complex128]) -> str - 7. (arg0: numpy.ndarray[numpy.complex64]) -> str + 1. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]) -> str + 2. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32]) -> str + 3. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.int32]) -> str + 4. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.uint16]) -> str + 5. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.int64]) -> str + 6. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.complex128]) -> str + 7. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.complex64]) -> str Invoked with: 'not an array' """ @@ -342,8 +343,8 @@ def test_overload_resolution(msg): assert m.overloaded3(np.array([1], dtype="intc")) == "int" expected_exc = """ overloaded3(): incompatible function arguments. The following argument types are supported: - 1. (arg0: numpy.ndarray[numpy.int32]) -> str - 2. (arg0: numpy.ndarray[numpy.float64]) -> str + 1. (arg0: numpy.typing.NDArray[numpy.int32]) -> str + 2. (arg0: numpy.typing.NDArray[numpy.float64]) -> str Invoked with: """ @@ -465,7 +466,7 @@ def test_array_resize(): assert b.shape == (8, 8) -@pytest.mark.xfail("env.PYPY") +@pytest.mark.xfail("env.PYPY or env.GRAALPY") def test_array_create_and_resize(): a = m.create_and_resize(2) assert a.size == 4 @@ -527,7 +528,7 @@ def test_index_using_ellipsis(): ], ) def test_format_descriptors_for_floating_point_types(test_func): - assert "numpy.ndarray[numpy.float" in test_func.__doc__ + assert "numpy.typing.ArrayLike, numpy.float" in test_func.__doc__ @pytest.mark.parametrize("forcecast", [False, True]) @@ -628,45 +629,75 @@ def UnwrapPyValueHolder(vhs): return [vh.value for vh in vhs] -def test_pass_array_pyobject_ptr_return_sum_str_values_ndarray(): +PASS_ARRAY_PYOBJECT_RETURN_SUM_STR_VALUES_FUNCTIONS = [ + m.pass_array_pyobject_ptr_return_sum_str_values, + m.pass_array_handle_return_sum_str_values, + m.pass_array_object_return_sum_str_values, +] + + +@pytest.mark.parametrize( + "pass_array", PASS_ARRAY_PYOBJECT_RETURN_SUM_STR_VALUES_FUNCTIONS +) +def test_pass_array_object_return_sum_str_values_ndarray(pass_array): # Intentionally all temporaries, do not change. assert ( - m.pass_array_pyobject_ptr_return_sum_str_values( - np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object) - ) + pass_array(np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object)) == "-3four5.0" ) -def test_pass_array_pyobject_ptr_return_sum_str_values_list(): +@pytest.mark.parametrize( + "pass_array", PASS_ARRAY_PYOBJECT_RETURN_SUM_STR_VALUES_FUNCTIONS +) +def test_pass_array_object_return_sum_str_values_list(pass_array): # Intentionally all temporaries, do not change. - assert ( - m.pass_array_pyobject_ptr_return_sum_str_values( - WrapWithPyValueHolder(2, "three", -4.0) - ) - == "2three-4.0" - ) + assert pass_array(WrapWithPyValueHolder(2, "three", -4.0)) == "2three-4.0" -def test_pass_array_pyobject_ptr_return_as_list(): +@pytest.mark.parametrize( + "pass_array", + [ + m.pass_array_pyobject_ptr_return_as_list, + m.pass_array_handle_return_as_list, + m.pass_array_object_return_as_list, + ], +) +def test_pass_array_object_return_as_list(pass_array): # Intentionally all temporaries, do not change. assert UnwrapPyValueHolder( - m.pass_array_pyobject_ptr_return_as_list( - np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object) - ) + pass_array(np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object)) ) == [-1, "two", 3.0] @pytest.mark.parametrize( - ("return_array_pyobject_ptr", "unwrap"), + ("return_array", "unwrap"), [ (m.return_array_pyobject_ptr_cpp_loop, list), + (m.return_array_handle_cpp_loop, list), + (m.return_array_object_cpp_loop, list), (m.return_array_pyobject_ptr_from_list, UnwrapPyValueHolder), + (m.return_array_handle_from_list, UnwrapPyValueHolder), + (m.return_array_object_from_list, UnwrapPyValueHolder), ], ) -def test_return_array_pyobject_ptr_cpp_loop(return_array_pyobject_ptr, unwrap): +def test_return_array_object_cpp_loop(return_array, unwrap): # Intentionally all temporaries, do not change. - arr_from_list = return_array_pyobject_ptr(WrapWithPyValueHolder(6, "seven", -8.0)) + arr_from_list = return_array(WrapWithPyValueHolder(6, "seven", -8.0)) assert isinstance(arr_from_list, np.ndarray) assert arr_from_list.dtype == np.dtype("O") assert unwrap(arr_from_list) == [6, "seven", -8.0] + + +def test_arraylike_signature(doc): + assert ( + doc(m.round_trip_array_t) + == "round_trip_array_t(x: typing.Annotated[numpy.typing.ArrayLike, numpy.float32]) -> numpy.typing.NDArray[numpy.float32]" + ) + assert ( + doc(m.round_trip_array_t_noconvert) + == "round_trip_array_t_noconvert(x: numpy.typing.NDArray[numpy.float32]) -> numpy.typing.NDArray[numpy.float32]" + ) + m.round_trip_array_t([1, 2, 3]) + with pytest.raises(TypeError, match="incompatible function arguments"): + m.round_trip_array_t_noconvert([1, 2, 3]) diff --git a/wrap/pybind11/tests/test_numpy_dtypes.cpp b/wrap/pybind11/tests/test_numpy_dtypes.cpp index ed77ec024..04bf19f3e 100644 --- a/wrap/pybind11/tests/test_numpy_dtypes.cpp +++ b/wrap/pybind11/tests/test_numpy_dtypes.cpp @@ -11,6 +11,9 @@ #include "pybind11_tests.h" +#include +#include + #ifdef __GNUC__ # define PYBIND11_PACKED(cls) cls __attribute__((__packed__)) #else @@ -266,6 +269,8 @@ py::array_t test_array_ctors(int i) { return fill(arr_t(buf_ndim1_null)); case 44: return fill(py::array(buf_ndim1_null)); + default: + break; } return arr_t(); } @@ -295,6 +300,15 @@ py::list test_dtype_ctors() { return list; } +template +py::array_t dispatch_array_increment(py::array_t arr) { + py::array_t res(arr.shape(0)); + for (py::ssize_t i = 0; i < arr.shape(0); ++i) { + res.mutable_at(i) = T(arr.at(i) + 1); + } + return res; +} + struct A {}; struct B {}; @@ -494,6 +508,98 @@ TEST_SUBMODULE(numpy_dtypes, m) { } return list; }); + m.def("test_dtype_num_of", []() -> py::list { + py::list res; +#define TEST_DTYPE(T) res.append(py::make_tuple(py::dtype::of().num(), py::dtype::num_of())); + TEST_DTYPE(bool) + TEST_DTYPE(signed char) + TEST_DTYPE(unsigned char) + TEST_DTYPE(short) + TEST_DTYPE(unsigned short) + TEST_DTYPE(int) + TEST_DTYPE(unsigned int) + TEST_DTYPE(long) + TEST_DTYPE(unsigned long) + TEST_DTYPE(long long) + TEST_DTYPE(unsigned long long) + TEST_DTYPE(float) + TEST_DTYPE(double) + TEST_DTYPE(long double) + TEST_DTYPE(std::complex) + TEST_DTYPE(std::complex) + TEST_DTYPE(std::complex) + TEST_DTYPE(int8_t) + TEST_DTYPE(uint8_t) + TEST_DTYPE(int16_t) + TEST_DTYPE(uint16_t) + TEST_DTYPE(int32_t) + TEST_DTYPE(uint32_t) + TEST_DTYPE(int64_t) + TEST_DTYPE(uint64_t) +#undef TEST_DTYPE + return res; + }); + m.def("test_dtype_normalized_num", []() -> py::list { + py::list res; +#define TEST_DTYPE(NT, T) \ + res.append(py::make_tuple(py::dtype(py::detail::npy_api::NT).normalized_num(), \ + py::dtype::num_of())); + TEST_DTYPE(NPY_BOOL_, bool) + TEST_DTYPE(NPY_BYTE_, signed char); + TEST_DTYPE(NPY_UBYTE_, unsigned char); + TEST_DTYPE(NPY_SHORT_, short); + TEST_DTYPE(NPY_USHORT_, unsigned short); + TEST_DTYPE(NPY_INT_, int); + TEST_DTYPE(NPY_UINT_, unsigned int); + TEST_DTYPE(NPY_LONG_, long); + TEST_DTYPE(NPY_ULONG_, unsigned long); + TEST_DTYPE(NPY_LONGLONG_, long long); + TEST_DTYPE(NPY_ULONGLONG_, unsigned long long); + TEST_DTYPE(NPY_FLOAT_, float); + TEST_DTYPE(NPY_DOUBLE_, double); + TEST_DTYPE(NPY_LONGDOUBLE_, long double); + TEST_DTYPE(NPY_CFLOAT_, std::complex); + TEST_DTYPE(NPY_CDOUBLE_, std::complex); + TEST_DTYPE(NPY_CLONGDOUBLE_, std::complex); + TEST_DTYPE(NPY_INT8_, int8_t); + TEST_DTYPE(NPY_UINT8_, uint8_t); + TEST_DTYPE(NPY_INT16_, int16_t); + TEST_DTYPE(NPY_UINT16_, uint16_t); + TEST_DTYPE(NPY_INT32_, int32_t); + TEST_DTYPE(NPY_UINT32_, uint32_t); + TEST_DTYPE(NPY_INT64_, int64_t); + TEST_DTYPE(NPY_UINT64_, uint64_t); +#undef TEST_DTYPE + return res; + }); + m.def("test_dtype_switch", [](const py::array &arr) -> py::array { + switch (arr.dtype().normalized_num()) { + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + case py::dtype::num_of(): + return dispatch_array_increment(arr); + default: + throw std::runtime_error("Unsupported dtype"); + } + }); m.def("test_dtype_methods", []() { py::list list; auto dt1 = py::dtype::of(); diff --git a/wrap/pybind11/tests/test_numpy_dtypes.py b/wrap/pybind11/tests/test_numpy_dtypes.py index 8ae239ed8..685a76fd3 100644 --- a/wrap/pybind11/tests/test_numpy_dtypes.py +++ b/wrap/pybind11/tests/test_numpy_dtypes.py @@ -188,6 +188,28 @@ def test_dtype(simple_dtype): chr(np.dtype(ch).flags) for ch in expected_chars ] + for a, b in m.test_dtype_num_of(): + assert a == b + + for a, b in m.test_dtype_normalized_num(): + assert a == b + + arr = np.array([4, 84, 21, 36]) + # Note: "ulong" does not work in NumPy 1.x, so we use "L" + assert (m.test_dtype_switch(arr.astype("byte")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("ubyte")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("short")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("ushort")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("intc")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("uintc")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("long")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("L")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("longlong")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("ulonglong")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("single")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("double")) == arr + 1).all() + assert (m.test_dtype_switch(arr.astype("longdouble")) == arr + 1).all() + def test_recarray(simple_dtype, packed_dtype): elements = [(False, 0, 0.0, -0.0), (True, 1, 1.5, -2.5), (False, 2, 3.0, -5.0)] @@ -351,7 +373,7 @@ def test_complex_array(): def test_signature(doc): assert ( doc(m.create_rec_nested) - == "create_rec_nested(arg0: int) -> numpy.ndarray[NestedStruct]" + == "create_rec_nested(arg0: int) -> numpy.typing.NDArray[NestedStruct]" ) diff --git a/wrap/pybind11/tests/test_numpy_vectorize.py b/wrap/pybind11/tests/test_numpy_vectorize.py index ce38d72d9..0768759d1 100644 --- a/wrap/pybind11/tests/test_numpy_vectorize.py +++ b/wrap/pybind11/tests/test_numpy_vectorize.py @@ -150,7 +150,7 @@ def test_docs(doc): assert ( doc(m.vectorized_func) == """ - vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object + vectorized_func(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.int32], arg1: typing.Annotated[numpy.typing.ArrayLike, numpy.float32], arg2: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]) -> object """ ) @@ -212,12 +212,12 @@ def test_passthrough_arguments(doc): + ", ".join( [ "arg0: float", - "arg1: numpy.ndarray[numpy.float64]", - "arg2: numpy.ndarray[numpy.float64]", - "arg3: numpy.ndarray[numpy.int32]", + "arg1: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]", + "arg2: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]", + "arg3: typing.Annotated[numpy.typing.ArrayLike, numpy.int32]", "arg4: int", "arg5: m.numpy_vectorize.NonPODClass", - "arg6: numpy.ndarray[numpy.float64]", + "arg6: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]", ] ) + ") -> object" diff --git a/wrap/pybind11/tests/test_opaque_types.cpp b/wrap/pybind11/tests/test_opaque_types.cpp index 0386dba03..da3866cd0 100644 --- a/wrap/pybind11/tests/test_opaque_types.cpp +++ b/wrap/pybind11/tests/test_opaque_types.cpp @@ -18,7 +18,7 @@ // This also deliberately doesn't use the below StringList type alias to test // that MAKE_OPAQUE can handle a type containing a `,`. (The `std::allocator` // bit is just the default `std::vector` allocator). -PYBIND11_MAKE_OPAQUE(std::vector>); +PYBIND11_MAKE_OPAQUE(std::vector>) using StringList = std::vector>; @@ -65,7 +65,7 @@ TEST_SUBMODULE(opaque_types, m) { m.def("return_unique_ptr", []() -> std::unique_ptr { auto *result = new StringList(); - result->push_back("some value"); + result->emplace_back("some value"); return std::unique_ptr(result); }); diff --git a/wrap/pybind11/tests/test_opaque_types.py b/wrap/pybind11/tests/test_opaque_types.py index 342086436..56c9b5db1 100644 --- a/wrap/pybind11/tests/test_opaque_types.py +++ b/wrap/pybind11/tests/test_opaque_types.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest +import env from pybind11_tests import ConstructorStats, UserType from pybind11_tests import opaque_types as m @@ -30,7 +31,9 @@ def test_pointers(msg): living_before = ConstructorStats.get(UserType).alive() assert m.get_void_ptr_value(m.return_void_ptr()) == 0x1234 assert m.get_void_ptr_value(UserType()) # Should also work for other C++ types - assert ConstructorStats.get(UserType).alive() == living_before + + if not env.GRAALPY: + assert ConstructorStats.get(UserType).alive() == living_before with pytest.raises(TypeError) as excinfo: m.get_void_ptr_value([1, 2, 3]) # This should not work diff --git a/wrap/pybind11/tests/test_operator_overloading.py b/wrap/pybind11/tests/test_operator_overloading.py index b6760902d..47949042d 100644 --- a/wrap/pybind11/tests/test_operator_overloading.py +++ b/wrap/pybind11/tests/test_operator_overloading.py @@ -2,10 +2,12 @@ from __future__ import annotations import pytest +import env from pybind11_tests import ConstructorStats from pybind11_tests import operators as m +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_operator_overloading(): v1 = m.Vector2(1, 2) v2 = m.Vector(3, -1) @@ -49,6 +51,9 @@ def test_operator_overloading(): v2 /= v1 assert str(v2) == "[2.000000, 8.000000]" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.Vector2) assert cstats.alive() == 3 del v1 @@ -83,6 +88,7 @@ def test_operator_overloading(): assert cstats.move_assignments == 0 +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_operators_notimplemented(): """#393: need to return NotSupported to ensure correct arithmetic operator behavior""" diff --git a/wrap/pybind11/tests/test_pickling.py b/wrap/pybind11/tests/test_pickling.py index ad67a1df9..d3551efc1 100644 --- a/wrap/pybind11/tests/test_pickling.py +++ b/wrap/pybind11/tests/test_pickling.py @@ -20,7 +20,7 @@ def test_pickle_simple_callable(): # all C Python versions. with pytest.raises(TypeError) as excinfo: pickle.dumps(m.simple_callable) - assert re.search("can.*t pickle .*PyCapsule.* object", str(excinfo.value)) + assert re.search("can.*t pickle .*[Cc]apsule.* object", str(excinfo.value)) @pytest.mark.parametrize("cls_name", ["Pickleable", "PickleableNew"]) @@ -93,3 +93,20 @@ def test_roundtrip_simple_cpp_derived(): # Issue #3062: pickleable base C++ classes can incur object slicing # if derived typeid is not registered with pybind11 assert not m.check_dynamic_cast_SimpleCppDerived(p2) + + +def test_new_style_pickle_getstate_pos_only(): + assert ( + re.match( + r"^__getstate__\(self: [\w\.]+, /\)", m.PickleableNew.__getstate__.__doc__ + ) + is not None + ) + if hasattr(m, "PickleableWithDictNew"): + assert ( + re.match( + r"^__getstate__\(self: [\w\.]+, /\)", + m.PickleableWithDictNew.__getstate__.__doc__, + ) + is not None + ) diff --git a/wrap/pybind11/tests/test_pytypes.cpp b/wrap/pybind11/tests/test_pytypes.cpp index 7c30978ce..5160e9f40 100644 --- a/wrap/pybind11/tests/test_pytypes.cpp +++ b/wrap/pybind11/tests/test_pytypes.cpp @@ -7,12 +7,21 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "pybind11_tests.h" #include +//__has_include has been part of C++17, no need to check it +#if defined(PYBIND11_CPP20) && __has_include() +# if !defined(PYBIND11_COMPILER_CLANG) || __clang_major__ >= 16 // llvm/llvm-project#52696 +# define PYBIND11_TEST_PYTYPES_HAS_RANGES +# include +# endif +#endif + namespace external { namespace detail { bool check(PyObject *o) { return PyFloat_Check(o) != 0; } @@ -109,7 +118,7 @@ void m_defs(py::module_ &m) { } // namespace handle_from_move_only_type_with_operator_PyObject -#if defined(__cpp_nontype_template_parameter_class) +#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) namespace literals { enum Color { RED = 0, BLUE = 1 }; @@ -129,6 +138,45 @@ typedef py::typing::TypeVar<"V"> TypeVarV; } // namespace typevar #endif +// Custom type for testing arg_name/return_name type hints +// RealNumber: +// * in arguments -> float | int +// * in return -> float +// The choice of types is not really useful, but just made different for testing purposes. +// According to `PEP 484 – Type Hints` annotating with `float` also allows `int`, +// so using `float | int` could be replaced by just `float`. + +struct RealNumber { + double value; +}; + +namespace pybind11 { +namespace detail { + +template <> +struct type_caster { + PYBIND11_TYPE_CASTER(RealNumber, io_name("Union[float, int]", "float")); + + static handle cast(const RealNumber &number, return_value_policy, handle) { + return py::float_(number.value).release(); + } + + bool load(handle src, bool convert) { + // If we're in no-convert mode, only load if given a float + if (!convert && !py::isinstance(src)) { + return false; + } + if (!py::isinstance(src) && !py::isinstance(src)) { + return false; + } + value.value = src.cast(); + return true; + } +}; + +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(pytypes, m) { m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); @@ -142,6 +190,18 @@ TEST_SUBMODULE(pytypes, m) { m.def("get_iterator", [] { return py::iterator(); }); // test_iterable m.def("get_iterable", [] { return py::iterable(); }); + m.def("get_first_item_from_iterable", [](const py::iterable &iter) { + // This tests the postfix increment operator + py::iterator it = iter.begin(); + py::iterator it2 = it++; + return *it2; + }); + m.def("get_second_item_from_iterable", [](const py::iterable &iter) { + // This tests the prefix increment operator + py::iterator it = iter.begin(); + ++it; + return *it; + }); m.def("get_frozenset_from_iterable", [](const py::iterable &iter) { return py::frozenset(iter); }); m.def("get_list_from_iterable", [](const py::iterable &iter) { return py::list(iter); }); @@ -905,12 +965,25 @@ TEST_SUBMODULE(pytypes, m) { m.def("annotate_optional_to_object", [](py::typing::Optional &o) -> py::object { return o; }); -#if defined(__cpp_nontype_template_parameter_class) +#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) py::enum_(m, "Color") .value("RED", literals::Color::RED) .value("BLUE", literals::Color::BLUE); m.def("annotate_literal", [](literals::LiteralFoo &o) -> py::object { return o; }); + // Literal with `@`, `%`, `{`, `}`, and `->` + m.def("identity_literal_exclamation", [](const py::typing::Literal<"\"!\""> &x) { return x; }); + m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); + m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); + m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); + m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); + m.def("identity_literal_arrow_with_io_name", + [](const py::typing::Literal<"\"->\""> &x, const RealNumber &) { return x; }); + m.def("identity_literal_arrow_with_callable", + [](const py::typing::Callable\""> &, + const RealNumber &)> &x) { return x; }); + m.def("identity_literal_all_special_chars", + [](const py::typing::Literal<"\"!@!!->{%}\""> &x) { return x; }); m.def("annotate_generic_containers", [](const py::typing::List &l) -> py::typing::List { return l; @@ -919,8 +992,203 @@ TEST_SUBMODULE(pytypes, m) { m.def("annotate_listT_to_T", [](const py::typing::List &l) -> typevar::TypeVarT { return l[0]; }); m.def("annotate_object_to_T", [](const py::object &o) -> typevar::TypeVarT { return o; }); - m.attr("if_defined__cpp_nontype_template_parameter_class") = true; + m.attr("defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL") = true; #else - m.attr("if_defined__cpp_nontype_template_parameter_class") = false; + m.attr("defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL") = false; #endif + +#if defined(PYBIND11_TEST_PYTYPES_HAS_RANGES) + + // test_tuple_ranges + m.def("tuple_iterator_default_initialization", []() { + using TupleIterator = decltype(std::declval().begin()); + static_assert(std::random_access_iterator); + return TupleIterator{} == TupleIterator{}; + }); + + m.def("transform_tuple_plus_one", [](py::tuple &tpl) { + py::list ret{}; + for (auto it : tpl | std::views::transform([](auto &o) { return py::cast(o) + 1; })) { + ret.append(py::int_(it)); + } + return ret; + }); + + // test_list_ranges + m.def("list_iterator_default_initialization", []() { + using ListIterator = decltype(std::declval().begin()); + static_assert(std::random_access_iterator); + return ListIterator{} == ListIterator{}; + }); + + m.def("transform_list_plus_one", [](py::list &lst) { + py::list ret{}; + for (auto it : lst | std::views::transform([](auto &o) { return py::cast(o) + 1; })) { + ret.append(py::int_(it)); + } + return ret; + }); + + // test_dict_ranges + m.def("dict_iterator_default_initialization", []() { + using DictIterator = decltype(std::declval().begin()); + static_assert(std::forward_iterator); + return DictIterator{} == DictIterator{}; + }); + + m.def("transform_dict_plus_one", [](py::dict &dct) { + py::list ret{}; + for (auto it : dct | std::views::transform([](auto &o) { + return std::pair{py::cast(o.first) + 1, + py::cast(o.second) + 1}; + })) { + ret.append(py::make_tuple(py::int_(it.first), py::int_(it.second))); + } + return ret; + }); + + m.attr("defined_PYBIND11_TEST_PYTYPES_HAS_RANGES") = true; +#else + m.attr("defined_PYBIND11_TEST_PYTYPES_HAS_RANGES") = false; +#endif + +#if defined(__cpp_inline_variables) + // Exercises const char* overload: + m.attr_with_type_hint>("list_int") = py::list(); + // Exercises py::handle overload: + m.attr_with_type_hint>(py::str("set_str")) = py::set(); + + struct Empty {}; + py::class_(m, "EmptyAnnotationClass"); + + struct Static {}; + auto static_class = py::class_(m, "Static"); + static_class.def(py::init()); + static_class.attr_with_type_hint>("x") = 1.0; + static_class.attr_with_type_hint>>( + "dict_str_int") + = py::dict(); + + struct Instance {}; + auto instance = py::class_(m, "Instance", py::dynamic_attr()); + instance.def(py::init()); + instance.attr_with_type_hint("y"); + + m.def("attr_with_type_hint_float_x", + [](py::handle obj) { obj.attr_with_type_hint("x"); }); + + m.attr_with_type_hint>("CONST_INT") = 3; + + m.attr("defined___cpp_inline_variables") = true; +#else + m.attr("defined___cpp_inline_variables") = false; +#endif + m.def("half_of_number", [](const RealNumber &x) { return RealNumber{x.value / 2}; }); + m.def( + "half_of_number_convert", + [](const RealNumber &x) { return RealNumber{x.value / 2}; }, + py::arg("x")); + m.def( + "half_of_number_noconvert", + [](const RealNumber &x) { return RealNumber{x.value / 2}; }, + py::arg("x").noconvert()); + // std::vector + m.def("half_of_number_vector", [](const std::vector &x) { + std::vector result; + result.reserve(x.size()); + for (auto num : x) { + result.push_back(RealNumber{num.value / 2}); + } + return result; + }); + // Tuple + m.def("half_of_number_tuple", [](const py::typing::Tuple &x) { + py::typing::Tuple result + = py::make_tuple(RealNumber{x[0].cast().value / 2}, + RealNumber{x[1].cast().value / 2}); + return result; + }); + // Tuple + m.def("half_of_number_tuple_ellipsis", + [](const py::typing::Tuple &x) { + py::typing::Tuple result(x.size()); + for (size_t i = 0; i < x.size(); ++i) { + result[i] = x[i].cast().value / 2; + } + return result; + }); + // Dict + m.def("half_of_number_dict", [](const py::typing::Dict &x) { + py::typing::Dict result; + for (auto it : x) { + result[it.first] = RealNumber{it.second.cast().value / 2}; + } + return result; + }); + // List + m.def("half_of_number_list", [](const py::typing::List &x) { + py::typing::List result; + for (auto num : x) { + result.append(RealNumber{num.cast().value / 2}); + } + return result; + }); + // List> + m.def("half_of_number_nested_list", + [](const py::typing::List> &x) { + py::typing::List> result_lists; + for (auto nums : x) { + py::typing::List result; + for (auto num : nums) { + result.append(RealNumber{num.cast().value / 2}); + } + result_lists.append(result); + } + return result_lists; + }); + // Set + m.def("identity_set", [](const py::typing::Set &x) { return x; }); + // Iterable + m.def("identity_iterable", [](const py::typing::Iterable &x) { return x; }); + // Iterator + m.def("identity_iterator", [](const py::typing::Iterator &x) { return x; }); + // Callable identity + m.def("identity_callable", + [](const py::typing::Callable &x) { return x; }); + // Callable identity + m.def("identity_callable_ellipsis", + [](const py::typing::Callable &x) { return x; }); + // Nested Callable identity + m.def("identity_nested_callable", + [](const py::typing::Callable( + py::typing::Callable)> &x) { return x; }); + // Callable + m.def("apply_callable", + [](const RealNumber &x, const py::typing::Callable &f) { + return f(x).cast(); + }); + // Callable + m.def("apply_callable_ellipsis", + [](const RealNumber &x, const py::typing::Callable &f) { + return f(x).cast(); + }); + // Union + m.def("identity_union", [](const py::typing::Union &x) { return x; }); + // Optional + m.def("identity_optional", [](const py::typing::Optional &x) { return x; }); + // TypeGuard + m.def("check_type_guard", + [](const py::typing::List &x) + -> py::typing::TypeGuard> { + for (const auto &item : x) { + if (!py::isinstance(item)) { + return false; + } + } + return true; + }); + // TypeIs + m.def("check_type_is", [](const py::object &x) -> py::typing::TypeIs { + return py::isinstance(x); + }); } diff --git a/wrap/pybind11/tests/test_pytypes.py b/wrap/pybind11/tests/test_pytypes.py index 30931e0b9..18932311e 100644 --- a/wrap/pybind11/tests/test_pytypes.py +++ b/wrap/pybind11/tests/test_pytypes.py @@ -52,6 +52,11 @@ def test_from_iterable(pytype, from_iter_func): def test_iterable(doc): assert doc(m.get_iterable) == "get_iterable() -> Iterable" + lst = [1, 2, 3] + i = m.get_first_item_from_iterable(lst) + assert i == 1 + i = m.get_second_item_from_iterable(lst) + assert i == 2 def test_float(doc): @@ -62,13 +67,13 @@ def test_list(capture, doc): assert m.list_no_args() == [] assert m.list_ssize_t() == [] assert m.list_size_t() == [] - lins = [1, 2] - m.list_insert_ssize_t(lins) - assert lins == [1, 83, 2] - m.list_insert_size_t(lins) - assert lins == [1, 83, 2, 57] - m.list_clear(lins) - assert lins == [] + lst = [1, 2] + m.list_insert_ssize_t(lst) + assert lst == [1, 83, 2] + m.list_insert_size_t(lst) + assert lst == [1, 83, 2, 57] + m.list_clear(lst) + assert lst == [] with capture: lst = m.get_list() @@ -262,6 +267,7 @@ def test_str(doc): m.str_from_std_string_input, ], ) +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_surrogate_pairs_unicode_error(func): input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass") with pytest.raises(UnicodeDecodeError): @@ -420,6 +426,7 @@ def test_accessor_moves(): pytest.skip("Not defined: PYBIND11_HANDLE_REF_DEBUG") +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_constructors(): """C++ default and converting constructors are equivalent to type calls in Python""" types = [bytes, bytearray, str, bool, int, float, tuple, list, dict, set] @@ -712,6 +719,7 @@ def test_pass_bytes_or_unicode_to_string_types(): m.pass_to_pybind11_str(malformed_utf8) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") @pytest.mark.parametrize( ("create_weakref", "create_weakref_with_callback"), [ @@ -765,7 +773,10 @@ def test_weakref_err(create_weakref, has_callback): ob = C() # Should raise TypeError on CPython - with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext(): + cm = pytest.raises(TypeError) + if env.PYPY or env.GRAALPY: + cm = contextlib.nullcontext() + with cm: _ = create_weakref(ob, callback) if has_callback else create_weakref(ob) @@ -1025,19 +1036,52 @@ def test_optional_object_annotations(doc): @pytest.mark.skipif( - not m.if_defined__cpp_nontype_template_parameter_class, - reason="C++20 feature not available.", + not m.defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL, + reason="C++20 non-type template args feature not available.", ) def test_literal(doc): assert ( doc(m.annotate_literal) == 'annotate_literal(arg0: Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]) -> object' ) + # The characters !, @, %, {, } and -> are used in the signature parser as special characters, but Literal should escape those for the parser to work. + assert ( + doc(m.identity_literal_exclamation) + == 'identity_literal_exclamation(arg0: Literal["!"]) -> Literal["!"]' + ) + assert ( + doc(m.identity_literal_at) + == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]' + ) + assert ( + doc(m.identity_literal_percent) + == 'identity_literal_percent(arg0: Literal["%"]) -> Literal["%"]' + ) + assert ( + doc(m.identity_literal_curly_open) + == 'identity_literal_curly_open(arg0: Literal["{"]) -> Literal["{"]' + ) + assert ( + doc(m.identity_literal_curly_close) + == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' + ) + assert ( + doc(m.identity_literal_arrow_with_io_name) + == 'identity_literal_arrow_with_io_name(arg0: Literal["->"], arg1: Union[float, int]) -> Literal["->"]' + ) + assert ( + doc(m.identity_literal_arrow_with_callable) + == 'identity_literal_arrow_with_callable(arg0: Callable[[Literal["->"], Union[float, int]], float]) -> Callable[[Literal["->"], Union[float, int]], float]' + ) + assert ( + doc(m.identity_literal_all_special_chars) + == 'identity_literal_all_special_chars(arg0: Literal["!@!!->{%}"]) -> Literal["!@!!->{%}"]' + ) @pytest.mark.skipif( - not m.if_defined__cpp_nontype_template_parameter_class, - reason="C++20 feature not available.", + not m.defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL, + reason="C++20 non-type template args feature not available.", ) def test_typevar(doc): assert ( @@ -1048,3 +1092,238 @@ def test_typevar(doc): assert doc(m.annotate_listT_to_T) == "annotate_listT_to_T(arg0: list[T]) -> T" assert doc(m.annotate_object_to_T) == "annotate_object_to_T(arg0: object) -> T" + + +@pytest.mark.skipif( + not m.defined_PYBIND11_TEST_PYTYPES_HAS_RANGES, + reason=" not available.", +) +@pytest.mark.parametrize( + ("tested_tuple", "expected"), + [((1,), [2]), ((3, 4), [4, 5]), ((7, 8, 9), [8, 9, 10])], +) +def test_tuple_ranges(tested_tuple, expected): + assert m.tuple_iterator_default_initialization() + assert m.transform_tuple_plus_one(tested_tuple) == expected + + +@pytest.mark.skipif( + not m.defined_PYBIND11_TEST_PYTYPES_HAS_RANGES, + reason=" not available.", +) +@pytest.mark.parametrize( + ("tested_list", "expected"), [([1], [2]), ([3, 4], [4, 5]), ([7, 8, 9], [8, 9, 10])] +) +def test_list_ranges(tested_list, expected): + assert m.list_iterator_default_initialization() + assert m.transform_list_plus_one(tested_list) == expected + + +@pytest.mark.skipif( + not m.defined_PYBIND11_TEST_PYTYPES_HAS_RANGES, + reason=" not available.", +) +@pytest.mark.parametrize( + ("tested_dict", "expected"), + [ + ({1: 2}, [(2, 3)]), + ({3: 4, 5: 6}, [(4, 5), (6, 7)]), + ({7: 8, 9: 10, 11: 12}, [(8, 9), (10, 11), (12, 13)]), + ], +) +def test_dict_ranges(tested_dict, expected): + assert m.dict_iterator_default_initialization() + assert m.transform_dict_plus_one(tested_dict) == expected + + +# https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older +def get_annotations_helper(o): + if isinstance(o, type): + return o.__dict__.get("__annotations__", None) + return getattr(o, "__annotations__", None) + + +@pytest.mark.skipif( + not m.defined___cpp_inline_variables, + reason="C++17 feature __cpp_inline_variables not available.", +) +def test_module_attribute_types() -> None: + module_annotations = get_annotations_helper(m) + + assert module_annotations["list_int"] == "list[int]" + assert module_annotations["set_str"] == "set[str]" + + +@pytest.mark.skipif( + not m.defined___cpp_inline_variables, + reason="C++17 feature __cpp_inline_variables not available.", +) +@pytest.mark.skipif( + sys.version_info < (3, 10), + reason="get_annotations function does not exist until Python3.10", +) +def test_get_annotations_compliance() -> None: + from inspect import get_annotations + + module_annotations = get_annotations(m) + + assert module_annotations["list_int"] == "list[int]" + assert module_annotations["set_str"] == "set[str]" + + +@pytest.mark.skipif( + not m.defined___cpp_inline_variables, + reason="C++17 feature __cpp_inline_variables not available.", +) +def test_class_attribute_types() -> None: + empty_annotations = get_annotations_helper(m.EmptyAnnotationClass) + static_annotations = get_annotations_helper(m.Static) + instance_annotations = get_annotations_helper(m.Instance) + + assert empty_annotations is None + assert static_annotations["x"] == "ClassVar[float]" + assert static_annotations["dict_str_int"] == "ClassVar[dict[str, int]]" + + assert m.Static.x == 1.0 + + m.Static.x = 3.0 + static = m.Static() + assert static.x == 3.0 + + static.dict_str_int["hi"] = 3 + assert m.Static().dict_str_int == {"hi": 3} + + assert instance_annotations["y"] == "float" + instance1 = m.Instance() + instance1.y = 4.0 + + instance2 = m.Instance() + instance2.y = 5.0 + + assert instance1.y != instance2.y + + +@pytest.mark.skipif( + not m.defined___cpp_inline_variables, + reason="C++17 feature __cpp_inline_variables not available.", +) +def test_redeclaration_attr_with_type_hint() -> None: + obj = m.Instance() + m.attr_with_type_hint_float_x(obj) + assert get_annotations_helper(obj)["x"] == "float" + with pytest.raises( + RuntimeError, match=r'^__annotations__\["x"\] was set already\.$' + ): + m.attr_with_type_hint_float_x(obj) + + +@pytest.mark.skipif( + not m.defined___cpp_inline_variables, + reason="C++17 feature __cpp_inline_variables not available.", +) +def test_final_annotation() -> None: + module_annotations = get_annotations_helper(m) + assert module_annotations["CONST_INT"] == "Final[int]" + + +def test_arg_return_type_hints(doc): + assert doc(m.half_of_number) == "half_of_number(arg0: Union[float, int]) -> float" + assert ( + doc(m.half_of_number_convert) + == "half_of_number_convert(x: Union[float, int]) -> float" + ) + assert ( + doc(m.half_of_number_noconvert) == "half_of_number_noconvert(x: float) -> float" + ) + assert m.half_of_number(2.0) == 1.0 + assert m.half_of_number(2) == 1.0 + assert m.half_of_number(0) == 0 + assert isinstance(m.half_of_number(0), float) + assert not isinstance(m.half_of_number(0), int) + # std::vector + assert ( + doc(m.half_of_number_vector) + == "half_of_number_vector(arg0: list[Union[float, int]]) -> list[float]" + ) + # Tuple + assert ( + doc(m.half_of_number_tuple) + == "half_of_number_tuple(arg0: tuple[Union[float, int], Union[float, int]]) -> tuple[float, float]" + ) + # Tuple + assert ( + doc(m.half_of_number_tuple_ellipsis) + == "half_of_number_tuple_ellipsis(arg0: tuple[Union[float, int], ...]) -> tuple[float, ...]" + ) + # Dict + assert ( + doc(m.half_of_number_dict) + == "half_of_number_dict(arg0: dict[str, Union[float, int]]) -> dict[str, float]" + ) + # List + assert ( + doc(m.half_of_number_list) + == "half_of_number_list(arg0: list[Union[float, int]]) -> list[float]" + ) + # List> + assert ( + doc(m.half_of_number_nested_list) + == "half_of_number_nested_list(arg0: list[list[Union[float, int]]]) -> list[list[float]]" + ) + # Set + assert ( + doc(m.identity_set) + == "identity_set(arg0: set[Union[float, int]]) -> set[float]" + ) + # Iterable + assert ( + doc(m.identity_iterable) + == "identity_iterable(arg0: Iterable[Union[float, int]]) -> Iterable[float]" + ) + # Iterator + assert ( + doc(m.identity_iterator) + == "identity_iterator(arg0: Iterator[Union[float, int]]) -> Iterator[float]" + ) + # Callable identity + assert ( + doc(m.identity_callable) + == "identity_callable(arg0: Callable[[Union[float, int]], float]) -> Callable[[Union[float, int]], float]" + ) + # Callable identity + assert ( + doc(m.identity_callable_ellipsis) + == "identity_callable_ellipsis(arg0: Callable[..., float]) -> Callable[..., float]" + ) + # Nested Callable identity + assert ( + doc(m.identity_nested_callable) + == "identity_nested_callable(arg0: Callable[[Callable[[Union[float, int]], float]], Callable[[Union[float, int]], float]]) -> Callable[[Callable[[Union[float, int]], float]], Callable[[Union[float, int]], float]]" + ) + # Callable + assert ( + doc(m.apply_callable) + == "apply_callable(arg0: Union[float, int], arg1: Callable[[Union[float, int]], float]) -> float" + ) + # Callable + assert ( + doc(m.apply_callable_ellipsis) + == "apply_callable_ellipsis(arg0: Union[float, int], arg1: Callable[..., float]) -> float" + ) + # Union + assert ( + doc(m.identity_union) + == "identity_union(arg0: Union[Union[float, int], str]) -> Union[float, str]" + ) + # Optional + assert ( + doc(m.identity_optional) + == "identity_optional(arg0: Optional[Union[float, int]]) -> Optional[float]" + ) + # TypeGuard + assert ( + doc(m.check_type_guard) + == "check_type_guard(arg0: list[object]) -> TypeGuard[list[float]]" + ) + # TypeIs + assert doc(m.check_type_is) == "check_type_is(arg0: object) -> TypeIs[float]" diff --git a/wrap/pybind11/tests/test_sequences_and_iterators.cpp b/wrap/pybind11/tests/test_sequences_and_iterators.cpp index 4a1d37f4d..c997b7300 100644 --- a/wrap/pybind11/tests/test_sequences_and_iterators.cpp +++ b/wrap/pybind11/tests/test_sequences_and_iterators.cpp @@ -86,8 +86,8 @@ private: }; using NonCopyableIntPair = std::pair; -PYBIND11_MAKE_OPAQUE(std::vector); -PYBIND11_MAKE_OPAQUE(std::vector); +PYBIND11_MAKE_OPAQUE(std::vector) +PYBIND11_MAKE_OPAQUE(std::vector) template py::list test_random_access_iterator(PythonType x) { diff --git a/wrap/pybind11/tests/test_sequences_and_iterators.py b/wrap/pybind11/tests/test_sequences_and_iterators.py index f609f553d..6fba6fba3 100644 --- a/wrap/pybind11/tests/test_sequences_and_iterators.py +++ b/wrap/pybind11/tests/test_sequences_and_iterators.py @@ -1,8 +1,11 @@ from __future__ import annotations +import re + import pytest from pytest import approx # noqa: PT013 +import env from pybind11_tests import ConstructorStats from pybind11_tests import sequences_and_iterators as m @@ -110,7 +113,8 @@ def test_sequence(): cstats = ConstructorStats.get(m.Sequence) s = m.Sequence(5) - assert cstats.values() == ["of size", "5"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "5"] assert "Sequence" in repr(s) assert len(s) == 5 @@ -123,16 +127,19 @@ def test_sequence(): assert s[3] == approx(56.78, rel=1e-05) rev = reversed(s) - assert cstats.values() == ["of size", "5"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "5"] rev2 = s[::-1] - assert cstats.values() == ["of size", "5"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "5"] it = iter(m.Sequence(0)) for _ in range(3): # __next__ must continue to raise StopIteration with pytest.raises(StopIteration): next(it) - assert cstats.values() == ["of size", "0"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "0"] expected = [0, 56.78, 0, 0, 12.34] assert rev == approx(expected, rel=1e-05) @@ -140,10 +147,14 @@ def test_sequence(): assert rev == rev2 rev[0::2] = m.Sequence([2.0, 2.0, 2.0]) - assert cstats.values() == ["of size", "3", "from std::vector"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "3", "from std::vector"] assert rev == approx([2, 56.78, 2, 0, 2], rel=1e-05) + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + assert cstats.alive() == 4 del it assert cstats.alive() == 3 @@ -244,16 +255,12 @@ def test_python_iterator_in_cpp(): def test_iterator_passthrough(): """#181: iterator passthrough did not compile""" - from pybind11_tests.sequences_and_iterators import iterator_passthrough - values = [3, 5, 7, 9, 11, 13, 15] - assert list(iterator_passthrough(iter(values))) == values + assert list(m.iterator_passthrough(iter(values))) == values def test_iterator_rvp(): """#388: Can't make iterators via make_iterator() with different r/v policies""" - import pybind11_tests.sequences_and_iterators as m - assert list(m.make_iterator_1()) == [1, 2, 3] assert list(m.make_iterator_2()) == [1, 2, 3] assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2())) @@ -265,3 +272,25 @@ def test_carray_iterator(): arr_h = m.CArrayHolder(*args_gt) args = list(arr_h) assert args_gt == args + + +def test_generated_dunder_methods_pos_only(): + string_map = m.StringMap({"hi": "bye", "black": "white"}) + for it in ( + m.make_iterator_1(), + m.make_iterator_2(), + m.iterator_passthrough(iter([3, 5, 7])), + iter(m.Sequence(5)), + iter(string_map), + string_map.items(), + string_map.values(), + iter(m.CArrayHolder(*[float(i) for i in range(3)])), + ): + assert ( + re.match(r"^__iter__\(self: [\w\.]+, /\)", type(it).__iter__.__doc__) + is not None + ) + assert ( + re.match(r"^__next__\(self: [\w\.]+, /\)", type(it).__next__.__doc__) + is not None + ) diff --git a/wrap/pybind11/tests/test_smart_ptr.cpp b/wrap/pybind11/tests/test_smart_ptr.cpp index 496073b3c..4ab43953f 100644 --- a/wrap/pybind11/tests/test_smart_ptr.cpp +++ b/wrap/pybind11/tests/test_smart_ptr.cpp @@ -11,6 +11,9 @@ #include "object.h" #include "pybind11_tests.h" +// This breaks on PYBIND11_DECLARE_HOLDER_TYPE +PYBIND11_WARNING_DISABLE_GCC("-Wpedantic") + namespace { // This is just a wrapper around unique_ptr, but with extra fields to deliberately bloat up the @@ -279,13 +282,13 @@ struct holder_helper> { } // namespace PYBIND11_NAMESPACE // Make pybind aware of the ref-counted wrapper type (s): -PYBIND11_DECLARE_HOLDER_TYPE(T, ref, true); +PYBIND11_DECLARE_HOLDER_TYPE(T, ref, true) // The following is not required anymore for std::shared_ptr, but it should compile without error: -PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr); -PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr); -PYBIND11_DECLARE_HOLDER_TYPE(T, custom_unique_ptr); -PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_with_addressof_operator); -PYBIND11_DECLARE_HOLDER_TYPE(T, unique_ptr_with_addressof_operator); +PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr) +PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr) +PYBIND11_DECLARE_HOLDER_TYPE(T, custom_unique_ptr) +PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_with_addressof_operator) +PYBIND11_DECLARE_HOLDER_TYPE(T, unique_ptr_with_addressof_operator) TEST_SUBMODULE(smart_ptr, m) { // Please do not interleave `struct` and `class` definitions with bindings code, diff --git a/wrap/pybind11/tests/test_smart_ptr.py b/wrap/pybind11/tests/test_smart_ptr.py index bf0ae4aeb..ab8a1ce62 100644 --- a/wrap/pybind11/tests/test_smart_ptr.py +++ b/wrap/pybind11/tests/test_smart_ptr.py @@ -2,10 +2,13 @@ from __future__ import annotations import pytest +import env # noqa: F401 + m = pytest.importorskip("pybind11_tests.smart_ptr") from pybind11_tests import ConstructorStats # noqa: E402 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_smart_ptr(capture): # Object1 for i, o in enumerate( @@ -118,6 +121,7 @@ def test_smart_ptr_refcounting(): assert m.test_object1_refcounting() +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_unique_nodelete(): o = m.MyObject4(23) assert o.value == 23 @@ -129,6 +133,7 @@ def test_unique_nodelete(): assert cstats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_unique_nodelete4a(): o = m.MyObject4a(23) assert o.value == 23 @@ -140,6 +145,7 @@ def test_unique_nodelete4a(): assert cstats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_unique_deleter(): m.MyObject4a(0) o = m.MyObject4b(23) @@ -156,6 +162,7 @@ def test_unique_deleter(): assert cstats4b.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_large_holder(): o = m.MyObject5(5) assert o.value == 5 @@ -165,6 +172,7 @@ def test_large_holder(): assert cstats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_shared_ptr_and_references(): s = m.SharedPtrRef() stats = ConstructorStats.get(m.A) @@ -196,6 +204,7 @@ def test_shared_ptr_and_references(): assert stats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_shared_ptr_from_this_and_references(): s = m.SharedFromThisRef() stats = ConstructorStats.get(m.B) @@ -242,6 +251,7 @@ def test_shared_ptr_from_this_and_references(): assert y is z +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_only_holder(): a = m.TypeWithMoveOnlyHolder.make() b = m.TypeWithMoveOnlyHolder.make_as_object() @@ -253,6 +263,7 @@ def test_move_only_holder(): assert stats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_holder_with_addressof_operator(): # this test must not throw exception from c++ a = m.TypeForHolderWithAddressOf.make() @@ -283,6 +294,7 @@ def test_holder_with_addressof_operator(): assert stats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_only_holder_with_addressof_operator(): a = m.TypeForMoveOnlyHolderWithAddressOf.make() a.print_object() @@ -301,9 +313,8 @@ def test_smart_ptr_from_default(): instance = m.HeldByDefaultHolder() with pytest.raises(RuntimeError) as excinfo: m.HeldByDefaultHolder.load_shared_ptr(instance) - assert ( - "Unable to load a custom holder type from a " - "default-holder instance" in str(excinfo.value) + assert "Unable to load a custom holder type from a default-holder instance" in str( + excinfo.value ) diff --git a/wrap/pybind11/tests/test_stl.cpp b/wrap/pybind11/tests/test_stl.cpp index 48c907ff3..9ddd951e0 100644 --- a/wrap/pybind11/tests/test_stl.cpp +++ b/wrap/pybind11/tests/test_stl.cpp @@ -16,6 +16,7 @@ # define PYBIND11_HAS_FILESYSTEM_IS_OPTIONAL #endif #include +#include #include #include @@ -59,7 +60,7 @@ struct visit_helper { } // namespace PYBIND11_NAMESPACE #endif -PYBIND11_MAKE_OPAQUE(std::vector>); +PYBIND11_MAKE_OPAQUE(std::vector>) /// Issue #528: templated constructor struct TplCtorClass { @@ -167,6 +168,14 @@ struct type_caster> } // namespace detail } // namespace PYBIND11_NAMESPACE +int pass_std_vector_int(const std::vector &v) { + int zum = 100; + for (const int i : v) { + zum += 2 * i; + } + return zum; +} + TEST_SUBMODULE(stl, m) { // test_vector m.def("cast_vector", []() { return std::vector{1}; }); @@ -193,6 +202,23 @@ TEST_SUBMODULE(stl, m) { m.def("cast_array", []() { return std::array{{1, 2}}; }); m.def("load_array", [](const std::array &a) { return a[0] == 1 && a[1] == 2; }); + struct NoDefaultCtor { + explicit constexpr NoDefaultCtor(int val) : val{val} {} + int val; + }; + + struct NoDefaultCtorArray { + explicit constexpr NoDefaultCtorArray(int i) + : arr{{NoDefaultCtor(10 + i), NoDefaultCtor(20 + i)}} {} + std::array arr; + }; + + // test_array_no_default_ctor + py::class_(m, "NoDefaultCtor").def_readonly("val", &NoDefaultCtor::val); + py::class_(m, "NoDefaultCtorArray") + .def(py::init()) + .def_readwrite("arr", &NoDefaultCtorArray::arr); + // test_valarray m.def("cast_valarray", []() { return std::valarray{1, 4, 9}; }); m.def("load_valarray", [](const std::valarray &v) { @@ -428,7 +454,57 @@ TEST_SUBMODULE(stl, m) { #ifdef PYBIND11_HAS_FILESYSTEM // test_fs_path m.attr("has_filesystem") = true; - m.def("parent_path", [](const std::filesystem::path &p) { return p.parent_path(); }); + m.def("parent_path", [](const std::filesystem::path &path) { return path.parent_path(); }); + m.def("parent_paths", [](const std::vector &paths) { + std::vector result; + result.reserve(paths.size()); + for (const auto &path : paths) { + result.push_back(path.parent_path()); + } + return result; + }); + m.def("parent_paths_list", [](const py::typing::List &paths) { + py::typing::List result; + for (auto path : paths) { + result.append(path.cast().parent_path()); + } + return result; + }); + m.def("parent_paths_nested_list", + [](const py::typing::List> &paths_lists) { + py::typing::List> result_lists; + for (auto paths : paths_lists) { + py::typing::List result; + for (auto path : paths) { + result.append(path.cast().parent_path()); + } + result_lists.append(result); + } + return result_lists; + }); + m.def("parent_paths_tuple", + [](const py::typing::Tuple &paths) { + py::typing::Tuple result + = py::make_tuple(paths[0].cast().parent_path(), + paths[1].cast().parent_path()); + return result; + }); + m.def("parent_paths_tuple_ellipsis", + [](const py::typing::Tuple &paths) { + py::typing::Tuple result(paths.size()); + for (size_t i = 0; i < paths.size(); ++i) { + result[i] = paths[i].cast().parent_path(); + } + return result; + }); + m.def("parent_paths_dict", + [](const py::typing::Dict &paths) { + py::typing::Dict result; + for (auto it : paths) { + result[it.first] = it.second.cast().parent_path(); + } + return result; + }); #endif #ifdef PYBIND11_TEST_VARIANT @@ -546,4 +622,30 @@ TEST_SUBMODULE(stl, m) { []() { return new std::vector(4513); }, // Without explicitly specifying `take_ownership`, this function leaks. py::return_value_policy::take_ownership); + + m.def("pass_std_vector_int", pass_std_vector_int); + m.def("pass_std_vector_pair_int", [](const std::vector> &v) { + int zum = 0; + for (const auto &ij : v) { + zum += ij.first * 100 + ij.second; + } + return zum; + }); + m.def("pass_std_array_int_2", [](const std::array &a) { + return pass_std_vector_int(std::vector(a.begin(), a.end())) + 1; + }); + m.def("pass_std_set_int", [](const std::set &s) { + int zum = 200; + for (const int i : s) { + zum += 3 * i; + } + return zum; + }); + m.def("pass_std_map_int", [](const std::map &m) { + int zum = 500; + for (const auto &p : m) { + zum += p.first * 1000 + p.second; + } + return zum; + }); } diff --git a/wrap/pybind11/tests/test_stl.py b/wrap/pybind11/tests/test_stl.py index 65fda54cc..f2ff727d9 100644 --- a/wrap/pybind11/tests/test_stl.py +++ b/wrap/pybind11/tests/test_stl.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest +import env # noqa: F401 from pybind11_tests import ConstructorStats, UserType from pybind11_tests import stl as m @@ -48,6 +49,13 @@ def test_array(doc): ) +def test_array_no_default_ctor(): + lst = m.NoDefaultCtorArray(3) + assert [e.val for e in lst.arr] == [13, 23] + lst.arr = m.NoDefaultCtorArray(4).arr + assert [e.val for e in lst.arr] == [14, 24] + + def test_valarray(doc): """std::valarray <-> list""" lst = m.cast_valarray() @@ -238,7 +246,7 @@ def test_reference_sensitive_optional(): @pytest.mark.skipif(not hasattr(m, "has_filesystem"), reason="no ") -def test_fs_path(): +def test_fs_path(doc): from pathlib import Path class PseudoStrPath: @@ -249,11 +257,59 @@ def test_fs_path(): def __fspath__(self): return b"foo/bar" + # Single argument assert m.parent_path(Path("foo/bar")) == Path("foo") assert m.parent_path("foo/bar") == Path("foo") assert m.parent_path(b"foo/bar") == Path("foo") assert m.parent_path(PseudoStrPath()) == Path("foo") assert m.parent_path(PseudoBytesPath()) == Path("foo") + assert ( + doc(m.parent_path) + == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> pathlib.Path" + ) + # std::vector + assert m.parent_paths(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] + assert ( + doc(m.parent_paths) + == "parent_paths(arg0: list[Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]" + ) + # py::typing::List + assert m.parent_paths_list(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] + assert ( + doc(m.parent_paths_list) + == "parent_paths_list(arg0: list[Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]" + ) + # Nested py::typing::List + assert m.parent_paths_nested_list([["foo/bar"], ["foo/baz", "foo/buzz"]]) == [ + [Path("foo")], + [Path("foo"), Path("foo")], + ] + assert ( + doc(m.parent_paths_nested_list) + == "parent_paths_nested_list(arg0: list[list[Union[os.PathLike, str, bytes]]]) -> list[list[pathlib.Path]]" + ) + # py::typing::Tuple + assert m.parent_paths_tuple(("foo/bar", "foo/baz")) == (Path("foo"), Path("foo")) + assert ( + doc(m.parent_paths_tuple) + == "parent_paths_tuple(arg0: tuple[Union[os.PathLike, str, bytes], Union[os.PathLike, str, bytes]]) -> tuple[pathlib.Path, pathlib.Path]" + ) + # py::typing::Dict + assert m.parent_paths_dict( + { + "key1": Path("foo/bar"), + "key2": "foo/baz", + "key3": b"foo/buzz", + } + ) == { + "key1": Path("foo"), + "key2": Path("foo"), + "key3": Path("foo"), + } + assert ( + doc(m.parent_paths_dict) + == "parent_paths_dict(arg0: dict[str, Union[os.PathLike, str, bytes]]) -> dict[str, pathlib.Path]" + ) @pytest.mark.skipif(not hasattr(m, "load_variant"), reason="no ") @@ -355,6 +411,7 @@ def test_function_with_string_and_vector_string_arg(): assert m.func_with_string_or_vector_string_arg_overload("A") == 3 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_stl_ownership(): cstats = ConstructorStats.get(m.Placeholder) assert cstats.alive() == 0 @@ -381,3 +438,129 @@ def test_return_vector_bool_raw_ptr(): v = m.return_vector_bool_raw_ptr() assert isinstance(v, list) assert len(v) == 4513 + + +@pytest.mark.parametrize( + ("fn", "offset"), [(m.pass_std_vector_int, 0), (m.pass_std_array_int_2, 1)] +) +def test_pass_std_vector_int(fn, offset): + assert fn([7, 13]) == 140 + offset + assert fn({6, 2}) == 116 + offset + assert fn({"x": 8, "y": 11}.values()) == 138 + offset + assert fn({3: None, 9: None}.keys()) == 124 + offset + assert fn(i for i in [4, 17]) == 142 + offset + assert fn(map(lambda i: i * 3, [8, 7])) == 190 + offset # noqa: C417 + with pytest.raises(TypeError): + fn({"x": 0, "y": 1}) + with pytest.raises(TypeError): + fn({}) + + +def test_pass_std_vector_pair_int(): + fn = m.pass_std_vector_pair_int + assert fn({1: 2, 3: 4}.items()) == 406 + assert fn(zip([5, 17], [13, 9])) == 2222 + + +def test_list_caster_fully_consumes_generator_object(): + def gen_invalid(): + yield from [1, 2.0, 3] + + gen_obj = gen_invalid() + with pytest.raises(TypeError): + m.pass_std_vector_int(gen_obj) + assert not tuple(gen_obj) + + +def test_pass_std_set_int(): + fn = m.pass_std_set_int + assert fn({3, 15}) == 254 + assert fn({5: None, 12: None}.keys()) == 251 + with pytest.raises(TypeError): + fn([]) + with pytest.raises(TypeError): + fn({}) + with pytest.raises(TypeError): + fn({}.values()) + with pytest.raises(TypeError): + fn(i for i in []) + + +def test_set_caster_dict_keys_failure(): + dict_keys = {1: None, 2.0: None, 3: None}.keys() + # The asserts does not really exercise anything in pybind11, but if one of + # them fails in some future version of Python, the set_caster load + # implementation may need to be revisited. + assert tuple(dict_keys) == (1, 2.0, 3) + assert tuple(dict_keys) == (1, 2.0, 3) + with pytest.raises(TypeError): + m.pass_std_set_int(dict_keys) + assert tuple(dict_keys) == (1, 2.0, 3) + + +class FakePyMappingMissingItems: + def __getitem__(self, _): + raise RuntimeError("Not expected to be called.") + + +class FakePyMappingWithItems(FakePyMappingMissingItems): + def items(self): + return ((1, 3), (2, 4)) + + +class FakePyMappingBadItems(FakePyMappingMissingItems): + def items(self): + return ((1, 2), (3, "x")) + + +class FakePyMappingItemsNotCallable(FakePyMappingMissingItems): + @property + def items(self): + return ((1, 2), (3, 4)) + + +class FakePyMappingItemsWithArg(FakePyMappingMissingItems): + def items(self, _): + return ((1, 2), (3, 4)) + + +class FakePyMappingGenObj(FakePyMappingMissingItems): + def __init__(self, gen_obj): + super().__init__() + self.gen_obj = gen_obj + + def items(self): + yield from self.gen_obj + + +def test_pass_std_map_int(): + fn = m.pass_std_map_int + assert fn({1: 2, 3: 4}) == 4506 + with pytest.raises(TypeError): + fn([]) + assert fn(FakePyMappingWithItems()) == 3507 + with pytest.raises(TypeError): + fn(FakePyMappingMissingItems()) + with pytest.raises(TypeError): + fn(FakePyMappingBadItems()) + with pytest.raises(TypeError): + fn(FakePyMappingItemsNotCallable()) + with pytest.raises(TypeError): + fn(FakePyMappingItemsWithArg()) + + +@pytest.mark.parametrize( + ("items", "expected_exception"), + [ + (((1, 2), (3, "x"), (4, 5)), TypeError), + (((1, 2), (3, 4, 5), (6, 7)), ValueError), + ], +) +def test_map_caster_fully_consumes_generator_object(items, expected_exception): + def gen_invalid(): + yield from items + + gen_obj = gen_invalid() + with pytest.raises(expected_exception): + m.pass_std_map_int(FakePyMappingGenObj(gen_obj)) + assert not tuple(gen_obj) diff --git a/wrap/pybind11/tests/test_stl_binders.py b/wrap/pybind11/tests/test_stl_binders.py index 09e784e2e..9856ba462 100644 --- a/wrap/pybind11/tests/test_stl_binders.py +++ b/wrap/pybind11/tests/test_stl_binders.py @@ -302,6 +302,25 @@ def test_map_delitem(): assert list(mm) == ["b"] assert list(mm.items()) == [("b", 2.5)] + with pytest.raises(KeyError) as excinfo: + mm["a_long_key"] + assert "a_long_key" in str(excinfo.value) + + with pytest.raises(KeyError) as excinfo: + del mm["a_long_key"] + assert "a_long_key" in str(excinfo.value) + + cut_length = 100 + k_very_long = "ab" * cut_length + "xyz" + with pytest.raises(KeyError) as excinfo: + mm[k_very_long] + assert k_very_long in str(excinfo.value) + k_very_long += "@" + with pytest.raises(KeyError) as excinfo: + mm[k_very_long] + k_repr = k_very_long[:cut_length] + "✄✄✄" + k_very_long[-cut_length:] + assert k_repr in str(excinfo.value) + um = m.UnorderedMapStringDouble() um["ua"] = 1.1 um["ub"] = 2.6 diff --git a/wrap/pybind11/tests/test_tagbased_polymorphic.cpp b/wrap/pybind11/tests/test_tagbased_polymorphic.cpp index 12ba6532f..13e5ed319 100644 --- a/wrap/pybind11/tests/test_tagbased_polymorphic.cpp +++ b/wrap/pybind11/tests/test_tagbased_polymorphic.cpp @@ -74,6 +74,7 @@ std::vector> create_zoo() { // simulate some new type of Dog that the Python bindings // haven't been updated for; it should still be considered // a Dog, not just an Animal. + // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) ret.emplace_back(new Dog("Ginger", Dog::Kind(150))); ret.emplace_back(new Chihuahua("Hertzl")); @@ -144,4 +145,4 @@ TEST_SUBMODULE(tagbased_polymorphic, m) { .def(py::init()) .def("purr", &Panther::purr); m.def("create_zoo", &create_zoo); -}; +} diff --git a/wrap/pybind11/tests/test_thread.cpp b/wrap/pybind11/tests/test_thread.cpp index e727109d7..eabf39afa 100644 --- a/wrap/pybind11/tests/test_thread.cpp +++ b/wrap/pybind11/tests/test_thread.cpp @@ -28,6 +28,9 @@ struct IntStruct { int value; }; +struct EmptyStruct {}; +EmptyStruct SharedInstance; + } // namespace TEST_SUBMODULE(thread, m) { @@ -61,6 +64,9 @@ TEST_SUBMODULE(thread, m) { }, py::call_guard()); + py::class_(m, "EmptyStruct") + .def_readonly_static("SharedInstance", &SharedInstance); + // NOTE: std::string_view also uses loader_life_support to ensure that // the string contents remain alive, but that's a C++ 17 feature. } diff --git a/wrap/pybind11/tests/test_thread.py b/wrap/pybind11/tests/test_thread.py index 4541a305e..e9d7bafb2 100644 --- a/wrap/pybind11/tests/test_thread.py +++ b/wrap/pybind11/tests/test_thread.py @@ -1,7 +1,10 @@ from __future__ import annotations +import sys import threading +import pytest + from pybind11_tests import thread as m @@ -24,6 +27,7 @@ class Thread(threading.Thread): raise self.e +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_implicit_conversion(): a = Thread(m.test) b = Thread(m.test) @@ -34,6 +38,7 @@ def test_implicit_conversion(): x.join() +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_implicit_conversion_no_gil(): a = Thread(m.test_no_gil) b = Thread(m.test_no_gil) @@ -42,3 +47,22 @@ def test_implicit_conversion_no_gil(): x.start() for x in [c, b, a]: x.join() + + +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") +def test_bind_shared_instance(): + nb_threads = 4 + b = threading.Barrier(nb_threads) + + def access_shared_instance(): + b.wait() + for _ in range(1000): + m.EmptyStruct.SharedInstance # noqa: B018 + + threads = [ + threading.Thread(target=access_shared_instance) for _ in range(nb_threads) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() diff --git a/wrap/pybind11/tests/test_type_caster_pyobject_ptr.cpp b/wrap/pybind11/tests/test_type_caster_pyobject_ptr.cpp index a45c08b64..758c4ee09 100644 --- a/wrap/pybind11/tests/test_type_caster_pyobject_ptr.cpp +++ b/wrap/pybind11/tests/test_type_caster_pyobject_ptr.cpp @@ -37,7 +37,8 @@ struct WithPyObjectPtrReturnTrampoline : WithPyObjectPtrReturn { std::string call_return_pyobject_ptr(const WithPyObjectPtrReturn *base_class_ptr) { PyObject *returned_obj = base_class_ptr->return_pyobject_ptr(); -#if !defined(PYPY_VERSION) // It is not worth the trouble doing something special for PyPy. +// It is not worth the trouble doing something special for PyPy/GraalPy +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) if (Py_REFCNT(returned_obj) != 1) { py::pybind11_fail(__FILE__ ":" PYBIND11_TOSTRING(__LINE__)); } diff --git a/wrap/pybind11/tests/test_type_caster_std_function_specializations.cpp b/wrap/pybind11/tests/test_type_caster_std_function_specializations.cpp new file mode 100644 index 000000000..89213ddb1 --- /dev/null +++ b/wrap/pybind11/tests/test_type_caster_std_function_specializations.cpp @@ -0,0 +1,46 @@ +#include +#include + +#include "pybind11_tests.h" + +namespace py = pybind11; + +namespace { + +struct SpecialReturn { + int value = 99; +}; + +} // namespace + +namespace pybind11 { +namespace detail { +namespace type_caster_std_function_specializations { + +template +struct func_wrapper : func_wrapper_base { + using func_wrapper_base::func_wrapper_base; + SpecialReturn operator()(Args... args) const { + gil_scoped_acquire acq; + SpecialReturn result; + try { + result = hfunc.f(std::forward(args)...).template cast(); + } catch (error_already_set &) { + result.value += 1; + } + result.value += 100; + return result; + } +}; + +} // namespace type_caster_std_function_specializations +} // namespace detail +} // namespace pybind11 + +TEST_SUBMODULE(type_caster_std_function_specializations, m) { + py::class_(m, "SpecialReturn") + .def(py::init<>()) + .def_readwrite("value", &SpecialReturn::value); + m.def("call_callback_with_special_return", + [](const std::function &func) { return func(); }); +} diff --git a/wrap/pybind11/tests/test_type_caster_std_function_specializations.py b/wrap/pybind11/tests/test_type_caster_std_function_specializations.py new file mode 100644 index 000000000..9e45d4f59 --- /dev/null +++ b/wrap/pybind11/tests/test_type_caster_std_function_specializations.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pybind11_tests import type_caster_std_function_specializations as m + + +def test_callback_with_special_return(): + def return_special(): + return m.SpecialReturn() + + def raise_exception(): + raise ValueError("called raise_exception.") + + assert return_special().value == 99 + assert m.call_callback_with_special_return(return_special).value == 199 + assert m.call_callback_with_special_return(raise_exception).value == 200 diff --git a/wrap/pybind11/tests/test_unnamed_namespace_a.cpp b/wrap/pybind11/tests/test_unnamed_namespace_a.cpp index 2152e64bd..26e9cb751 100644 --- a/wrap/pybind11/tests/test_unnamed_namespace_a.cpp +++ b/wrap/pybind11/tests/test_unnamed_namespace_a.cpp @@ -10,7 +10,6 @@ TEST_SUBMODULE(unnamed_namespace_a, m) { } else { m.attr("unnamed_namespace_a_any_struct") = py::none(); } - m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION; m.attr("defined_WIN32_or__WIN32") = #if defined(WIN32) || defined(_WIN32) true; diff --git a/wrap/pybind11/tests/test_unnamed_namespace_a.py b/wrap/pybind11/tests/test_unnamed_namespace_a.py index 0fa1fa323..fabf1312a 100644 --- a/wrap/pybind11/tests/test_unnamed_namespace_a.py +++ b/wrap/pybind11/tests/test_unnamed_namespace_a.py @@ -5,13 +5,7 @@ import pytest from pybind11_tests import unnamed_namespace_a as m from pybind11_tests import unnamed_namespace_b as mb -XFAIL_CONDITION = ( - "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))" - " or " - "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32" - " and " - "(m.defined___clang__ or m.defined__LIBCPP_VERSION))" -) +XFAIL_CONDITION = "not m.defined_WIN32_or__WIN32 and (m.defined___clang__ or m.defined__LIBCPP_VERSION)" XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319" diff --git a/wrap/pybind11/tests/test_virtual_functions.cpp b/wrap/pybind11/tests/test_virtual_functions.cpp index 93b136ad3..a6164eb81 100644 --- a/wrap/pybind11/tests/test_virtual_functions.cpp +++ b/wrap/pybind11/tests/test_virtual_functions.cpp @@ -589,4 +589,4 @@ void initialize_inherited_virtuals(py::module_ &m) { // Fix issue #1454 (crash when acquiring/releasing GIL on another thread in Python 2.7) m.def("test_gil", &test_gil); m.def("test_gil_from_thread", &test_gil_from_thread); -}; +} diff --git a/wrap/pybind11/tests/test_virtual_functions.py b/wrap/pybind11/tests/test_virtual_functions.py index 08acaa190..617c87b8e 100644 --- a/wrap/pybind11/tests/test_virtual_functions.py +++ b/wrap/pybind11/tests/test_virtual_functions.py @@ -1,8 +1,10 @@ from __future__ import annotations +import sys + import pytest -import env # noqa: F401 +import env m = pytest.importorskip("pybind11_tests.virtual_functions") from pybind11_tests import ConstructorStats # noqa: E402 @@ -80,6 +82,9 @@ def test_override(capture, msg): """ ) + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.ExampleVirt) assert cstats.alive() == 3 del ex12, ex12p, ex12p2 @@ -89,6 +94,7 @@ def test_override(capture, msg): assert cstats.move_constructions >= 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alias_delay_initialization1(capture): """`A` only initializes its trampoline class when we inherit from it @@ -128,6 +134,7 @@ def test_alias_delay_initialization1(capture): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alias_delay_initialization2(capture): """`A2`, unlike the above, is configured to always initialize the alias @@ -186,7 +193,7 @@ def test_alias_delay_initialization2(capture): # PyPy: Reference count > 1 causes call with noncopyable instance # to fail in ncv1.print_nc() -@pytest.mark.xfail("env.PYPY") +@pytest.mark.xfail("env.PYPY or env.GRAALPY") @pytest.mark.skipif( not hasattr(m, "NCVirt"), reason="NCVirt does not work on Intel/PGI/NVCC compilers" ) @@ -435,6 +442,7 @@ def test_inherited_virtuals(): assert obj.say_everything() == "BT -7" +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") def test_issue_1454(): # Fix issue #1454 (crash when acquiring/releasing GIL on another thread in Python 2.7) m.test_gil() diff --git a/wrap/pybind11/tests/test_warnings.cpp b/wrap/pybind11/tests/test_warnings.cpp new file mode 100644 index 000000000..e76f21249 --- /dev/null +++ b/wrap/pybind11/tests/test_warnings.cpp @@ -0,0 +1,46 @@ +/* + tests/test_warnings.cpp -- usage of warnings::warn() and warnings categories. + + Copyright (c) 2024 Jan Iwaszkiewicz + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include + +#include "pybind11_tests.h" + +#include + +TEST_SUBMODULE(warnings_, m) { + + // Test warning mechanism base + m.def("warn_and_return_value", []() { + std::string message = "This is simple warning"; + py::warnings::warn(message.c_str(), PyExc_Warning); + return 21; + }); + + m.def("warn_with_default_category", []() { py::warnings::warn("This is RuntimeWarning"); }); + + m.def("warn_with_different_category", + []() { py::warnings::warn("This is FutureWarning", PyExc_FutureWarning); }); + + m.def("warn_with_invalid_category", + []() { py::warnings::warn("Invalid category", PyExc_Exception); }); + + // Test custom warnings + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store ex_storage; + ex_storage.call_once_and_store_result([&]() { + return py::warnings::new_warning_type(m, "CustomWarning", PyExc_DeprecationWarning); + }); + + m.def("warn_with_custom_type", []() { + py::warnings::warn("This is CustomWarning", ex_storage.get_stored()); + return 37; + }); + + m.def("register_duplicate_warning", + [m]() { py::warnings::new_warning_type(m, "CustomWarning", PyExc_RuntimeWarning); }); +} diff --git a/wrap/pybind11/tests/test_warnings.py b/wrap/pybind11/tests/test_warnings.py new file mode 100644 index 000000000..4313432c3 --- /dev/null +++ b/wrap/pybind11/tests/test_warnings.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import warnings + +import pytest + +import pybind11_tests # noqa: F401 +from pybind11_tests import warnings_ as m + + +@pytest.mark.parametrize( + ("expected_category", "expected_message", "expected_value", "module_function"), + [ + (Warning, "This is simple warning", 21, m.warn_and_return_value), + (RuntimeWarning, "This is RuntimeWarning", None, m.warn_with_default_category), + (FutureWarning, "This is FutureWarning", None, m.warn_with_different_category), + ], +) +def test_warning_simple( + expected_category, expected_message, expected_value, module_function +): + with pytest.warns(Warning) as excinfo: + value = module_function() + + assert issubclass(excinfo[0].category, expected_category) + assert str(excinfo[0].message) == expected_message + assert value == expected_value + + +def test_warning_wrong_subclass_fail(): + with pytest.raises(Exception) as excinfo: + m.warn_with_invalid_category() + + assert issubclass(excinfo.type, RuntimeError) + assert ( + str(excinfo.value) + == "pybind11::warnings::warn(): cannot raise warning, category must be a subclass of PyExc_Warning!" + ) + + +def test_warning_double_register_fail(): + with pytest.raises(Exception) as excinfo: + m.register_duplicate_warning() + + assert issubclass(excinfo.type, RuntimeError) + assert ( + str(excinfo.value) + == 'pybind11::warnings::new_warning_type(): an attribute with name "CustomWarning" exists already.' + ) + + +def test_warning_register(): + assert m.CustomWarning is not None + + with pytest.warns(m.CustomWarning) as excinfo: + warnings.warn("This is warning from Python!", m.CustomWarning, stacklevel=1) + + assert issubclass(excinfo[0].category, DeprecationWarning) + assert str(excinfo[0].message) == "This is warning from Python!" + + +def test_warning_custom(): + with pytest.warns(m.CustomWarning) as excinfo: + value = m.warn_with_custom_type() + + assert issubclass(excinfo[0].category, DeprecationWarning) + assert str(excinfo[0].message) == "This is CustomWarning" + assert value == 37 diff --git a/wrap/pybind11/tools/FindPythonLibsNew.cmake b/wrap/pybind11/tools/FindPythonLibsNew.cmake index 283b4e298..7351bcaa6 100644 --- a/wrap/pybind11/tools/FindPythonLibsNew.cmake +++ b/wrap/pybind11/tools/FindPythonLibsNew.cmake @@ -92,7 +92,7 @@ endif() # Use the Python interpreter to find the libs. if(NOT PythonLibsNew_FIND_VERSION) - set(PythonLibsNew_FIND_VERSION "3.7") + set(PythonLibsNew_FIND_VERSION "3.8") endif() if(NOT CMAKE_VERSION VERSION_LESS "3.27") @@ -200,6 +200,16 @@ if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) endif() if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_EXTENSION) get_filename_component(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) + if((NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "") AND (NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" + STREQUAL "${PYTHON_MODULE_EXTENSION}")) + message( + AUTHOR_WARNING, + "SETUPTOOLS_EXT_SUFFIX is set to \"$ENV{SETUPTOOLS_EXT_SUFFIX}\", " + "but the auto-calculated Python extension suffix is \"${PYTHON_MODULE_EXTENSION}\". " + "This may cause problems when importing the Python extensions. " + "If you are using cross-compiling Python, you may need to " + "set PYTHON_MODULE_EXTENSION manually.") + endif() endif() # Make sure the Python has the same pointer-size as the chosen compiler diff --git a/wrap/pybind11/tools/make_changelog.py b/wrap/pybind11/tools/make_changelog.py index daa966f20..b499d06ba 100755 --- a/wrap/pybind11/tools/make_changelog.py +++ b/wrap/pybind11/tools/make_changelog.py @@ -59,9 +59,9 @@ for issue in issues: msg += "." msg += f"\n `#{issue.number} <{issue.html_url}>`_" - for cat in cats: + for cat, cat_list in cats.items(): if issue.title.lower().startswith(f"{cat}:"): - cats[cat].append(msg) + cat_list.append(msg) break else: cats["unknown"].append(msg) diff --git a/wrap/pybind11/tools/pybind11Common.cmake b/wrap/pybind11/tools/pybind11Common.cmake index 8467b45d2..450a14156 100644 --- a/wrap/pybind11/tools/pybind11Common.cmake +++ b/wrap/pybind11/tools/pybind11Common.cmake @@ -2,7 +2,7 @@ Adds the following targets:: - pybind11::pybind11 - link to headers and pybind11 + pybind11::pybind11 - link to Python headers and pybind11::headers pybind11::module - Adds module links pybind11::embed - Adds embed links pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) @@ -18,11 +18,7 @@ Adds the following functions:: #]======================================================] -# CMake 3.10 has an include_guard command, but we can't use that yet -# include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::pybind11) - return() -endif() +include_guard(GLOBAL) # If we are in subdirectory mode, all IMPORTED targets must be GLOBAL. If we # are in CONFIG mode, they should be "normal" targets instead. @@ -75,26 +71,36 @@ set_property( APPEND PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11) +# -------------- emscripten requires exceptions enabled ------------- +# _pybind11_no_exceptions is a private mechanism to disable this addition. +# Please open an issue if you need to use it; it will be removed if no one +# needs it. +if(CMAKE_SYSTEM_NAME MATCHES Emscripten AND NOT _pybind11_no_exceptions) + if(is_config) + set(_tmp_config_target pybind11::pybind11_headers) + else() + set(_tmp_config_target pybind11_headers) + endif() + + set_property( + TARGET ${_tmp_config_target} + APPEND + PROPERTY INTERFACE_LINK_OPTIONS -fexceptions) + set_property( + TARGET ${_tmp_config_target} + APPEND + PROPERTY INTERFACE_COMPILE_OPTIONS -fexceptions) + unset(_tmp_config_target) +endif() + # --------------------------- link helper --------------------------- add_library(pybind11::python_link_helper IMPORTED INTERFACE ${optional_global}) -if(CMAKE_VERSION VERSION_LESS 3.13) - # In CMake 3.11+, you can set INTERFACE properties via the normal methods, and - # this would be simpler. - set_property( - TARGET pybind11::python_link_helper - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "$<$:-undefined dynamic_lookup>") -else() - # link_options was added in 3.13+ - # This is safer, because you are ensured the deduplication pass in CMake will not consider - # these separate and remove one but not the other. - set_property( - TARGET pybind11::python_link_helper - APPEND - PROPERTY INTERFACE_LINK_OPTIONS "$<$:LINKER:-undefined,dynamic_lookup>") -endif() +set_property( + TARGET pybind11::python_link_helper + APPEND + PROPERTY INTERFACE_LINK_OPTIONS "$<$:LINKER:-undefined,dynamic_lookup>") # ------------------------ Windows extras ------------------------- @@ -110,22 +116,14 @@ if(MSVC) # That's also clang-cl # /MP enables multithreaded builds (relevant when there are many files) for MSVC if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # no Clang no Intel - if(CMAKE_VERSION VERSION_LESS 3.11) - set_property( - TARGET pybind11::windows_extras - APPEND - PROPERTY INTERFACE_COMPILE_OPTIONS $<$>:/MP>) - else() - # Only set these options for C++ files. This is important so that, for - # instance, projects that include other types of source files like CUDA - # .cu files don't get these options propagated to nvcc since that would - # cause the build to fail. - set_property( - TARGET pybind11::windows_extras - APPEND - PROPERTY INTERFACE_COMPILE_OPTIONS - $<$>:$<$:/MP>>) - endif() + # Only set these options for C++ files. This is important so that, for + # instance, projects that include other types of source files like CUDA + # .cu files don't get these options propagated to nvcc since that would + # cause the build to fail. + set_property( + TARGET pybind11::windows_extras + APPEND + PROPERTY INTERFACE_COMPILE_OPTIONS $<$>:$<$:/MP>>) endif() endif() @@ -329,7 +327,7 @@ function(_pybind11_generate_lto target prefer_thin_lto) if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") # Do nothing - elseif(CMAKE_SYSTEM_PROCESSOR MATCHES emscripten) + elseif(CMAKE_SYSTEM_NAME MATCHES Emscripten) # This compile is very costly when cross-compiling, so set this without checking set(PYBIND11_LTO_CXX_FLAGS "-flto${thin}${cxx_append}") set(PYBIND11_LTO_LINKER_FLAGS "-flto${thin}${linker_append}") @@ -371,11 +369,7 @@ function(_pybind11_generate_lto target prefer_thin_lto) set(is_debug "$,$>") set(not_debug "$") set(cxx_lang "$") - if(MSVC AND CMAKE_VERSION VERSION_LESS 3.11) - set(genex "${not_debug}") - else() - set(genex "$") - endif() + set(genex "$") set_property( TARGET ${target} APPEND @@ -390,17 +384,10 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() if(PYBIND11_LTO_LINKER_FLAGS) - if(CMAKE_VERSION VERSION_LESS 3.11) - set_property( - TARGET ${target} - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "$<${not_debug}:${PYBIND11_LTO_LINKER_FLAGS}>") - else() - set_property( - TARGET ${target} - APPEND - PROPERTY INTERFACE_LINK_OPTIONS "$<${not_debug}:${PYBIND11_LTO_LINKER_FLAGS}>") - endif() + set_property( + TARGET ${target} + APPEND + PROPERTY INTERFACE_LINK_OPTIONS "$<${not_debug}:${PYBIND11_LTO_LINKER_FLAGS}>") endif() endfunction() diff --git a/wrap/pybind11/tools/pybind11Config.cmake.in b/wrap/pybind11/tools/pybind11Config.cmake.in index 304f1d907..2d9fa94f6 100644 --- a/wrap/pybind11/tools/pybind11Config.cmake.in +++ b/wrap/pybind11/tools/pybind11Config.cmake.in @@ -84,7 +84,7 @@ you can either use the basic targets, or use the FindPython tools: # Python method: Python_add_library(MyModule2 src2.cpp) - target_link_libraries(MyModule2 pybind11::headers) + target_link_libraries(MyModule2 PUBLIC pybind11::headers) set_target_properties(MyModule2 PROPERTIES INTERPROCEDURAL_OPTIMIZATION ON CXX_VISIBILITY_PRESET ON diff --git a/wrap/pybind11/tools/pybind11GuessPythonExtSuffix.cmake b/wrap/pybind11/tools/pybind11GuessPythonExtSuffix.cmake index c5fb3b42c..b550f3935 100644 --- a/wrap/pybind11/tools/pybind11GuessPythonExtSuffix.cmake +++ b/wrap/pybind11/tools/pybind11GuessPythonExtSuffix.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.15...3.30) function(pybind11_guess_python_module_extension python) diff --git a/wrap/pybind11/tools/pybind11NewTools.cmake b/wrap/pybind11/tools/pybind11NewTools.cmake index a8b0800bb..087784c22 100644 --- a/wrap/pybind11/tools/pybind11NewTools.cmake +++ b/wrap/pybind11/tools/pybind11NewTools.cmake @@ -5,10 +5,6 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -if(CMAKE_VERSION VERSION_LESS 3.12) - message(FATAL_ERROR "You cannot use the new FindPython module with CMake < 3.12") -endif() - include_guard(DIRECTORY) get_property( @@ -56,7 +52,7 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) endif() find_package( - Python 3.7 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component} + Python 3.8 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component} ${_pybind11_quiet} ${_pybind11_global_keyword}) # If we are in submodule mode, export the Python targets to global targets. @@ -175,6 +171,16 @@ if(NOT _PYBIND11_CROSSCOMPILING) set(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXTENSION}" CACHE INTERNAL "") + if((NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "") + AND (NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "${PYTHON_MODULE_EXTENSION}")) + message( + AUTHOR_WARNING, + "SETUPTOOLS_EXT_SUFFIX is set to \"$ENV{SETUPTOOLS_EXT_SUFFIX}\", " + "but the auto-calculated Python extension suffix is \"${PYTHON_MODULE_EXTENSION}\". " + "This may cause problems when importing the Python extensions. " + "If you are using cross-compiling Python, you may need to " + "set PYTHON_MODULE_EXTENSION manually.") + endif() endif() endif() else() @@ -236,7 +242,6 @@ if(TARGET ${_Python}::Python) PROPERTY INTERFACE_LINK_LIBRARIES ${_Python}::Python) endif() -# CMake 3.15+ has this if(TARGET ${_Python}::Module) set_property( TARGET pybind11::module @@ -279,10 +284,6 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() - if(MSVC) - target_link_libraries(${target_name} PRIVATE pybind11::windows_extras) - endif() - # -fvisibility=hidden is required to allow multiple modules compiled against # different pybind versions to work properly, and for some features (e.g. # py::module_local). We force it on everything inside the `pybind11` @@ -316,7 +317,7 @@ function(pybind11_add_module target_name) if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 # Use case-insensitive comparison to match the result of $ string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO|NONE) # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/wrap/pybind11/tools/pybind11Tools.cmake b/wrap/pybind11/tools/pybind11Tools.cmake index bed5e0803..b4df3df5b 100644 --- a/wrap/pybind11/tools/pybind11Tools.cmake +++ b/wrap/pybind11/tools/pybind11Tools.cmake @@ -5,13 +5,7 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -# include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::python_headers) - return() -endif() - -# Built-in in CMake 3.5+ -include(CMakeParseArguments) +include_guard(GLOBAL) if(pybind11_FIND_QUIETLY) set(_pybind11_quiet QUIET) @@ -43,7 +37,7 @@ endif() # A user can set versions manually too set(Python_ADDITIONAL_VERSIONS - "3.12;3.11;3.10;3.9;3.8;3.7" + "3.12;3.11;3.10;3.9;3.8" CACHE INTERNAL "") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") @@ -116,36 +110,19 @@ if(PYTHON_IS_DEBUG) PROPERTY INTERFACE_COMPILE_DEFINITIONS Py_DEBUG) endif() -# The <3.11 code here does not support release/debug builds at the same time, like on vcpkg -if(CMAKE_VERSION VERSION_LESS 3.11) - set_property( - TARGET pybind11::module - APPEND - PROPERTY - INTERFACE_LINK_LIBRARIES - pybind11::python_link_helper - "$<$,$>:$>" - ) +# The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside +# of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are +# target_link_library keywords rather than real libraries. +add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE) +target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES}) +target_link_libraries( + pybind11::module + INTERFACE + pybind11::python_link_helper + "$<$,$>:pybind11::_ClassicPythonLibraries>") - set_property( - TARGET pybind11::embed - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11 $) -else() - # The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside - # of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are - # target_link_library keywords rather than real libraries. - add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE) - target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES}) - target_link_libraries( - pybind11::module - INTERFACE - pybind11::python_link_helper - "$<$,$>:pybind11::_ClassicPythonLibraries>") - - target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11 - pybind11::_ClassicPythonLibraries) -endif() +target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11 + pybind11::_ClassicPythonLibraries) function(pybind11_extension name) # The prefix and extension are provided by FindPythonLibsNew.cmake @@ -219,7 +196,7 @@ function(pybind11_add_module target_name) if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 # Use case-insensitive comparison to match the result of $ string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO|NONE) pybind11_strip(${target_name}) endif() endif() diff --git a/wrap/pybind11/tools/setup_global.py.in b/wrap/pybind11/tools/setup_global.py.in index 885ac5c72..99b8a2b29 100644 --- a/wrap/pybind11/tools/setup_global.py.in +++ b/wrap/pybind11/tools/setup_global.py.in @@ -26,12 +26,14 @@ class InstallHeadersNested(install_headers): main_headers = glob.glob("pybind11/include/pybind11/*.h") +conduit_headers = sum([glob.glob(f"pybind11/include/pybind11/conduit/*.{ext}") + for ext in ("h", "txt")], []) detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") eigen_headers = glob.glob("pybind11/include/pybind11/eigen/*.h") stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc") -headers = main_headers + detail_headers + stl_headers + eigen_headers +headers = main_headers + conduit_headers + detail_headers + eigen_headers + stl_headers cmdclass = {"install_headers": InstallHeadersNested} $extra_cmd @@ -55,6 +57,7 @@ setup( (base + "share/cmake/pybind11", cmake_files), (base + "share/pkgconfig", pkgconfig_files), (base + "include/pybind11", main_headers), + (base + "include/pybind11/conduit", conduit_headers), (base + "include/pybind11/detail", detail_headers), (base + "include/pybind11/eigen", eigen_headers), (base + "include/pybind11/stl", stl_headers), diff --git a/wrap/pybind11/tools/setup_main.py.in b/wrap/pybind11/tools/setup_main.py.in index 6358cc7b9..e04dc8204 100644 --- a/wrap/pybind11/tools/setup_main.py.in +++ b/wrap/pybind11/tools/setup_main.py.in @@ -14,15 +14,18 @@ setup( packages=[ "pybind11", "pybind11.include.pybind11", + "pybind11.include.pybind11.conduit", "pybind11.include.pybind11.detail", "pybind11.include.pybind11.eigen", "pybind11.include.pybind11.stl", + "pybind11.share", "pybind11.share.cmake.pybind11", "pybind11.share.pkgconfig", ], package_data={ "pybind11": ["py.typed"], "pybind11.include.pybind11": ["*.h"], + "pybind11.include.pybind11.conduit": ["*.h", "*.txt"], "pybind11.include.pybind11.detail": ["*.h"], "pybind11.include.pybind11.eigen": ["*.h"], "pybind11.include.pybind11.stl": ["*.h"], @@ -38,7 +41,10 @@ setup( ], "pipx.run": [ "pybind11 = pybind11.__main__:main", - ] + ], + "pkg_config": [ + "pybind11 = pybind11.share.pkgconfig", + ], }, cmdclass=cmdclass ) diff --git a/wrap/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake b/wrap/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake index 0de2c0169..ac90e039b 100644 --- a/wrap/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake +++ b/wrap/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.15...3.30) # Tests for pybind11_guess_python_module_extension # Run using `cmake -P tools/test-pybind11GuessPythonExtSuffix.cmake` diff --git a/wrap/scripts/pybind_wrap.py b/wrap/scripts/pybind_wrap.py index 8785ed7b5..41f39b406 100644 --- a/wrap/scripts/pybind_wrap.py +++ b/wrap/scripts/pybind_wrap.py @@ -64,6 +64,10 @@ def main(): arg_parser.add_argument("--is_submodule", default=False, action="store_true") + arg_parser.add_argument("--xml_source", + type=str, + default="", + help="The path to the Doxygen-generated XML documentation") args = arg_parser.parse_args() top_module_namespaces = args.top_module_namespaces.split("::") @@ -79,6 +83,7 @@ def main(): top_module_namespaces=top_module_namespaces, ignore_classes=args.ignore, module_template=template_content, + xml_source=args.xml_source, ) if args.is_submodule: