Squashed 'wrap/' changes from 38a1bfa01..4cca84a07

4cca84a07 Merge pull request #170 from p-zach/docstrings
dd56512e3 Refactor and clean up extract_docstring
bf9e0c151 Merge pull request #171 from borglab/fix/ci
d25495646 Formatting and cleaning up
7bf44b52b test more recent versions of python
c0529799c update macos runner to macos-14
f02d577a9 Rename CMake option for clarity
ef9b37db7 Create Doxygen XML parser for Python docstrings
da6d69abd Add Cmake option for Python docstring generation
bb03b9681 Merge pull request #169 from borglab/pybind11-upgrade
a26cb2066 upgrade Pybind11 to 2.13.6

git-subtree-dir: wrap
git-subtree-split: 4cca84a07e01fb0c9280148af2bcf8031f88fb7d

.github/workflows/linux-ci.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Checkout

.github/workflows/macos-ci.yml

@@ -5,12 +5,12 @@ on: [pull_request]
 jobs:
   build:
     name: Tests for 🐍 ${{ matrix.python-version }}
-    runs-on: macos-12
+    runs-on: macos-14
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Checkout

CMakeLists.txt

@@ -20,6 +20,8 @@ set(INSTALL_LIB_DIR lib/${PROJECT_NAME})
 set(INSTALL_BIN_DIR bin/${PROJECT_NAME})
 set(INSTALL_INCLUDE_DIR include/${PROJECT_NAME})
 
+option(GTWRAP_ADD_DOCSTRINGS "Whether to add docstrings to the Python bindings from Doxygen-generated XML located at {project_root}/xml" OFF)
+
 # ##############################################################################
 # Package Configuration
 

cmake/PybindWrap.cmake

@@ -13,6 +13,14 @@ gtwrap_get_python_version(${WRAP_PYTHON_VERSION})
 message(STATUS "Setting Python version for wrapper")
 set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION})
 
+if(GTWRAP_ADD_DOCSTRINGS)
+  set(GTWRAP_PYTHON_DOCS_SOURCE "${CMAKE_SOURCE_DIR}/xml")
+  message(STATUS "Python docstring generation is on. XML source: '${GTWRAP_PYTHON_DOCS_SOURCE}'")
+else()
+  message(STATUS "Python docstring generation is off.")
+  set(GTWRAP_PYTHON_DOCS_SOURCE "")
+endif()
+
 # User-friendly Pybind11 wrapping and installing function. Builds a Pybind11
 # module from the provided interface_headers. For example, for the interface
 # header gtsam.h, this will build the wrap module 'gtsam_py.cc'.
@@ -82,6 +90,7 @@ function(
           --out "${cpp_file}"  --module_name ${module_name}
           --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes}
           --template ${module_template} --is_submodule ${_WRAP_BOOST_ARG}
+          --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}"
       DEPENDS "${interface_file}" ${module_template} "${module_name}/specializations/${interface}.h" "${module_name}/preamble/${interface}.h"
       VERBATIM)
 
@@ -100,6 +109,7 @@ function(
       --out "${generated_cpp}" --module_name ${module_name}
       --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes}
       --template ${module_template} ${_WRAP_BOOST_ARG}
+      --xml_source "${GTWRAP_PYTHON_DOCS_SOURCE}"
     DEPENDS "${main_interface}" ${module_template} "${module_name}/specializations/${main_interface_name}.h" "${module_name}/specializations/${main_interface_name}.h"
     VERBATIM)
 

gtwrap/pybind_wrapper.py

@@ -19,6 +19,7 @@ from typing import List
 import gtwrap.interface_parser as parser
 import gtwrap.template_instantiator as instantiator
 
+from gtwrap.xml_parser.xml_parser import XMLDocParser
 
 class PybindWrapper:
     """
@@ -30,7 +31,8 @@ class PybindWrapper:
                  top_module_namespaces='',
                  use_boost_serialization=False,
                  ignore_classes=(),                 
-                 module_template=""):
+                 module_template="",
+                 xml_source=""):
         self.module_name = module_name
         self.top_module_namespaces = top_module_namespaces
         self.use_boost_serialization = use_boost_serialization
@@ -44,6 +46,8 @@ class PybindWrapper:
             'nonlocal', 'yield', 'break', 'for', 'not', 'class', 'from', 'or',
             'continue', 'global', 'pass'
         ]
+        self.xml_source = xml_source
+        self.xml_parser = XMLDocParser()
 
         self.dunder_methods = ('len', 'contains', 'iter')
 
@@ -260,7 +264,7 @@ class PybindWrapper:
                '[]({opt_self}{opt_comma}{args_signature_with_names}){{'
                '{function_call}'
                '}}'
-               '{py_args_names}){suffix}'.format(
+               '{py_args_names}{docstring}){suffix}'.format(
                    prefix=prefix,
                    cdef="def_static" if is_static else "def",
                    py_method=py_method,
@@ -271,6 +275,12 @@ class PybindWrapper:
                    function_call=function_call,
                    py_args_names=py_args_names,
                    suffix=suffix,
+                   # Try to get the function's docstring from the Doxygen XML.
+                   # If extract_docstring errors or fails to find a docstring, it just prints a warning.
+                   # The incantation repr(...)[1:-1].replace('"', r'\"') replaces newlines with \n 
+                   # and " with \" so that the docstring can be put into a C++ string on a single line.
+                   docstring=', "' + repr(self.xml_parser.extract_docstring(self.xml_source, cpp_class, cpp_method, method.args.names()))[1:-1].replace('"', r'\"') + '"' 
+                       if self.xml_source != "" else "",
                ))
 
         # Create __repr__ override

gtwrap/xml_parser/xml_parser.py

@@ -0,0 +1,304 @@
+import os
+import sys
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+
+class XMLDocParser:
+    """
+    Parses and extracts docs from Doxygen-generated XML.
+    """
+
+    def __init__(self):
+        # Memory for overloaded functions with identical parameter name sets
+        self._memory = {}
+        # This is useful for investigating functions that cause problems for extract_docstring.
+        # Set this to true to have useful information for debugging this class, as in the CLI
+        # function at the bottom of this class.
+        self._verbose = False
+
+    def parse_xml(self, xml_file: str):
+        """
+        Get the ElementTree of an XML file given the file name.
+        If an error occurs, prints a warning and returns None.
+        """
+        try:
+            return ET.parse(xml_file)
+        except FileNotFoundError:
+            print(f"Warning: XML file '{xml_file}' not found.")
+            return None
+        except ET.ParseError:
+            print(f"Warning: Failed to parse XML file '{xml_file}'.")
+            return None
+
+    def extract_docstring(self, xml_folder: str, cpp_class: str,
+                          cpp_method: str, method_args_names: 'list[str]'):
+        """
+        Extract the docstrings for a C++ class's method from the Doxygen-generated XML.
+    
+        Args:
+            xml_folder (str): The path to the folder that contains all of the Doxygen-generated XML.
+            cpp_class (str): The name of the C++ class that contains the function whose docstring is to be extracted.
+            cpp_method (str): The name of the C++ method whose docstring is to be extracted.
+            method_args_names (list): A list of the names of the cpp_method's parameters.
+        """
+        self.print_if_verbose(f"Extracting docs for {cpp_class}.{cpp_method}")
+
+        # Get all of the member definitions in cpp_class with name cpp_method
+        maybe_member_defs = self.get_member_defs(xml_folder, cpp_class,
+                                                 cpp_method)
+
+        # Filter member definitions which don't match the given argument names
+        member_defs, ignored_params = self.filter_member_defs(
+            maybe_member_defs, method_args_names)
+
+        # Find which member to get docs from, if there are multiple that match in name and args
+        documenting_index = self.determine_documenting_index(
+            cpp_class, cpp_method, method_args_names)
+
+        # Extract the docs for the function that matches cpp_class.cpp_method(*method_args_names).
+        return self.get_formatted_docstring(member_defs[documenting_index],
+                                            ignored_params)
+
+    def get_member_defs(self, xml_folder: str, cpp_class: str,
+                        cpp_method: str):
+        """Get all of the member definitions in cpp_class with name cpp_method.
+
+        Args:
+            xml_folder (str): The folder containing the Doxygen XML documentation.
+            cpp_class (str): The name of the C++ class that contains the function whose docstring is to be extracted.
+            cpp_method (str): The name of the C++ method whose docstring is to be extracted.
+
+        Returns:
+            list: All of the member definitions in cpp_class with name cpp_method.
+        """
+        xml_folder_path = Path(xml_folder)
+
+        # Create the path to the Doxygen XML index file.
+        xml_index_file = xml_folder_path / "index.xml"
+
+        # Parse the index file
+        index_tree = self.parse_xml(xml_index_file)
+        if not index_tree:
+            self.print_if_verbose(f"Index file {xml_index_file} was empty.")
+            return ""
+
+        index_root = index_tree.getroot()
+
+        # Find the compound with name == cpp_class
+        class_index = index_root.find(f"./*[name='{cpp_class}']")
+
+        if class_index is None:
+            self.print_if_verbose(
+                f"Could not extract docs for {cpp_class}.{cpp_method}; class not found in index file."
+            )
+            return ""
+
+        # Create the path to the file with the documentation for cpp_class.
+        xml_class_file = xml_folder_path / class_index.attrib['refid'] + '.xml'
+
+        # Parse the class file
+        class_tree = self.parse_xml(xml_class_file)
+        if not class_tree:
+            self.print_if_verbose(f"Class file {xml_class_file} was empty.")
+            return ""
+
+        class_root = class_tree.getroot()
+
+        # Find the member(s) in cpp_class with name == cpp_method
+        maybe_member_defs = class_root.findall(
+            f"compounddef/sectiondef//*[name='{cpp_method}']")
+
+        return maybe_member_defs
+
+    def filter_member_defs(self, maybe_member_defs: list,
+                           method_args_names: list):
+        """
+        Remove member definitions which do not match the supplied argument names list.
+
+        Args:
+            maybe_member_defs (list): The list of all member definitions in the class which share the same name.
+            method_args_names (list): The list of argument names in the definition of the function whose documentation is desired.
+                Supplying the argument names allows for the filtering of overloaded functions with the same name but different arguments.
+
+        Returns:
+            tuple[list, list]: (the filtered member definitions, parameters which should be ignored because they are optional)
+        """
+        member_defs = []
+
+        # Optional parameters we should ignore if we encounter them in the docstring
+        ignored_params = []
+
+        # Filter out the members which don't match the method_args_names
+        for maybe_member_def in maybe_member_defs:
+            self.print_if_verbose(
+                f"Investigating member_def with argstring {maybe_member_def.find('argsstring').text}"
+            )
+            # Find the number of required parameters and the number of total parameters from the
+            # Doxygen XML for this member_def
+            params = maybe_member_def.findall("param")
+            num_tot_params = len(params)
+            # Calculate required params by subtracting the number of optional params (params where defval is
+            # set--defval means default value) from the number of total params
+            num_req_params = num_tot_params - sum([
+                1 if param.find("defval") is not None else 0
+                for param in params
+            ])
+
+            # If the number of parameters in method_args_names matches neither number, eliminate this member_def
+            # This is done because wrap generates a python wrapper function twice for every function with
+            # optional parameters: one with none of the optional parameters, and one with all of the optional
+            # parameters, required.
+            if len(method_args_names) != num_req_params and len(
+                    method_args_names) != num_tot_params:
+                self.print_if_verbose(
+                    f"Wrong number of parameters: got {len(method_args_names)}, expected required {num_req_params} or total {num_tot_params}."
+                )
+                continue
+
+            # If the parameter names don't match, eliminate this member_def
+            eliminate = False
+            for i, arg_name in enumerate(method_args_names):
+                # Try to find the name of the parameter in the XML
+                param_name = params[i].find(
+                    "declname"
+                )  # declname is the tag that usually contains the param name
+                # If we couldn't find the declname, try the defname (used uncommonly)
+                if param_name is None:
+                    param_name = params[i].find("defname")
+                if param_name is None:
+                    # Can't find the name for this parameter. This may be an unreachable statement but Doxygen is
+                    # not well-documented enough to rely on a <declname> or a <defname> always being defined inside a <param>.
+                    eliminate = True
+                    continue
+                # Eliminate if any param name doesn't match the expected name
+                if arg_name != param_name.text:
+                    eliminate = True
+            if eliminate:
+                self.print_if_verbose("Names didn't match.")
+                continue
+
+            # At this point, this member_def can be assumed to be the desired function (or is indistinguishable
+            # from it based on all of the reliable information we have--if this is the case, we need to rely on
+            # the _memory to give the correct docs for each.)
+            member_defs.append(maybe_member_def)
+            self.print_if_verbose("Confirmed as correct function.")
+
+            # Remember which parameters to ignore, if any
+            for i in range(len(method_args_names), num_tot_params):
+                ignored_params.append(params[i].find("declname").text)
+
+        return member_defs, ignored_params
+
+    def determine_documenting_index(self, cpp_class: str, cpp_method: str,
+                                    method_args_names: list,
+                                    member_defs: list):
+        """
+        Determine which member definition to retrieve documentation from, if there are multiple.
+
+        Args:
+            cpp_class (str): The name of the C++ class that contains the function whose docstring is to be extracted.
+            cpp_method (str): The name of the C++ method whose docstring is to be extracted.
+            method_args_names (list): A list of the names of the cpp_method's parameters.
+            member_defs (list): All of the member definitions of cpp_class which match cpp_method in name 
+                and whose arguments have the same names as method_args_names.
+
+        Returns:
+            int: The index indicating which member definition to document.
+        """
+        # If there are multiple member defs that match the method args names,
+        # remember how many we've encountered already so that we can return
+        # the docs for the first one we haven't yet extracted.
+        # This is only relevant if there are overloaded functions where the
+        # parameter types are different but the parameter names are the same,
+        # e.g. foo(int bar) and foo(string bar). The parameter types cannot be
+        # relied on because they cannot be assumed to be the same between GTSAM
+        # implementation and pybind11 generated wrapper, e.g. OptionalJacobian
+        # in GTSAM becomes Eigen::Matrix in the pybind11 code.
+        documenting_index = 0
+        if len(member_defs) > 1:
+            function_key = f"{cpp_class}.{cpp_method}({','.join(method_args_names) if method_args_names else ''})"
+            if function_key in self._memory:
+                self._memory[function_key] += 1
+                documenting_index = self._memory[function_key]
+            else:
+                self._memory[function_key] = 0
+
+        return documenting_index
+
+    def get_formatted_docstring(self,
+                                member_def: 'xml.etree.ElementTree.Element',
+                                ignored_params: list):
+        """Gets the formatted docstring for the supplied XML element representing a member definition.
+
+        Args:
+            member_def (xml.etree.ElementTree.Element): The member definition to document.
+            ignored_params (list): The optional parameters which should be ignored, if any.
+
+        Returns:
+            str: The formatted docstring.
+        """
+        docstring = ""
+
+        brief_description = member_def.find(".//briefdescription")
+        detailed_description = member_def.find(".//detaileddescription")
+
+        # Add the brief description first, if it exists.
+        if brief_description is not None:
+            for para in brief_description.findall("para"):
+                docstring += "".join(t for t in para.itertext() if t.strip())
+
+        # Add the detailed description. This includes the parameter list and the return value.
+        if detailed_description is not None:
+            docstring += "\n"
+            # Add non-parameter detailed description
+            for element in list(detailed_description):
+                if element.tag == "para" and "parameterlist" not in [
+                        e.tag for e in element
+                ]:
+                    docstring += "".join(
+                        t for t in element.itertext() if t.strip()) + " "
+
+            # Add parameter docs
+            parameter_list = detailed_description.find(".//parameterlist")
+            if parameter_list is not None:
+                for i, parameter_item in enumerate(
+                        parameter_list.findall(".//parameteritem")):
+                    name = parameter_item.find(".//parametername").text
+                    desc = parameter_item.find(
+                        ".//parameterdescription/para").text
+                    if name not in ignored_params:
+                        docstring += f"{name.strip() if name else f'[Parameter {i}]'}: {desc.strip() if desc else 'No description provided'}\n"
+
+            # Add return value docs
+            return_sect = detailed_description.find(".//simplesect")
+            if return_sect is not None and return_sect.attrib[
+                    "kind"] == "return" and return_sect.find(
+                        "para").text is not None:
+                docstring += f"Returns: {return_sect.find('para').text.strip()}"
+
+        return docstring.strip()
+
+    def print_if_verbose(self, text: str):
+        """
+        Print text if the parser is in verbose mode.
+        """
+        if self._verbose:
+            print(text)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 5:
+        print(
+            "Usage: python xml_parser.py <doxygen_xml_folder> <cpp_class> <cpp_method> <method_args_names (comma-separated)>"
+        )
+    else:
+        parser = XMLDocParser()
+        parser._verbose = True
+        xml_file = sys.argv[1]
+        extracted_doc = parser.extract_docstring(xml_file, sys.argv[2],
+                                                 sys.argv[3],
+                                                 sys.argv[4].split(","))
+
+        print()
+        print(extracted_doc.strip())

pybind11/.clang-tidy

@@ -57,10 +57,12 @@ Checks: |
   readability-string-compare,
   readability-suspicious-call-argument,
   readability-uniqueptr-delete-release,
+  -bugprone-chained-comparison,
   -bugprone-easily-swappable-parameters,
   -bugprone-exception-escape,
   -bugprone-reserved-identifier,
   -bugprone-unused-raii,
+  -performance-enum-size,
 
 CheckOptions:
 - key:             modernize-use-equals-default.IgnoreMacros

pybind11/.codespell-ignore-lines

@@ -12,6 +12,17 @@ template <op_id id, op_type ot, typename L, typename R>
     template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
     class_ &def(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
     class_ &def_cast(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+    int valu;
+    explicit movable_int(int v) : valu{v} {}
+    movable_int(movable_int &&other) noexcept : valu(other.valu) { other.valu = 91; }
+    explicit indestructible_int(int v) : valu{v} {}
+    REQUIRE(hld.as_raw_ptr_unowned<zombie>()->valu == 19);
+        REQUIRE(othr.valu == 19);
+        REQUIRE(orig.valu == 91);
+        (m.pass_valu, "Valu", "pass_valu:Valu(_MvCtor)*_CpCtor"),
+atyp_valu        rtrn_valu() { atyp_valu obj{"Valu"}; return obj; }
+    assert m.atyp_valu().get_mtxt() == "Valu"
+// valu(e), ref(erence), ptr or p (pointer), r = rvalue, m = mutable, c = const,
 @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"])
 struct IntStruct {
     explicit IntStruct(int v) : value(v){};

pybind11/.github/CONTRIBUTING.md

@@ -81,7 +81,7 @@ nox -s build
 ### Full setup
 
 To setup an ideal development environment, run the following commands on a
-system with CMake 3.14+:
+system with CMake 3.15+:
 
 ```bash
 python3 -m venv venv
@@ -96,8 +96,8 @@ Tips:
 * You can use `virtualenv` (faster, from PyPI) instead of `venv`.
 * You can select any name for your environment folder; if it contains "env" it
   will be ignored by git.
-* If you don't have CMake 3.14+, just add "cmake" to the pip install command.
-* You can use `-DPYBIND11_FINDPYTHON=ON` to use FindPython on CMake 3.12+
+* If you don't have CMake 3.15+, just add "cmake" to the pip install command.
+* You can use `-DPYBIND11_FINDPYTHON=ON` to use FindPython.
 * In classic mode, you may need to set `-DPYTHON_EXECUTABLE=/path/to/python`.
   FindPython uses `-DPython_ROOT_DIR=/path/to` or
   `-DPython_EXECUTABLE=/path/to/python`.
@@ -149,8 +149,8 @@ To run the tests, you can "build" the check target:
 cmake --build build --target check
 ```
 
-`--target` can be spelled `-t` in CMake 3.15+. You can also run individual
-tests with these targets:
+`--target` can be spelled `-t`. You can also run individual tests with these
+targets:
 
 * `pytest`: Python tests only, using the
 [pytest](https://docs.pytest.org/en/stable/) framework

pybind11/.github/workflows/ci.yml

@@ -39,6 +39,8 @@ jobs:
         - 'pypy-3.8'
         - 'pypy-3.9'
         - 'pypy-3.10'
+        - 'pypy-3.11'
+        - 'graalpy-24.1'
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the
@@ -64,9 +66,45 @@ jobs:
           # Inject a couple Windows 2019 runs
           - runs-on: windows-2019
             python: '3.9'
+          # Inject a few runs with different runtime libraries
+          - runs-on: windows-2022
+            python: '3.9'
+            args: >
+              -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
+          - runs-on: windows-2022
+            python: '3.10'
+            args: >
+              -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDLL
+          # This needs a python built with MTd
+          # - runs-on: windows-2022
+          #   python: '3.11'
+          #   args: >
+          #     -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug
+          - runs-on: windows-2022
+            python: '3.12'
+            args: >
+              -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebugDLL
           # Extra ubuntu latest job
           - runs-on: ubuntu-latest
             python: '3.11'
+          # Run tests with py::smart_holder as the default holder
+          # with recent (or ideally latest) released Python version.
+          - runs-on: ubuntu-latest
+            python: '3.12'
+            args: >
+              -DCMAKE_CXX_FLAGS="-DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE"
+          - runs-on: macos-13
+            python: '3.12'
+            args: >
+              -DCMAKE_CXX_FLAGS="-DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE"
+          - runs-on: windows-2022
+            python: '3.12'
+            args: >
+              -DCMAKE_CXX_FLAGS="/DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE /GR /EHsc"
+        exclude:
+          # The setup-python action currently doesn't have graalpy for windows
+          - python: 'graalpy-24.1'
+            runs-on: 'windows-2022'
 
 
     name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}"
@@ -122,6 +160,7 @@ jobs:
         -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON
         -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON
         -DPYBIND11_NUMPY_1_ONLY=ON
+        -DPYBIND11_PYTEST_ARGS=-v
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=11
@@ -151,6 +190,7 @@ jobs:
         -DPYBIND11_WERROR=ON
         -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF
         -DPYBIND11_NUMPY_1_ONLY=ON
+        -DPYBIND11_PYTEST_ARGS=-v
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
@@ -170,6 +210,7 @@ jobs:
       run: >
         cmake -S . -B build3
         -DPYBIND11_WERROR=ON
+        -DPYBIND11_PYTEST_ARGS=-v
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
@@ -243,7 +284,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }} (deadsnakes)
-      uses: deadsnakes/action@v3.1.0
+      uses: deadsnakes/action@v3.2.0
       with:
         python-version: ${{ matrix.python-version }}
         debug: ${{ matrix.python-debug }}
@@ -310,22 +351,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        clang:
-          - 3.6
-          - 3.7
-          - 3.9
-          - 7
-          - 9
-          - dev
-        std:
-          - 11
         container_suffix:
           - ""
         include:
           - clang: 5
             std: 14
-          - clang: 10
-            std: 17
           - clang: 11
             std: 20
           - clang: 12
@@ -340,6 +370,12 @@ jobs:
           - clang: 16
             std: 20
             container_suffix: "-bullseye"
+          - clang: 17
+            std: 20
+            container_suffix: "-bookworm"
+          - clang: 18
+            std: 20
+            container_suffix: "-bookworm"
 
     name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64"
     container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}"
@@ -497,11 +533,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { gcc: 7, std: 11 }
-          - { gcc: 7, std: 17 }
-          - { gcc: 8, std: 14 }
-          - { gcc: 8, std: 17 }
+          - { gcc: 9, std: 20 }
           - { gcc: 10, std: 17 }
+          - { gcc: 10, std: 20 }
           - { gcc: 11, std: 20 }
           - { gcc: 12, std: 20 }
           - { gcc: 13, std: 20 }
@@ -719,9 +753,9 @@ jobs:
 
   # This tests an "install" with the CMake tools
   install-classic:
-    name: "🐍 3.7 • Debian • x86 •  Install"
+    name: "🐍 3.9 • Debian • x86 •  Install"
     runs-on: ubuntu-latest
-    container: i386/debian:buster
+    container: i386/debian:bullseye
 
     steps:
     - uses: actions/checkout@v1  # v1 is required to run inside docker
@@ -801,7 +835,6 @@ jobs:
       fail-fast: false
       matrix:
         python:
-        - '3.7'
         - '3.8'
         - '3.9'
         - '3.10'
@@ -819,8 +852,6 @@ jobs:
             args: -DCMAKE_CXX_STANDARD=20
           - python: '3.8'
             args: -DCMAKE_CXX_STANDARD=17
-          - python: '3.7'
-            args: -DCMAKE_CXX_STANDARD=14
 
 
     name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}"
@@ -999,7 +1030,6 @@ jobs:
           git
           mingw-w64-${{matrix.env}}-gcc
           mingw-w64-${{matrix.env}}-python-pip
-          mingw-w64-${{matrix.env}}-python-numpy
           mingw-w64-${{matrix.env}}-cmake
           mingw-w64-${{matrix.env}}-make
           mingw-w64-${{matrix.env}}-python-pytest
@@ -1011,7 +1041,7 @@ jobs:
       with:
         msystem: ${{matrix.sys}}
         install: >-
-          git
+          mingw-w64-${{matrix.env}}-python-numpy
           mingw-w64-${{matrix.env}}-python-scipy
           mingw-w64-${{matrix.env}}-eigen3
 
@@ -1109,7 +1139,7 @@ jobs:
         uses: jwlawson/actions-setup-cmake@v2.0
 
       - name: Install ninja-build tool
-        uses: seanmiddleditch/gha-setup-ninja@v5
+        uses: seanmiddleditch/gha-setup-ninja@v6
 
       - name: Run pip installs
         run: |

pybind11/.github/workflows/configure.yml

@@ -31,7 +31,7 @@ jobs:
         include:
         - runs-on: ubuntu-20.04
           arch: x64
-          cmake: "3.5"
+          cmake: "3.15"
 
         - runs-on: ubuntu-20.04
           arch: x64
@@ -39,22 +39,22 @@ jobs:
 
         - runs-on: macos-13
           arch: x64
-          cmake: "3.7"
+          cmake: "3.15"
 
         - runs-on: windows-2019
           arch: x64 # x86 compilers seem to be missing on 2019 image
           cmake: "3.18"
 
-    name: 🐍 3.7 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }}
+    name: 🐍 3.8 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }}
     runs-on: ${{ matrix.runs-on }}
 
     steps:
     - uses: actions/checkout@v4
 
-    - name: Setup Python 3.7
+    - name: Setup Python 3.8
       uses: actions/setup-python@v5
       with:
-        python-version: 3.7
+        python-version: 3.8
         architecture: ${{ matrix.arch }}
 
     - name: Prepare env

pybind11/.github/workflows/emscripten.yaml

@@ -0,0 +1,30 @@
+name: WASM
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+    - master
+    - stable
+    - v*
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-wasm-emscripten:
+    name: Pyodide wheel
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+        fetch-depth: 0
+
+    - uses: pypa/cibuildwheel@v2.23
+      env:
+        PYODIDE_BUILD_EXPORTS: whole_archive
+      with:
+        package-dir: tests
+        only: cp312-pyodide_wasm32

pybind11/.github/workflows/format.yml

@@ -41,7 +41,7 @@ jobs:
     # in .github/CONTRIBUTING.md and update as needed.
     name: Clang-Tidy
     runs-on: ubuntu-latest
-    container: silkeh/clang:15-bullseye
+    container: silkeh/clang:18-bookworm
     steps:
     - uses: actions/checkout@v4
 

pybind11/.github/workflows/pip.yml

@@ -91,18 +91,19 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && github.event.action == 'published'
     needs: [packaging]
-    environment: pypi
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pybind11
     permissions:
       id-token: write
       attestations: write
-      contents: read
 
     steps:
     # Downloads all to directories matching the artifact names
     - uses: actions/download-artifact@v4
 
     - name: Generate artifact attestation for sdist and wheel
-      uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2
+      uses: actions/attest-build-provenance@bd77c077858b8d561b7a36cbe48ef4cc642ca39d # v2.2.2
       with:
         subject-path: "*/pybind11*"
 
@@ -110,8 +111,10 @@ jobs:
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         packages-dir: standard/
+        attestations: true
 
     - name: Publish global package
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         packages-dir: global/
+        attestations: true

pybind11/.pre-commit-config.yaml

@@ -25,14 +25,14 @@ repos:
 
 # Clang format the codebase automatically
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: "v18.1.5"
+  rev: "v19.1.7"
   hooks:
   - id: clang-format
     types_or: [c++, c, cuda]
 
 # Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.7
+  rev: v0.9.9
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
@@ -40,7 +40,7 @@ repos:
 
 # Check static types with mypy
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: "v1.10.0"
+  rev: "v1.15.0"
   hooks:
   - id: mypy
     args: []
@@ -62,7 +62,7 @@ repos:
 
 # Standard hooks
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: "v4.6.0"
+  rev: "v5.0.0"
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
@@ -76,10 +76,11 @@ repos:
   - id: mixed-line-ending
   - id: requirements-txt-fixer
   - id: trailing-whitespace
+    exclude: \.patch?$
 
 # Also code format the docs
 - repo: https://github.com/adamchainz/blacken-docs
-  rev: "1.16.0"
+  rev: "1.19.1"
   hooks:
   - id: blacken-docs
     additional_dependencies:
@@ -90,10 +91,11 @@ repos:
   rev: "v1.5.5"
   hooks:
   - id: remove-tabs
+    exclude: (^docs/.*|\.patch)?$
 
 # Avoid directional quotes
 - repo: https://github.com/sirosen/texthooks
-  rev: "0.6.6"
+  rev: "0.6.8"
   hooks:
   - id: fix-ligatures
   - id: fix-smartquotes
@@ -108,7 +110,7 @@ repos:
 
 # Checks the manifest for missing files (native support)
 - repo: https://github.com/mgedmin/check-manifest
-  rev: "0.49"
+  rev: "0.50"
   hooks:
   - id: check-manifest
     # This is a slow hook, so only run this if --hook-stage manual is passed
@@ -119,7 +121,7 @@ repos:
 # Use tools/codespell_ignore_lines_from_errors.py
 # to rebuild .codespell-ignore-lines
 - repo: https://github.com/codespell-project/codespell
-  rev: "v2.3.0"
+  rev: "v2.4.1"
   hooks:
   - id: codespell
     exclude: ".supp$"
@@ -142,14 +144,14 @@ repos:
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v3.2.2"
+  rev: "v3.3.4"
   hooks:
   - id: pylint
     files: ^pybind11
 
 # Check schemas on some of our YAML files
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.28.4
+  rev: 0.31.2
   hooks:
   - id: check-readthedocs
   - id: check-github-workflows

pybind11/CMakeLists.txt

@@ -10,16 +10,7 @@ if(NOT CMAKE_VERSION VERSION_LESS "3.27")
   cmake_policy(GET CMP0148 _pybind11_cmp0148)
 endif()
 
-cmake_minimum_required(VERSION 3.5)
-
-# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with
-# some versions of VS that have a patched CMake 3.11. This forces us to emulate
-# the behavior using the following workaround:
-if(${CMAKE_VERSION} VERSION_LESS 3.29)
-  cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
-else()
-  cmake_policy(VERSION 3.29)
-endif()
+cmake_minimum_required(VERSION 3.15...3.30)
 
 if(_pybind11_cmp0148)
   cmake_policy(SET CMP0148 ${_pybind11_cmp0148})
@@ -27,9 +18,7 @@ if(_pybind11_cmp0148)
 endif()
 
 # Avoid infinite recursion if tests include this as a subdirectory
-if(DEFINED PYBIND11_MASTER_PROJECT)
-  return()
-endif()
+include_guard(GLOBAL)
 
 # Extract project version from source
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/pybind11/detail/common.h"
@@ -74,14 +63,6 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
 
   set(PYBIND11_MASTER_PROJECT ON)
 
-  if(OSX AND CMAKE_VERSION VERSION_LESS 3.7)
-    # Bug in macOS CMake < 3.7 is unable to download catch
-    message(WARNING "CMAKE 3.7+ needed on macOS to download catch, and newer HIGHLY recommended")
-  elseif(WINDOWS AND CMAKE_VERSION VERSION_LESS 3.8)
-    # Only tested with 3.8+ in CI.
-    message(WARNING "CMAKE 3.8+ tested on Windows, previous versions untested")
-  endif()
-
   message(STATUS "CMake ${CMAKE_VERSION}")
 
   if(CMAKE_CXX_STANDARD)
@@ -133,8 +114,7 @@ cmake_dependent_option(
   "Install pybind11 headers in Python include directory instead of default installation prefix"
   OFF "PYBIND11_INSTALL" OFF)
 
-cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" ${_pybind11_findpython_default}
-                       "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF)
+option(PYBIND11_FINDPYTHON "Force new FindPython" ${_pybind11_findpython_default})
 
 # Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests
 # (makes transition easier while we support both modes).
@@ -149,17 +129,26 @@ endif()
 set(PYBIND11_HEADERS
     include/pybind11/detail/class.h
     include/pybind11/detail/common.h
+    include/pybind11/detail/cpp_conduit.h
     include/pybind11/detail/descr.h
+    include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
+    include/pybind11/detail/struct_smart_holder.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h
+    include/pybind11/detail/using_smart_holder.h
+    include/pybind11/detail/value_and_holder.h
+    include/pybind11/detail/exception_translation.h
     include/pybind11/attr.h
     include/pybind11/buffer_info.h
     include/pybind11/cast.h
     include/pybind11/chrono.h
     include/pybind11/common.h
     include/pybind11/complex.h
+    include/pybind11/conduit/pybind11_conduit_v1.h
+    include/pybind11/conduit/pybind11_platform_abi_id.h
+    include/pybind11/conduit/wrap_include_python_h.h
     include/pybind11/options.h
     include/pybind11/eigen.h
     include/pybind11/eigen/common.h
@@ -178,11 +167,13 @@ set(PYBIND11_HEADERS
     include/pybind11/stl.h
     include/pybind11/stl_bind.h
     include/pybind11/stl/filesystem.h
+    include/pybind11/trampoline_self_life_support.h
     include/pybind11/type_caster_pyobject_ptr.h
-    include/pybind11/typing.h)
+    include/pybind11/typing.h
+    include/pybind11/warnings.h)
 
 # Compare with grep and warn if mismatched
-if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)
+if(PYBIND11_MASTER_PROJECT)
   file(
     GLOB_RECURSE _pybind11_header_check
     LIST_DIRECTORIES false
@@ -200,10 +191,7 @@ if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)
   endif()
 endif()
 
-# CMake 3.12 added list(TRANSFORM <list> PREPEND
-# But we can't use it yet
-string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" PYBIND11_HEADERS
-               "${PYBIND11_HEADERS}")
+list(TRANSFORM PYBIND11_HEADERS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
 
 # Cache variable so this can be used in parent projects
 set(pybind11_INCLUDE_DIR
@@ -273,25 +261,11 @@ if(PYBIND11_INSTALL)
     tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
     INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
-  if(CMAKE_VERSION VERSION_LESS 3.14)
-    # Remove CMAKE_SIZEOF_VOID_P from ConfigVersion.cmake since the library does
-    # not depend on architecture specific settings or libraries.
-    set(_PYBIND11_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P})
-    unset(CMAKE_SIZEOF_VOID_P)
-
-    write_basic_package_version_file(
-      ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
-      VERSION ${PROJECT_VERSION}
-      COMPATIBILITY AnyNewerVersion)
-
-    set(CMAKE_SIZEOF_VOID_P ${_PYBIND11_CMAKE_SIZEOF_VOID_P})
-  else()
-    # CMake 3.14+ natively supports header-only libraries
-    write_basic_package_version_file(
-      ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
-      VERSION ${PROJECT_VERSION}
-      COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT)
-  endif()
+  # CMake natively supports header-only libraries
+  write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT)
 
   install(
     FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake

pybind11/README.rst

@@ -1,7 +1,7 @@
 .. figure:: https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png
    :alt: pybind11 logo
 
-**pybind11 — Seamless operability between C++11 and Python**
+**pybind11 (v3)  — Seamless interoperability between C++ and Python**
 
 |Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| |CI| |Build status|
 
@@ -34,7 +34,7 @@ dependency.
 Think of this library as a tiny self-contained version of Boost.Python
 with everything stripped away that isn't relevant for binding
 generation. Without comments, the core header files only require ~4K
-lines of code and depend on Python (3.7+, or PyPy) and the C++
+lines of code and depend on Python (3.8+, or PyPy) and the C++
 standard library. This compact implementation was possible thanks to
 some C++11 language features (specifically: tuples, lambda functions and
 variadic templates). Since its creation, this library has grown beyond
@@ -79,7 +79,7 @@ Goodies
 In addition to the core functionality, pybind11 provides some extra
 goodies:
 
-- Python 3.7+, and PyPy3 7.3 are supported with an implementation-agnostic
+- Python 3.8+, and PyPy3 7.3 are supported with an implementation-agnostic
   interface (pybind11 2.9 was the last version to support Python 2 and 3.5).
 
 - It is possible to bind C++11 lambda functions with captured
@@ -134,11 +134,34 @@ About
 
 This project was created by `Wenzel
 Jakob <http://rgl.epfl.ch/people/wjakob>`_. Significant features and/or
-improvements to the code were contributed by Jonas Adler, Lori A. Burns,
-Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel
-Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov, Johan Mabille, Tomasz Miąsko,
-Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim
-Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart.
+improvements to the code were contributed by
+Jonas Adler,
+Lori A. Burns,
+Sylvain Corlay,
+Eric Cousineau,
+Aaron Gokaslan,
+Ralf Grosse-Kunstleve,
+Trent Houliston,
+Axel Huebl,
+@hulucc,
+Yannick Jadoul,
+Sergey Lyskov,
+Johan Mabille,
+Tomasz Miąsko,
+Dean Moldovan,
+Ben Pritchard,
+Jason Rhinelander,
+Boris Schäling,
+Pim Schellart,
+Henry Schreiner,
+Ivan Smirnov,
+Dustin Spicuzza,
+Boris Staletic,
+Ethan Steinberg,
+Patrick Stewart,
+Ivor Wanders,
+and
+Xiaofei Wang.
 
 We thank Google for a generous financial contribution to the continuous
 integration infrastructure used by this project.

pybind11/docs/advanced/cast/custom.rst

@@ -1,35 +1,53 @@
 Custom type casters
 ===================
 
-In very rare cases, applications may require custom type casters that cannot be
-expressed using the abstractions provided by pybind11, thus requiring raw
-Python C API calls. This is fairly advanced usage and should only be pursued by
-experts who are familiar with the intricacies of Python reference counting.
+Some applications may prefer custom type casters that convert between existing
+Python types and C++ types, similar to the ``list`` ↔ ``std::vector``
+and ``dict`` ↔ ``std::map`` conversions which are built into pybind11.
+Implementing custom type casters is fairly advanced usage.
+While it is recommended to use the pybind11 API as much as possible, more complex examples may
+require familiarity with the intricacies of the Python C API.
+You can refer to the `Python/C API Reference Manual <https://docs.python.org/3/c-api/index.html>`_
+for more information.
 
-The following snippets demonstrate how this works for a very simple ``inty``
-type that that should be convertible from Python types that provide a
-``__int__(self)`` method.
+The following snippets demonstrate how this works for a very simple ``Point2D`` type.
+We want this type to be convertible to C++ from Python types implementing the
+``Sequence`` protocol and having two elements of type ``float``.
+When returned from C++ to Python, it should be converted to a Python ``tuple[float, float]``.
+For this type we could provide Python bindings for different arithmetic functions implemented
+in C++ (here demonstrated by a simple ``negate`` function).
+
+..
+    PLEASE KEEP THE CODE BLOCKS IN SYNC WITH
+        tests/test_docs_advanced_cast_custom.cpp
+        tests/test_docs_advanced_cast_custom.py
+    Ideally, change the test, run pre-commit (incl. clang-format),
+    then copy the changed code back here.
+    Also use TEST_SUBMODULE in tests, but PYBIND11_MODULE in docs.
 
 .. code-block:: cpp
 
-    struct inty { long long_value; };
+    namespace user_space {
 
-    void print(inty s) {
-        std::cout << s.long_value << std::endl;
-    }
+    struct Point2D {
+        double x;
+        double y;
+    };
 
-The following Python snippet demonstrates the intended usage from the Python side:
+    Point2D negate(const Point2D &point) { return Point2D{-point.x, -point.y}; }
+
+    } // namespace user_space
+
+
+The following Python snippet demonstrates the intended usage of ``negate`` from the Python side:
 
 .. code-block:: python
 
-    class A:
-        def __int__(self):
-            return 123
+    from my_math_module import docs_advanced_cast_custom as m
 
-
-    from example import print
-
-    print(A())
+    point1 = [1.0, -1.0]
+    point2 = m.negate(point1)
+    assert point2 == (-1.0, 1.0)
 
 To register the necessary conversion routines, it is necessary to add an
 instantiation of the ``pybind11::detail::type_caster<T>`` template.
@@ -38,47 +56,57 @@ type is explicitly allowed.
 
 .. code-block:: cpp
 
-    namespace PYBIND11_NAMESPACE { namespace detail {
-        template <> struct type_caster<inty> {
-        public:
-            /**
-             * This macro establishes the name 'inty' in
-             * function signatures and declares a local variable
-             * 'value' of type inty
-             */
-            PYBIND11_TYPE_CASTER(inty, const_name("inty"));
+    namespace pybind11 {
+    namespace detail {
 
-            /**
-             * Conversion part 1 (Python->C++): convert a PyObject into a inty
-             * instance or return false upon failure. The second argument
-             * indicates whether implicit conversions should be applied.
-             */
-            bool load(handle src, bool) {
-                /* Extract PyObject from handle */
-                PyObject *source = src.ptr();
-                /* Try converting into a Python integer value */
-                PyObject *tmp = PyNumber_Long(source);
-                if (!tmp)
+    template <>
+    struct type_caster<user_space::Point2D> {
+        // This macro inserts a lot of boilerplate code and sets the type hint.
+        // `io_name` is used to specify different type hints for arguments and return values.
+        // The signature of our negate function would then look like:
+        // `negate(Sequence[float]) -> tuple[float, float]`
+        PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]"));
+
+        // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments
+        // are used to indicate the return value policy and parent object (for
+        // return_value_policy::reference_internal) and are often ignored by custom casters.
+        // The return value should reflect the type hint specified by the second argument of `io_name`.
+        static handle
+        cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) {
+            return py::make_tuple(number.x, number.y).release();
+        }
+
+        // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The
+        // second argument indicates whether implicit conversions should be allowed.
+        // The accepted types should reflect the type hint specified by the first argument of
+        // `io_name`.
+        bool load(handle src, bool /*convert*/) {
+            // Check if handle is a Sequence
+            if (!py::isinstance<py::sequence>(src)) {
+                return false;
+            }
+            auto seq = py::reinterpret_borrow<py::sequence>(src);
+            // Check if exactly two values are in the Sequence
+            if (seq.size() != 2) {
+                return false;
+            }
+            // Check if each element is either a float or an int
+            for (auto item : seq) {
+                if (!py::isinstance<py::float_>(item) && !py::isinstance<py::int_>(item)) {
                     return false;
-                /* Now try to convert into a C++ int */
-                value.long_value = PyLong_AsLong(tmp);
-                Py_DECREF(tmp);
-                /* Ensure return code was OK (to avoid out-of-range errors etc) */
-                return !(value.long_value == -1 && !PyErr_Occurred());
+                }
             }
+            value.x = seq[0].cast<double>();
+            value.y = seq[1].cast<double>();
+            return true;
+        }
+    };
 
-            /**
-             * Conversion part 2 (C++ -> Python): convert an inty instance into
-             * a Python object. The second and third arguments are used to
-             * indicate the return value policy and parent object (for
-             * ``return_value_policy::reference_internal``) and are generally
-             * ignored by implicit casters.
-             */
-            static handle cast(inty src, return_value_policy /* policy */, handle /* parent */) {
-                return PyLong_FromLong(src.long_value);
-            }
-        };
-    }} // namespace PYBIND11_NAMESPACE::detail
+    } // namespace detail
+    } // namespace pybind11
+
+    // Bind the negate function
+    PYBIND11_MODULE(docs_advanced_cast_custom, m) { m.def("negate", user_space::negate); }
 
 .. note::
 
@@ -86,8 +114,22 @@ type is explicitly allowed.
     that ``T`` is default-constructible (``value`` is first default constructed
     and then ``load()`` assigns to it).
 
+.. note::
+    For further information on the ``return_value_policy`` argument of ``cast`` refer to :ref:`return_value_policies`.
+    To learn about the ``convert`` argument of ``load`` see :ref:`nonconverting_arguments`.
+
 .. warning::
 
     When using custom type casters, it's important to declare them consistently
-    in every compilation unit of the Python extension module. Otherwise,
+    in every compilation unit of the Python extension module to satisfy the C++ One Definition Rule
+    (`ODR <https://en.cppreference.com/w/cpp/language/definition>`_). Otherwise,
     undefined behavior can ensue.
+
+.. note::
+
+    Using the type hint ``Sequence[float]`` signals to static type checkers, that not only tuples may be
+    passed, but any type implementing the Sequence protocol, e.g., ``list[float]``.
+    Unfortunately, that loses the length information ``tuple[float, float]`` provides.
+    One way of still providing some length information in type hints is using ``typing.Annotated``, e.g.,
+    ``Annotated[Sequence[float], 2]``, or further add libraries like
+    `annotated-types <https://github.com/annotated-types/annotated-types>`_.

pybind11/docs/advanced/cast/eigen.rst

@@ -259,7 +259,7 @@ copying to take place:
         "small"_a            // <- This one can be copied if needed
     );
 
-With the above binding code, attempting to call the the ``some_method(m)``
+With the above binding code, attempting to call the ``some_method(m)``
 method on a ``MyClass`` object, or attempting to call ``some_function(m, m2)``
 will raise a ``RuntimeError`` rather than making a temporary copy of the array.
 It will, however, allow the ``m2`` argument to be copied into a temporary if

pybind11/docs/advanced/cast/overview.rst

@@ -151,7 +151,7 @@ as arguments and return values, refer to the section on binding :ref:`classes`.
 +------------------------------------+---------------------------+-----------------------------------+
 | ``std::variant<...>``              | Type-safe union (C++17)   | :file:`pybind11/stl.h`            |
 +------------------------------------+---------------------------+-----------------------------------+
-| ``std::filesystem::path<T>``       | STL path (C++17) [#]_     | :file:`pybind11/stl/filesystem.h` |
+| ``std::filesystem::path``          | STL path (C++17) [#]_     | :file:`pybind11/stl/filesystem.h` |
 +------------------------------------+---------------------------+-----------------------------------+
 | ``std::function<...>``             | STL polymorphic function  | :file:`pybind11/functional.h`     |
 +------------------------------------+---------------------------+-----------------------------------+
@@ -167,4 +167,4 @@ as arguments and return values, refer to the section on binding :ref:`classes`.
 +------------------------------------+---------------------------+-----------------------------------+
 
 .. [#] ``std::filesystem::path`` is converted to ``pathlib.Path`` and
-   ``os.PathLike`` is converted to ``std::filesystem::path``.
+   can be loaded from ``os.PathLike``, ``str``, and ``bytes``.

pybind11/docs/advanced/cast/stl.rst

@@ -162,15 +162,15 @@ the declaration
 
 .. code-block:: cpp
 
-    PYBIND11_MAKE_OPAQUE(std::vector<int>);
+    PYBIND11_MAKE_OPAQUE(std::vector<int>)
 
 before any binding code (e.g. invocations to ``class_::def()``, etc.). This
 macro must be specified at the top level (and outside of any namespaces), since
 it adds a template instantiation of ``type_caster``. If your binding code consists of
 multiple compilation units, it must be present in every file (typically via a
 common header) preceding any usage of ``std::vector<int>``. Opaque types must
-also have a corresponding ``class_`` declaration to associate them with a name
-in Python, and to define a set of available operations, e.g.:
+also have a corresponding ``py::class_`` declaration to associate them with a
+name in Python, and to define a set of available operations, e.g.:
 
 .. code-block:: cpp
 
@@ -207,8 +207,8 @@ The following example showcases usage of :file:`pybind11/stl_bind.h`:
     // Don't forget this
     #include <pybind11/stl_bind.h>
 
-    PYBIND11_MAKE_OPAQUE(std::vector<int>);
-    PYBIND11_MAKE_OPAQUE(std::map<std::string, double>);
+    PYBIND11_MAKE_OPAQUE(std::vector<int>)
+    PYBIND11_MAKE_OPAQUE(std::map<std::string, double>)
 
     // ...
 

pybind11/docs/advanced/classes.rst

@@ -64,7 +64,7 @@ helper class that is defined as follows:
 
 .. code-block:: cpp
 
-    class PyAnimal : public Animal {
+    class PyAnimal : public Animal, py::trampoline_self_life_support {
     public:
         /* Inherit the constructors */
         using Animal::Animal;
@@ -80,6 +80,18 @@ helper class that is defined as follows:
         }
     };
 
+The ``py::trampoline_self_life_support`` base class is needed to ensure
+that a ``std::unique_ptr`` can safely be passed between Python and C++. To
+steer clear of notorious pitfalls (e.g. inheritance slicing), it is best
+practice to always use the base class, in combination with
+``py::smart_holder``.
+
+.. note::
+    For completeness, the base class has no effect if a holder other than
+    ``py::smart_holder`` used, including the default ``std::unique_ptr<T>``.
+    Please think twice, though, the pitfalls are very real, and the overhead
+    for using the safer ``py::smart_holder`` is very likely to be in the noise.
+
 The macro :c:macro:`PYBIND11_OVERRIDE_PURE` should be used for pure virtual
 functions, and :c:macro:`PYBIND11_OVERRIDE` should be used for functions which have
 a default implementation.  There are also two alternate macros
@@ -95,18 +107,18 @@ The binding code also needs a few minor adaptations (highlighted):
     :emphasize-lines: 2,3
 
     PYBIND11_MODULE(example, m) {
-        py::class_<Animal, PyAnimal /* <--- trampoline*/>(m, "Animal")
+        py::class_<Animal, PyAnimal /* <--- trampoline */, py::smart_holder>(m, "Animal")
             .def(py::init<>())
             .def("go", &Animal::go);
 
-        py::class_<Dog, Animal>(m, "Dog")
+        py::class_<Dog, Animal, py::smart_holder>(m, "Dog")
             .def(py::init<>());
 
         m.def("call_go", &call_go);
     }
 
 Importantly, pybind11 is made aware of the trampoline helper class by
-specifying it as an extra template argument to :class:`class_`. (This can also
+specifying it as an extra template argument to ``py::class_``. (This can also
 be combined with other template arguments such as a custom holder type; the
 order of template types does not matter).  Following this, we are able to
 define a constructor as usual.
@@ -116,9 +128,9 @@ Bindings should be made against the actual class, not the trampoline helper clas
 .. code-block:: cpp
     :emphasize-lines: 3
 
-    py::class_<Animal, PyAnimal /* <--- trampoline*/>(m, "Animal");
+    py::class_<Animal, PyAnimal /* <--- trampoline */, py::smart_holder>(m, "Animal");
         .def(py::init<>())
-        .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */
+        .def("go", &Animal::go); /* <--- DO NOT USE &PyAnimal::go HERE */
 
 Note, however, that the above is sufficient for allowing python classes to
 extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the
@@ -244,13 +256,13 @@ override the ``name()`` method):
 
 .. code-block:: cpp
 
-    class PyAnimal : public Animal {
+    class PyAnimal : public Animal, py::trampoline_self_life_support {
     public:
         using Animal::Animal; // Inherit constructors
         std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Animal, go, n_times); }
         std::string name() override { PYBIND11_OVERRIDE(std::string, Animal, name, ); }
     };
-    class PyDog : public Dog {
+    class PyDog : public Dog, py::trampoline_self_life_support {
     public:
         using Dog::Dog; // Inherit constructors
         std::string go(int n_times) override { PYBIND11_OVERRIDE(std::string, Dog, go, n_times); }
@@ -272,7 +284,7 @@ declare or override any virtual methods itself:
 .. code-block:: cpp
 
     class Husky : public Dog {};
-    class PyHusky : public Husky {
+    class PyHusky : public Husky, py::trampoline_self_life_support {
     public:
         using Husky::Husky; // Inherit constructors
         std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, Husky, go, n_times); }
@@ -287,13 +299,15 @@ follows:
 
 .. code-block:: cpp
 
-    template <class AnimalBase = Animal> class PyAnimal : public AnimalBase {
+    template <class AnimalBase = Animal>
+    class PyAnimal : public AnimalBase, py::trampoline_self_life_support {
     public:
         using AnimalBase::AnimalBase; // Inherit constructors
         std::string go(int n_times) override { PYBIND11_OVERRIDE_PURE(std::string, AnimalBase, go, n_times); }
         std::string name() override { PYBIND11_OVERRIDE(std::string, AnimalBase, name, ); }
     };
-    template <class DogBase = Dog> class PyDog : public PyAnimal<DogBase> {
+    template <class DogBase = Dog>
+    class PyDog : public PyAnimal<DogBase>, py::trampoline_self_life_support {
     public:
         using PyAnimal<DogBase>::PyAnimal; // Inherit constructors
         // Override PyAnimal's pure virtual go() with a non-pure one:
@@ -311,9 +325,9 @@ The classes are then registered with pybind11 using:
 
 .. code-block:: cpp
 
-    py::class_<Animal, PyAnimal<>> animal(m, "Animal");
-    py::class_<Dog, Animal, PyDog<>> dog(m, "Dog");
-    py::class_<Husky, Dog, PyDog<Husky>> husky(m, "Husky");
+    py::class_<Animal, PyAnimal<>, py::smart_holder> animal(m, "Animal");
+    py::class_<Dog, Animal, PyDog<>, py::smart_holder> dog(m, "Dog");
+    py::class_<Husky, Dog, PyDog<Husky>, py::smart_holder> husky(m, "Husky");
     // ... add animal, dog, husky definitions
 
 Note that ``Husky`` did not require a dedicated trampoline template class at
@@ -499,12 +513,12 @@ an alias:
         // ...
         virtual ~Example() = default;
     };
-    class PyExample : public Example {
+    class PyExample : public Example, py::trampoline_self_life_support {
     public:
         using Example::Example;
         PyExample(Example &&base) : Example(std::move(base)) {}
     };
-    py::class_<Example, PyExample>(m, "Example")
+    py::class_<Example, PyExample, py::smart_holder>(m, "Example")
         // Returns an Example pointer.  If a PyExample is needed, the Example
         // instance will be moved via the extra constructor in PyExample, above.
         .def(py::init([]() { return new Example(); }))
@@ -550,9 +564,10 @@ pybind11. The underlying issue is that the ``std::unique_ptr`` holder type that
 is responsible for managing the lifetime of instances will reference the
 destructor even if no deallocations ever take place. In order to expose classes
 with private or protected destructors, it is possible to override the holder
-type via a holder type argument to ``class_``. Pybind11 provides a helper class
-``py::nodelete`` that disables any destructor invocations. In this case, it is
-crucial that instances are deallocated on the C++ side to avoid memory leaks.
+type via a holder type argument to ``py::class_``. Pybind11 provides a helper
+class ``py::nodelete`` that disables any destructor invocations. In this case,
+it is crucial that instances are deallocated on the C++ side to avoid memory
+leaks.
 
 .. code-block:: cpp
 
@@ -826,8 +841,7 @@ An instance can now be pickled as follows:
     always use the latest available version. Beware: failure to follow these
     instructions will cause important pybind11 memory allocation routines to be
     skipped during unpickling, which will likely lead to memory corruption
-    and/or segmentation faults. Python defaults to version 3 (Python 3-3.7) and
-    version 4 for Python 3.8+.
+    and/or segmentation faults.
 
 .. seealso::
 
@@ -872,7 +886,7 @@ Multiple Inheritance
 
 pybind11 can create bindings for types that derive from multiple base types
 (aka. *multiple inheritance*). To do so, specify all bases in the template
-arguments of the ``class_`` declaration:
+arguments of the ``py::class_`` declaration:
 
 .. code-block:: cpp
 
@@ -947,11 +961,11 @@ because of conflicting definitions on the external type:
     // dogs.cpp
 
     // Binding for external library class:
-    py::class<pets::Pet>(m, "Pet")
+    py::class_<pets::Pet>(m, "Pet")
         .def("name", &pets::Pet::name);
 
     // Binding for local extension class:
-    py::class<Dog, pets::Pet>(m, "Dog")
+    py::class_<Dog, pets::Pet>(m, "Dog")
         .def(py::init<std::string>());
 
 .. code-block:: cpp
@@ -959,11 +973,11 @@ because of conflicting definitions on the external type:
     // cats.cpp, in a completely separate project from the above dogs.cpp.
 
     // Binding for external library class:
-    py::class<pets::Pet>(m, "Pet")
+    py::class_<pets::Pet>(m, "Pet")
         .def("get_name", &pets::Pet::name);
 
     // Binding for local extending class:
-    py::class<Cat, pets::Pet>(m, "Cat")
+    py::class_<Cat, pets::Pet>(m, "Cat")
         .def(py::init<std::string>());
 
 .. code-block:: pycon
@@ -981,13 +995,13 @@ the ``py::class_`` constructor:
 .. code-block:: cpp
 
     // Pet binding in dogs.cpp:
-    py::class<pets::Pet>(m, "Pet", py::module_local())
+    py::class_<pets::Pet>(m, "Pet", py::module_local())
         .def("name", &pets::Pet::name);
 
 .. code-block:: cpp
 
     // Pet binding in cats.cpp:
-    py::class<pets::Pet>(m, "Pet", py::module_local())
+    py::class_<pets::Pet>(m, "Pet", py::module_local())
         .def("get_name", &pets::Pet::name);
 
 This makes the Python-side ``dogs.Pet`` and ``cats.Pet`` into distinct classes,
@@ -1105,7 +1119,7 @@ described trampoline:
         virtual int foo() const { return 42; }
     };
 
-    class Trampoline : public A {
+    class Trampoline : public A, py::trampoline_self_life_support {
     public:
         int foo() const override { PYBIND11_OVERRIDE(int, A, foo, ); }
     };
@@ -1115,7 +1129,7 @@ described trampoline:
         using A::foo;
     };
 
-    py::class_<A, Trampoline>(m, "A") // <-- `Trampoline` here
+    py::class_<A, Trampoline, py::smart_holder>(m, "A") // <-- `Trampoline` here
         .def("foo", &Publicist::foo); // <-- `Publicist` here, not `Trampoline`!
 
 Binding final classes
@@ -1196,7 +1210,7 @@ but once again each instantiation must be explicitly specified:
         T fn(V v);
     };
 
-    py::class<MyClass<int>>(m, "MyClassT")
+    py::class_<MyClass<int>>(m, "MyClassT")
         .def("fn", &MyClass<int>::fn<std::string>);
 
 Custom automatic downcasters

pybind11/docs/advanced/deadlock.md

@@ -0,0 +1,391 @@
+# Double locking, deadlocking, GIL
+
+[TOC]
+
+## Introduction
+
+### Overview
+
+In concurrent programming with locks, *deadlocks* can arise when more than one
+mutex is locked at the same time, and careful attention has to be paid to lock
+ordering to avoid this. Here we will look at a common situation that occurs in
+native extensions for CPython written in C++.
+
+### Deadlocks
+
+A deadlock can occur when more than one thread attempts to lock more than one
+mutex, and two of the threads lock two of the mutexes in different orders. For
+example, consider mutexes `mu1` and `mu2`, and threads T1 and T2, executing:
+
+|    | T1                  | T2                 |
+|--- | ------------------- | -------------------|
+|1   | `mu1.lock()`{.good} | `mu2.lock()`{.good}|
+|2   | `mu2.lock()`{.bad}  | `mu1.lock()`{.bad} |
+|3   | `/* work */`        | `/* work */`       |
+|4   | `mu2.unlock()`      | `mu1.unlock()`     |
+|5   | `mu1.unlock()`      | `mu2.unlock()`     |
+
+Now if T1 manages to lock `mu1` and T2 manages to lock `mu2` (as indicated in
+green), then both threads will block while trying to lock the respective other
+mutex (as indicated in red), but they are also unable to release the mutex that
+they have locked (step 5).
+
+**The problem** is that it is possible for one thread to attempt to lock `mu1`
+and then `mu2`, and for another thread to attempt to lock `mu2` and then `mu1`.
+Note that it does not matter if either mutex is unlocked at any intermediate
+point; what matters is only the order of any attempt to *lock* the mutexes. For
+example, the following, more complex series of operations is just as prone to
+deadlock:
+
+|    | T1                  | T2                 |
+|--- | ------------------- | -------------------|
+|1   | `mu1.lock()`{.good} | `mu1.lock()`{.good}|
+|2   | waiting for T2      | `mu2.lock()`{.good}|
+|3   | waiting for T2      | `/* work */`       |
+|3   | waiting for T2      | `mu1.unlock()`     |
+|3   | `mu2.lock()`{.bad}  | `/* work */`       |
+|3   | `/* work */`        | `mu1.lock()`{.bad} |
+|3   | `/* work */`        | `/* work */`       |
+|4   | `mu2.unlock()`      | `mu1.unlock()`     |
+|5   | `mu1.unlock()`      | `mu2.unlock()`     |
+
+When the mutexes involved in a locking sequence are known at compile-time, then
+avoiding deadlocks is “merely” a matter of arranging the lock
+operations carefully so as to only occur in one single, fixed order. However, it
+is also possible for mutexes to only be determined at runtime. A typical example
+of this is a database where each row has its own mutex. An operation that
+modifies two rows in a single transaction (e.g. “transferring an amount
+from one account to another”) must lock two row mutexes, but the locking
+order cannot be established at compile time. In this case, a dynamic
+“deadlock avoidance algorithm” is needed. (In C++, `std::lock`
+provides such an algorithm. An algorithm might use a non-blocking `try_lock`
+operation on a mutex, which can either succeed or fail to lock the mutex, but
+returns without blocking.)
+
+Conceptually, one could also consider it a deadlock if _the same_ thread
+attempts to lock a mutex that it has already locked (e.g. when some locked
+operation accidentally recurses into itself): `mu.lock();`{.good}
+`mu.lock();`{.bad} However, this is a slightly separate issue: Typical mutexes
+are either of _recursive_ or _non-recursive_ kind. A recursive mutex allows
+repeated locking and requires balanced unlocking. A non-recursive mutex can be
+implemented more efficiently, and/but for efficiency reasons does not actually
+guarantee a deadlock on second lock. Instead, the API simply forbids such use,
+making it a precondition that the thread not already hold the mutex, with
+undefined behaviour on violation.
+
+### “Once” initialization
+
+A common programming problem is to have an operation happen precisely once, even
+if requested concurrently. While it is clear that we need to track in some
+shared state somewhere whether the operation has already happened, it is worth
+noting that this state only ever transitions, once, from `false` to `true`. This
+is considerably simpler than a general shared state that can change values
+arbitrarily. Next, we also need a mechanism for all but one thread to block
+until the initialization has completed, which we can provide with a mutex. The
+simplest solution just always locks the mutex:
+
+```c++
+// The "once" mechanism:
+constinit absl::Mutex mu(absl::kConstInit);
+constinit bool init_done = false;
+
+// The operation of interest:
+void f();
+
+void InitOnceNaive() {
+  absl::MutexLock lock(&mu);
+  if (!init_done) {
+    f();
+    init_done = true;
+  }
+}
+```
+
+This works, but the efficiency-minded reader will observe that once the
+operation has completed, all future lock contention on the mutex is
+unnecessary. This leads to the (in)famous “double-locking”
+algorithm, which was historically hard to write correctly. The idea is to check
+the boolean *before* locking the mutex, and avoid locking if the operation has
+already completed. However, accessing shared state concurrently when at least
+one access is a write is prone to causing a data race and needs to be done
+according to an appropriate concurrent programming model. In C++ we use atomic
+variables:
+
+```c++
+// The "once" mechanism:
+constinit absl::Mutex mu(absl::kConstInit);
+constinit std::atomic<bool> init_done = false;
+
+// The operation of interest:
+void f();
+
+void InitOnceWithFastPath() {
+  if (!init_done.load(std::memory_order_acquire)) {
+    absl::MutexLock lock(&mu);
+    if (!init_done.load(std::memory_order_relaxed)) {
+      f();
+      init_done.store(true, std::memory_order_release);
+    }
+  }
+}
+```
+
+Checking the flag now happens without holding the mutex lock, and if the
+operation has already completed, we return immediately. After locking the mutex,
+we need to check the flag again, since multiple threads can reach this point.
+
+*Atomic details.* Since the atomic flag variable is accessed concurrently, we
+have to think about the memory order of the accesses. There are two separate
+cases: The first, outer check outside the mutex lock, and the second, inner
+check under the lock. The outer check and the flag update form an
+acquire/release pair: *if* the load sees the value `true` (which must have been
+written by the store operation), then it also sees everything that happened
+before the store, namely the operation `f()`. By contrast, the inner check can
+use relaxed memory ordering, since in that case the mutex operations provide the
+necessary ordering: if the inner load sees the value `true`, it happened after
+the `lock()`, which happened after the `unlock()`, which happened after the
+store.
+
+The C++ standard library, and Abseil, provide a ready-made solution of this
+algorithm called `std::call_once`/`absl::call_once`. (The interface is the same,
+but the Abseil implementation is possibly better.)
+
+```c++
+// The "once" mechanism:
+constinit absl::once_flag init_flag;
+
+// The operation of interest:
+void f();
+
+void InitOnceWithCallOnce() {
+  absl::call_once(once_flag, f);
+}
+```
+
+Even though conceptually this is performing the same algorithm, this
+implementation has some considerable advantages: The `once_flag` type is a small
+and trivial, integer-like type and is trivially destructible. Not only does it
+take up less space than a mutex, it also generates less code since it does not
+have to run a destructor, which would need to be added to the program's global
+destructor list.
+
+The final clou comes with the C++ semantics of a `static` variable declared at
+block scope: According to [[stmt.dcl]](https://eel.is/c++draft/stmt.dcl#3):
+
+> Dynamic initialization of a block variable with static storage duration or
+> thread storage duration is performed the first time control passes through its
+> declaration; such a variable is considered initialized upon the completion of
+> its initialization. [...] If control enters the declaration concurrently while
+> the variable is being initialized, the concurrent execution shall wait for
+> completion of the initialization.
+
+This is saying that the initialization of a local, `static` variable precisely
+has the &ldquo;once&rdquo; semantics that we have been discussing. We can
+therefore write the above example as follows:
+
+```c++
+// The operation of interest:
+void f();
+
+void InitOnceWithStatic() {
+  static int unused = (f(), 0);
+}
+```
+
+This approach is by far the simplest and easiest, but the big difference is that
+the mutex (or mutex-like object) in this implementation is no longer visible or
+in the user&rsquo;s control. This is perfectly fine if the initializer is
+simple, but if the initializer itself attempts to lock any other mutex
+(including by initializing another static variable!), then we have no control
+over the lock ordering!
+
+Finally, you may have noticed the `constinit`s around the earlier code. Both
+`constinit` and `constexpr` specifiers on a declaration mean that the variable
+is *constant-initialized*, which means that no initialization is performed at
+runtime (the initial value is already known at compile time). This in turn means
+that a static variable guard mutex may not be needed, and static initialization
+never blocks. The difference between the two is that a `constexpr`-specified
+variable is also `const`, and a variable cannot be `constexpr` if it has a
+non-trivial destructor. Such a destructor also means that the guard mutex is
+needed after all, since the destructor must be registered to run at exit,
+conditionally on initialization having happened.
+
+## Python, CPython, GIL
+
+With CPython, a Python program can call into native code. To this end, the
+native code registers callback functions with the Python runtime via the CPython
+API. In order to ensure that the internal state of the Python runtime remains
+consistent, there is a single, shared mutex called the &ldquo;global interpreter
+lock&rdquo;, or GIL for short. Upon entry of one of the user-provided callback
+functions, the GIL is locked (or &ldquo;held&rdquo;), so that no other mutations
+of the Python runtime state can occur until the native callback returns.
+
+Many native extensions do not interact with the Python runtime for at least some
+part of them, and so it is common for native extensions to _release_ the GIL, do
+some work, and then reacquire the GIL before returning. Similarly, when code is
+generally not holding the GIL but needs to interact with the runtime briefly, it
+will first reacquire the GIL. The GIL is reentrant, and constructions to acquire
+and subsequently release the GIL are common, and often don't worry about whether
+the GIL is already held.
+
+If the native code is written in C++ and contains local, `static` variables,
+then we are now dealing with at least _two_ mutexes: the static variable guard
+mutex, and the GIL from CPython.
+
+A common problem in such code is an operation with &ldquo;only once&rdquo;
+semantics that also ends up requiring the GIL to be held at some point. As per
+the above description of &ldquo;once&rdquo;-style techniques, one might find a
+static variable:
+
+```c++
+// CPython callback, assumes that the GIL is held on entry.
+PyObject* InvokeWidget(PyObject* self) {
+  static PyObject* impl = CreateWidget();
+  return PyObject_CallOneArg(impl, self);
+}
+```
+
+This seems reasonable, but bear in mind that there are two mutexes (the "guard
+mutex" and "the GIL"), and we must think about the lock order. Otherwise, if the
+callback is called from multiple threads, a deadlock may ensue.
+
+Let us consider what we can see here: On entry, the GIL is already locked, and
+we are locking the guard mutex. This is one lock order. Inside the initializer
+`CreateWidget`, with both mutexes already locked, the function can freely access
+the Python runtime.
+
+However, it is entirely possible that `CreateWidget` will want to release the
+GIL at one point and reacquire it later:
+
+```c++
+// Assumes that the GIL is held on entry.
+// Ensures that the GIL is held on exit.
+PyObject* CreateWidget() {
+  // ...
+  Py_BEGIN_ALLOW_THREADS  // releases GIL
+  // expensive work, not accessing the Python runtime
+  Py_END_ALLOW_THREADS    // acquires GIL, #!
+  // ...
+  return result;
+}
+```
+
+Now we have a second lock order: the guard mutex is locked, and then the GIL is
+locked (at `#!`). To see how this deadlocks, consider threads T1 and T2 both
+having the runtime attempt to call `InvokeWidget`. T1 locks the GIL and
+proceeds, locking the guard mutex and calling `CreateWidget`; T2 is blocked
+waiting for the GIL. Then T1 releases the GIL to do &ldquo;expensive
+work&rdquo;, and T2 awakes and locks the GIL. Now T2 is blocked trying to
+acquire the guard mutex, but T1 is blocked reacquiring the GIL (at `#!`).
+
+In other words: if we want to support &ldquo;once-called&rdquo; functions that
+can arbitrarily release and reacquire the GIL, as is very common, then the only
+lock order that we can ensure is: guard mutex first, GIL second.
+
+To implement this, we must rewrite our code. Naively, we could always release
+the GIL before a `static` variable with blocking initializer:
+
+```c++
+// CPython callback, assumes that the GIL is held on entry.
+PyObject* InvokeWidget(PyObject* self) {
+  Py_BEGIN_ALLOW_THREADS  // releases GIL
+  static PyObject* impl = CreateWidget();
+  Py_END_ALLOW_THREADS    // acquires GIL
+
+  return PyObject_CallOneArg(impl, self);
+}
+```
+
+But similar to the `InitOnceNaive` example above, this code cycles the GIL
+(possibly descheduling the thread) even when the static variable has already
+been initialized. If we want to avoid this, we need to abandon the use of a
+static variable, since we do not control the guard mutex well enough. Instead,
+we use an operation whose mutex locking is under our control, such as
+`call_once`. For example:
+
+```c++
+// CPython callback, assumes that the GIL is held on entry.
+PyObject* InvokeWidget(PyObject* self) {
+  static constinit PyObject* impl = nullptr;
+  static constinit std::atomic<bool> init_done = false;
+  static constinit absl::once_flag init_flag;
+
+  if (!init_done.load(std::memory_order_acquire)) {
+    Py_BEGIN_ALLOW_THREADS                       // releases GIL
+    absl::call_once(init_flag, [&]() {
+      PyGILState_STATE s = PyGILState_Ensure();  // acquires GIL
+      impl = CreateWidget();
+      PyGILState_Release(s);                     // releases GIL
+      init_done.store(true, std::memory_order_release);
+    });
+    Py_END_ALLOW_THREADS                         // acquires GIL
+  }
+
+  return PyObject_CallOneArg(impl, self);
+}
+```
+
+The lock order is now always guard mutex first, GIL second. Unfortunately we
+have to duplicate the &ldquo;double-checked done flag&rdquo;, effectively
+leading to triple checking, because the flag state inside the `absl::once_flag`
+is not accessible to the user. In other words, we cannot ask `init_flag` whether
+it has been used yet.
+
+However, we can perform one last, minor optimisation: since we assume that the
+GIL is held on entry, and again when the initializing operation returns, the GIL
+actually serializes access to our done flag variable, which therefore does not
+need to be atomic. (The difference to the previous, atomic code may be small,
+depending on the architecture. For example, on x86-64, acquire/release on a bool
+is nearly free ([demo](https://godbolt.org/z/P9vYWf4fE)).)
+
+```c++
+// CPython callback, assumes that the GIL is held on entry, and indeed anywhere
+// directly in this function (i.e. the GIL can be released inside CreateWidget,
+// but must be reaqcuired when that call returns).
+PyObject* InvokeWidget(PyObject* self) {
+  static constinit PyObject* impl = nullptr;
+  static constinit bool init_done = false;       // guarded by GIL
+  static constinit absl::once_flag init_flag;
+
+  if (!init_done) {
+    Py_BEGIN_ALLOW_THREADS                       // releases GIL
+                                                 // (multiple threads may enter here)
+    absl::call_once(init_flag, [&]() {
+                                                 // (only one thread enters here)
+      PyGILState_STATE s = PyGILState_Ensure();  // acquires GIL
+      impl = CreateWidget();
+      init_done = true;                          // (GIL is held)
+      PyGILState_Release(s);                     // releases GIL
+    });
+
+    Py_END_ALLOW_THREADS                         // acquires GIL
+  }
+
+  return PyObject_CallOneArg(impl, self);
+}
+```
+
+## Debugging tips
+
+*   Build with symbols.
+*   <kbd>Ctrl</kbd>-<kbd>C</kbd> sends `SIGINT`, <kbd>Ctrl</kbd>-<kbd>\\</kbd>
+    sends `SIGQUIT`. Both have their uses.
+*   Useful `gdb` commands:
+    *   `py-bt` prints a Python backtrace if you are in a Python frame.
+    *   `thread apply all bt 10` prints the top-10 frames for each thread. A
+        full backtrace can be prohibitively expensive, and the top few frames
+        are often good enough.
+    *   `p PyGILState_Check()` shows whether a thread is holding the GIL. For
+        all threads, run `thread apply all p PyGILState_Check()` to find out
+        which thread is holding the GIL.
+    *   The `static` variable guard mutex is accessed with functions like
+        `cxa_guard_acquire` (though this depends on ABI details and can vary).
+        The guard mutex itself contains information about which thread is
+        currently holding it.
+
+## Links
+
+*   Article on
+    [double-checked locking](https://preshing.com/20130930/double-checked-locking-is-fixed-in-cpp11/)
+*   [The Deadlock Empire](https://deadlockempire.github.io/), hands-on exercises
+    to construct deadlocks

pybind11/docs/advanced/embedding.rst

@@ -18,7 +18,7 @@ information, see :doc:`/compiling`.
 
 .. code-block:: cmake
 
-    cmake_minimum_required(VERSION 3.5...3.29)
+    cmake_minimum_required(VERSION 3.15...3.30)
     project(example)
 
     find_package(pybind11 REQUIRED)  # or `add_subdirectory(pybind11)`

pybind11/docs/advanced/exceptions.rst

@@ -368,8 +368,7 @@ Should they throw or fail to catch any exceptions in their call graph,
 the C++ runtime calls ``std::terminate()`` to abort immediately.
 
 Similarly, Python exceptions raised in a class's ``__del__`` method do not
-propagate, but are logged by Python as an unraisable error. In Python 3.8+, a
-`system hook is triggered
+propagate, but ``sys.unraisablehook()`` `is triggered
 <https://docs.python.org/3/library/sys.html#sys.unraisablehook>`_
 and an auditing event is logged.
 

pybind11/docs/advanced/functions.rst

@@ -81,9 +81,11 @@ The following table provides an overview of available policies:
 |                                                  | it is no longer used. Warning: undefined behavior will ensue when the C++  |
 |                                                  | side deletes an object that is still referenced and used by Python.        |
 +--------------------------------------------------+----------------------------------------------------------------------------+
-| :enum:`return_value_policy::reference_internal`  | Indicates that the lifetime of the return value is tied to the lifetime    |
-|                                                  | of a parent object, namely the implicit ``this``, or ``self`` argument of  |
-|                                                  | the called method or property. Internally, this policy works just like     |
+| :enum:`return_value_policy::reference_internal`  | If the return value is an lvalue reference or a pointer, the parent object |
+|                                                  | (the implicit ``this``, or ``self`` argument of the called method or       |
+|                                                  | property) is kept alive for at least the lifespan of the return value.     |
+|                                                  | **Otherwise this policy falls back to :enum:`return_value_policy::move`    |
+|                                                  | (see #5528).** Internally, this policy works just like                     |
 |                                                  | :enum:`return_value_policy::reference` but additionally applies a          |
 |                                                  | ``keep_alive<0, 1>`` *call policy* (described in the next section) that    |
 |                                                  | prevents the parent object from being garbage collected as long as the     |

pybind11/docs/advanced/misc.rst

@@ -62,7 +62,11 @@ will acquire the GIL before calling the Python callback. Similarly, the
 back into Python.
 
 When writing C++ code that is called from other C++ code, if that code accesses
-Python state, it must explicitly acquire and release the GIL.
+Python state, it must explicitly acquire and release the GIL. A separate
+document on deadlocks [#f8]_ elaborates on a particularly subtle interaction
+with C++'s block-scope static variable initializer guard mutexes.
+
+.. [#f8] See docs/advanced/deadlock.md
 
 The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be
 used to acquire and release the global interpreter lock in the body of a C++
@@ -76,7 +80,7 @@ could be realized as follows (important changes highlighted):
 .. code-block:: cpp
     :emphasize-lines: 8,30,31
 
-    class PyAnimal : public Animal {
+    class PyAnimal : public Animal, py::trampoline_self_life_support {
     public:
         /* Inherit the constructors */
         using Animal::Animal;
@@ -94,12 +98,12 @@ could be realized as follows (important changes highlighted):
     };
 
     PYBIND11_MODULE(example, m) {
-        py::class_<Animal, PyAnimal> animal(m, "Animal");
+        py::class_<Animal, PyAnimal, py::smart_holder> animal(m, "Animal");
         animal
             .def(py::init<>())
             .def("go", &Animal::go);
 
-        py::class_<Dog>(m, "Dog", animal)
+        py::class_<Dog, py::smart_holder>(m, "Dog", animal)
             .def(py::init<>());
 
         m.def("call_go", [](Animal *animal) -> std::string {
@@ -142,6 +146,9 @@ following checklist.
   destructors can sometimes get invoked in weird and unexpected circumstances as a result
   of exceptions.
 
+- C++ static block-scope variable initialization that calls back into Python can
+  cause deadlocks; see [#f8]_ for a detailed discussion.
+
 - You should try running your code in a debug build. That will enable additional assertions
   within pybind11 that will throw exceptions on certain GIL handling errors
   (reference counting operations).
@@ -181,7 +188,7 @@ from Section :ref:`inheritance`.
 Suppose now that ``Pet`` bindings are defined in a module named ``basic``,
 whereas the ``Dog`` bindings are defined somewhere else. The challenge is of
 course that the variable ``pet`` is not available anymore though it is needed
-to indicate the inheritance relationship to the constructor of ``class_<Dog>``.
+to indicate the inheritance relationship to the constructor of ``py::class_<Dog>``.
 However, it can be acquired as follows:
 
 .. code-block:: cpp
@@ -193,7 +200,7 @@ However, it can be acquired as follows:
         .def("bark", &Dog::bark);
 
 Alternatively, you can specify the base class as a template parameter option to
-``class_``, which performs an automated lookup of the corresponding Python
+``py::class_``, which performs an automated lookup of the corresponding Python
 type. Like the above code, however, this also requires invoking the ``import``
 function once to ensure that the pybind11 binding code of the module ``basic``
 has been executed:

pybind11/docs/advanced/smart_ptrs.rst

@@ -1,11 +1,70 @@
-Smart pointers
-##############
+Smart pointers & ``py::class_``
+###############################
 
-std::unique_ptr
-===============
+The binding generator for classes, ``py::class_``, can be passed a template
+type that denotes a special *holder* type that is used to manage references to
+the object. If no such holder type template argument is given, the default for
+a type ``T`` is ``std::unique_ptr<T>``.
 
-Given a class ``Example`` with Python bindings, it's possible to return
-instances wrapped in C++11 unique pointers, like so
+.. note::
+
+    A ``py::class_`` for a given C++ type ``T`` — and all its derived types —
+    can only use a single holder type.
+
+
+.. _smart_holder:
+
+``py::smart_holder``
+====================
+
+Starting with pybind11v3, ``py::smart_holder`` is built into pybind11. It is
+the recommended ``py::class_`` holder for most situations. However, for
+backward compatibility it is **not** the default holder, and there are no
+plans to make it the default holder in the future.
+
+It is extremely easy to use the safer and more versatile ``py::smart_holder``:
+simply add ``py::smart_holder`` to ``py::class_``:
+
+* ``py::class_<T>`` to
+
+* ``py::class_<T, py::smart_holder>``.
+
+.. note::
+
+    A shorthand, ``py::classh<T>``, is provided for
+    ``py::class_<T, py::smart_holder>``. The ``h`` in ``py::classh`` stands
+    for **smart_holder** but is shortened for brevity, ensuring it has the
+    same number of characters as ``py::class_``. This design choice facilitates
+    easy experimentation with ``py::smart_holder`` without introducing
+    distracting whitespace noise in diffs.
+
+The ``py::smart_holder`` functionality includes the following:
+
+* Support for **two-way** Python/C++ conversions for both
+  ``std::unique_ptr<T>`` and ``std::shared_ptr<T>`` **simultaneously**.
+
+* Passing a Python object back to C++ via ``std::unique_ptr<T>``, safely
+  **disowning** the Python object.
+
+* Safely passing "trampoline" objects (objects with C++ virtual function
+  overrides implemented in Python, see :ref:`overriding_virtuals`) via
+  ``std::unique_ptr<T>`` or ``std::shared_ptr<T>`` back to C++:
+  associated Python objects are automatically kept alive for the lifetime
+  of the smart-pointer.
+
+* Full support for ``std::enable_shared_from_this`` (`cppreference
+  <http://en.cppreference.com/w/cpp/memory/enable_shared_from_this>`_).
+
+
+``std::unique_ptr``
+===================
+
+This is the default ``py::class_`` holder and works as expected in
+most situations. However, handling base-and-derived classes involves a
+``reinterpret_cast``, which is, strictly speaking, undefined behavior.
+Also note that the ``std::unique_ptr`` holder only supports passing a
+``std::unique_ptr`` from C++ to Python, but not the other way around.
+For example, the following code works as expected with ``py::class_<Example>``:
 
 .. code-block:: cpp
 
@@ -15,116 +74,54 @@ instances wrapped in C++11 unique pointers, like so
 
     m.def("create_example", &create_example);
 
-In other words, there is nothing special that needs to be done. While returning
-unique pointers in this way is allowed, it is *illegal* to use them as function
-arguments. For instance, the following function signature cannot be processed
-by pybind11.
+However, this will fail with ``py::class_<Example>`` (but works with
+``py::class_<Example, py::smart_holder>``):
 
 .. code-block:: cpp
 
     void do_something_with_example(std::unique_ptr<Example> ex) { ... }
 
-The above signature would imply that Python needs to give up ownership of an
-object that is passed to this function, which is generally not possible (for
-instance, the object might be referenced elsewhere).
+.. note::
 
-std::shared_ptr
-===============
+    The ``reinterpret_cast`` mentioned above is `here
+    <https://github.com/pybind/pybind11/blob/30eb39ed79d1e2eeff15219ac00773034300a5e6/include/pybind11/cast.h#L235>`_.
+    For completeness: The same cast is also applied to ``py::smart_holder``,
+    but that is safe, because ``py::smart_holder`` is not templated.
 
-The binding generator for classes, :class:`class_`, can be passed a template
-type that denotes a special *holder* type that is used to manage references to
-the object.  If no such holder type template argument is given, the default for
-a type named ``Type`` is ``std::unique_ptr<Type>``, which means that the object
-is deallocated when Python's reference count goes to zero.
 
-It is possible to switch to other types of reference counting wrappers or smart
-pointers, which is useful in codebases that rely on them. For instance, the
-following snippet causes ``std::shared_ptr`` to be used instead.
+``std::shared_ptr``
+===================
+
+It is possible to use ``std::shared_ptr`` as the holder, for example:
 
 .. code-block:: cpp
 
-    py::class_<Example, std::shared_ptr<Example> /* <- holder type */> obj(m, "Example");
+    py::class_<Example, std::shared_ptr<Example> /* <- holder type */>(m, "Example");
 
-Note that any particular class can only be associated with a single holder type.
+Compared to using ``py::class_<Example, py::smart_holder>``, there are two noteworthy disadvantages:
 
-One potential stumbling block when using holder types is that they need to be
-applied consistently. Can you guess what's broken about the following binding
-code?
+* Because a ``py::class_`` for a given C++ type ``T`` can only use a
+  single holder type, ``std::unique_ptr<T>`` cannot even be passed from C++
+  to Python. This will become apparent only at runtime, often through a
+  segmentation fault.
 
-.. code-block:: cpp
+* Similar to the ``std::unique_ptr`` holder, the handling of base-and-derived
+  classes involves a ``reinterpret_cast`` that has strictly speaking undefined
+  behavior, although it works as expected in most situations.
 
-    class Child { };
-
-    class Parent {
-    public:
-       Parent() : child(std::make_shared<Child>()) { }
-       Child *get_child() { return child.get(); }  /* Hint: ** DON'T DO THIS ** */
-    private:
-        std::shared_ptr<Child> child;
-    };
-
-    PYBIND11_MODULE(example, m) {
-        py::class_<Child, std::shared_ptr<Child>>(m, "Child");
-
-        py::class_<Parent, std::shared_ptr<Parent>>(m, "Parent")
-           .def(py::init<>())
-           .def("get_child", &Parent::get_child);
-    }
-
-The following Python code will cause undefined behavior (and likely a
-segmentation fault).
-
-.. code-block:: python
-
-   from example import Parent
-
-   print(Parent().get_child())
-
-The problem is that ``Parent::get_child()`` returns a pointer to an instance of
-``Child``, but the fact that this instance is already managed by
-``std::shared_ptr<...>`` is lost when passing raw pointers. In this case,
-pybind11 will create a second independent ``std::shared_ptr<...>`` that also
-claims ownership of the pointer. In the end, the object will be freed **twice**
-since these shared pointers have no way of knowing about each other.
-
-There are two ways to resolve this issue:
-
-1. For types that are managed by a smart pointer class, never use raw pointers
-   in function arguments or return values. In other words: always consistently
-   wrap pointers into their designated holder types (such as
-   ``std::shared_ptr<...>``). In this case, the signature of ``get_child()``
-   should be modified as follows:
-
-.. code-block:: cpp
-
-    std::shared_ptr<Child> get_child() { return child; }
-
-2. Adjust the definition of ``Child`` by specifying
-   ``std::enable_shared_from_this<T>`` (see cppreference_ for details) as a
-   base class. This adds a small bit of information to ``Child`` that allows
-   pybind11 to realize that there is already an existing
-   ``std::shared_ptr<...>`` and communicate with it. In this case, the
-   declaration of ``Child`` should look as follows:
-
-.. _cppreference: http://en.cppreference.com/w/cpp/memory/enable_shared_from_this
-
-.. code-block:: cpp
-
-    class Child : public std::enable_shared_from_this<Child> { };
 
 .. _smart_pointers:
 
 Custom smart pointers
 =====================
 
-pybind11 supports ``std::unique_ptr`` and ``std::shared_ptr`` right out of the
-box. For any other custom smart pointer, transparent conversions can be enabled
-using a macro invocation similar to the following. It must be declared at the
-top namespace level before any binding code:
+For custom smart pointers (e.g. ``c10::intrusive_ptr`` in pytorch), transparent
+conversions can be enabled using a macro invocation similar to the following.
+It must be declared at the top namespace level before any binding code:
 
 .. code-block:: cpp
 
-    PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>);
+    PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>)
 
 The first argument of :func:`PYBIND11_DECLARE_HOLDER_TYPE` should be a
 placeholder name that is used as a template parameter of the second argument.
@@ -136,7 +133,7 @@ by default. Specify
 
 .. code-block:: cpp
 
-    PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>, true);
+    PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>, true)
 
 if ``SmartPtr<T>`` can always be initialized from a ``T*`` pointer without the
 risk of inconsistencies (such as multiple independent ``SmartPtr`` instances
@@ -154,7 +151,7 @@ specialized:
 .. code-block:: cpp
 
     // Always needed for custom holder types
-    PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>);
+    PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>)
 
     // Only needed if the type's `.get()` goes by another name
     namespace PYBIND11_NAMESPACE { namespace detail {
@@ -167,8 +164,70 @@ specialized:
 The above specialization informs pybind11 that the custom ``SmartPtr`` class
 provides ``.get()`` functionality via ``.getPointer()``.
 
+.. note::
+
+    The two noteworthy disadvantages mentioned under the ``std::shared_ptr``
+    section apply similarly to custom smart pointer holders, but there is no
+    established safe alternative in this case.
+
 .. seealso::
 
     The file :file:`tests/test_smart_ptr.cpp` contains a complete example
     that demonstrates how to work with custom reference-counting holder types
     in more detail.
+
+
+Be careful not to accidentally undermine automatic lifetime management
+======================================================================
+
+``py::class_``-wrapped objects automatically manage the lifetime of the
+wrapped C++ object, in collaboration with the chosen holder type.
+When wrapping C++ functions involving raw pointers, care needs to be taken
+to not inadvertently transfer ownership, resulting in multiple Python
+objects acting as owners, causing heap-use-after-free or double-free errors.
+For example:
+
+.. code-block:: cpp
+
+    class Child { };
+
+    class Parent {
+    public:
+       Parent() : child(std::make_shared<Child>()) { }
+       Child *get_child() { return child.get(); }  /* DANGER */
+    private:
+        std::shared_ptr<Child> child;
+    };
+
+    PYBIND11_MODULE(example, m) {
+        py::class_<Child, std::shared_ptr<Child>>(m, "Child");
+
+        py::class_<Parent, std::shared_ptr<Parent>>(m, "Parent")
+           .def(py::init<>())
+           .def("get_child", &Parent::get_child);  /* PROBLEM */
+    }
+
+The following Python code leads to undefined behavior, likely resulting in
+a segmentation fault.
+
+.. code-block:: python
+
+   from example import Parent
+
+   print(Parent().get_child())
+
+Part of the ``/* PROBLEM */`` here is that pybind11 falls back to using
+``return_value_policy::take_ownership`` as the default (see
+:ref:`return_value_policies`). The fact that the ``Child`` instance is
+already managed by ``std::shared_ptr<Child>`` is lost. Therefore pybind11
+will create a second independent ``std::shared_ptr<Child>`` that also
+claims ownership of the pointer, eventually leading to heap-use-after-free
+or double-free errors.
+
+There are various ways to resolve this issue, either by changing
+the ``Child`` or ``Parent`` C++ implementations (e.g. using
+``std::enable_shared_from_this<Child>`` as a base class for
+``Child``, or adding a member function to ``Parent`` that returns
+``std::shared_ptr<Child>``), or if that is not feasible, by using
+``return_value_policy::reference_internal``. What is the best approach
+depends on the exact situation.

pybind11/docs/basics.rst

@@ -78,6 +78,13 @@ For brevity, all code examples assume that the following two lines are present:
 
     namespace py = pybind11;
 
+.. note::
+
+    ``pybind11/pybind11.h`` includes ``Python.h``, as such it must be the first file
+    included in any source file or header for `the same reasons as Python.h`_.
+
+.. _`the same reasons as Python.h`: https://docs.python.org/3/extending/extending.html#a-simple-example
+
 Some features may require additional headers, but those will be specified as needed.
 
 .. _simple_example:
@@ -135,7 +142,7 @@ On Linux, the above example can be compiled using the following command:
 
 .. code-block:: bash
 
-    $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix)
+    $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3 -m pybind11 --extension-suffix)
 
 .. note::
 

pybind11/docs/benchmark.py

@@ -48,7 +48,7 @@ def generate_dummy_code_boost(nclasses=10):
     decl += "\n"
 
     for cl in range(nclasses):
-        decl += "class cl%03i {\n" % cl
+        decl += f"class cl{cl:03} {{\n"
         decl += "public:\n"
         bindings += f'    py::class_<cl{cl:03}>("cl{cl:03}")\n'
         for fn in range(nfns):
@@ -85,5 +85,5 @@ for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]:
         n2 = dt.datetime.now()
         elapsed = (n2 - n1).total_seconds()
         size = os.stat("test.so").st_size
-        print("   {%i, %f, %i}," % (nclasses * nfns, elapsed, size))
+        print(f"   {{{nclasses * nfns}, {elapsed:.6f}, {size}}},")
     print("}")

pybind11/docs/changelog.rst

@@ -15,6 +15,146 @@ IN DEVELOPMENT
 
 Changes will be summarized here periodically.
 
+New Features:
+
+* Support for Python 3.7 was removed. (Official end-of-life: 2023-06-27).
+  `#5191 <https://github.com/pybind/pybind11/pull/5191>`_
+
+* stl.h ``list|set|map_caster`` were made more user friendly: it is no longer
+  necessary to explicitly convert Python iterables to ``tuple()``, ``set()``,
+  or ``map()`` in many common situations.
+  `#4686 <https://github.com/pybind/pybind11/pull/4686>`_
+
+* Support for CMake older than 3.15 removed. CMake 3.15-3.30 supported.
+  `#5304 <https://github.com/pybind/pybind11/pull/5304>`_
+
+* The ``array_caster`` in pybind11/stl.h was enhanced to support value types that are not default-constructible.
+  `#5305 <https://github.com/pybind/pybind11/pull/5305>`_
+
+* Added ``py::warnings`` namespace with ``py::warnings::warn`` and ``py::warnings::new_warning_type`` that provides the interface for Python warnings.
+  `#5291 <https://github.com/pybind/pybind11/pull/5291>`_
+
+Version 2.13.6 (September 13, 2024)
+-----------------------------------
+
+New Features:
+
+* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all
+  ``py::class_``-wrapped types, to enable type-safe interoperability between
+  different independent Python/C++ bindings systems, including pybind11
+  versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on
+  pybind11 2.11.2, 2.12.1, and 2.13.6+.
+  `#5296 <https://github.com/pybind/pybind11/pull/5296>`_
+
+
+Bug fixes:
+
+* Using ``__cpp_nontype_template_args`` instead of ``__cpp_nontype_template_parameter_class``.
+  `#5330 <https://github.com/pybind/pybind11/pull/5330>`_
+
+* Properly translate C++ exception to Python exception when creating Python buffer from wrapped object.
+  `#5324 <https://github.com/pybind/pybind11/pull/5324>`_
+
+
+Documentation:
+
+* Adds an answer (FAQ) for "What is a highly conclusive and simple way to find memory leaks?".
+  `#5340 <https://github.com/pybind/pybind11/pull/5340>`_
+
+
+Version 2.13.5 (August 22, 2024)
+--------------------------------
+
+Bug fixes:
+
+* Fix includes when using Windows long paths (``\\?\`` prefix).
+  `#5321 <https://github.com/pybind/pybind11/pull/5321>`_
+
+* Support ``-Wpedantic`` in C++20 mode.
+  `#5322 <https://github.com/pybind/pybind11/pull/5322>`_
+
+* Fix and test ``<ranges>`` support for ``py::tuple`` and ``py::list``.
+  `#5314 <https://github.com/pybind/pybind11/pull/5314>`_
+
+Version 2.13.4 (August 14, 2024)
+--------------------------------
+
+Bug fixes:
+
+* Fix paths with spaces, including on Windows.
+  (Replaces regression from `#5302 <https://github.com/pybind/pybind11/pull/5302>`_)
+  `#4874 <https://github.com/pybind/pybind11/pull/4874>`_
+
+Documentation:
+
+* Remove repetitive words.
+  `#5308 <https://github.com/pybind/pybind11/pull/5308>`_
+
+
+Version 2.13.3 (August 13, 2024)
+--------------------------------
+
+Bug fixes:
+
+* Quote paths from pybind11-config
+  `#5302 <https://github.com/pybind/pybind11/pull/5302>`_
+
+
+* Fix typo in Emscripten support when in config mode (CMake)
+  `#5301 <https://github.com/pybind/pybind11/pull/5301>`_
+
+
+Version 2.13.2 (August 13, 2024)
+--------------------------------
+
+New Features:
+
+* A ``pybind11::detail::type_caster_std_function_specializations`` feature was added, to support specializations for
+  ``std::function``'s with return types that require custom to-Python conversion behavior (to primary use case is to catch and
+  convert exceptions).
+  `#4597 <https://github.com/pybind/pybind11/pull/4597>`_
+
+
+Changes:
+
+
+* Use ``PyMutex`` instead of ``std::mutex`` for internal locking in the free-threaded build.
+  `#5219 <https://github.com/pybind/pybind11/pull/5219>`_
+
+* Add a special type annotation for C++ empty tuple.
+  `#5214 <https://github.com/pybind/pybind11/pull/5214>`_
+
+* When compiling for WebAssembly, add the required exception flags (CMake 3.13+).
+  `#5298 <https://github.com/pybind/pybind11/pull/5298>`_
+
+Bug fixes:
+
+* Make ``gil_safe_call_once_and_store`` thread-safe in free-threaded CPython.
+  `#5246 <https://github.com/pybind/pybind11/pull/5246>`_
+
+* A missing ``#include <algorithm>`` in pybind11/typing.h was added to fix build errors (in case user code does not already depend
+  on that include).
+  `#5208 <https://github.com/pybind/pybind11/pull/5208>`_
+
+* Fix regression introduced in #5201 for GCC<10.3 in C++20 mode.
+  `#5205 <https://github.com/pybind/pybind11/pull/5205>`_
+
+
+.. fix(cmake)
+
+* Remove extra = when assigning flto value in the case for Clang in CMake.
+  `#5207 <https://github.com/pybind/pybind11/pull/5207>`_
+
+
+Tests:
+
+* Adding WASM testing to our CI (Pyodide / Emscripten via scikit-build-core).
+  `#4745 <https://github.com/pybind/pybind11/pull/4745>`_
+
+* clang-tidy (in GitHub Actions) was updated from clang 15 to clang 18.
+  `#5272 <https://github.com/pybind/pybind11/pull/5272>`_
+
+
 Version 2.13.1 (June 26, 2024)
 ------------------------------
 
@@ -129,6 +269,18 @@ Other:
 * Update docs and noxfile.
   `#5071 <https://github.com/pybind/pybind11/pull/5071>`_
 
+Version 2.12.1 (September 13, 2024)
+-----------------------------------
+
+New Features:
+
+* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all
+  ``py::class_``-wrapped types, to enable type-safe interoperability between
+  different independent Python/C++ bindings systems, including pybind11
+  versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on
+  pybind11 2.11.2, 2.12.1, and 2.13.6+.
+  `#5296 <https://github.com/pybind/pybind11/pull/5296>`_
+
 
 Version 2.12.0 (March 27, 2024)
 -------------------------------
@@ -304,6 +456,18 @@ Other:
 * An ``assert()`` was added to help Coverty avoid generating a false positive.
   `#4817 <https://github.com/pybind/pybind11/pull/4817>`_
 
+Version 2.11.2 (September 13, 2024)
+-----------------------------------
+
+New Features:
+
+* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all
+  ``py::class_``-wrapped types, to enable type-safe interoperability between
+  different independent Python/C++ bindings systems, including pybind11
+  versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on
+  pybind11 2.11.2, 2.12.1, and 2.13.6+.
+  `#5296 <https://github.com/pybind/pybind11/pull/5296>`_
+
 
 Version 2.11.1 (July 17, 2023)
 ------------------------------

pybind11/docs/classes.rst

@@ -34,11 +34,18 @@ The binding code for ``Pet`` looks as follows:
             .def("getName", &Pet::getName);
     }
 
-:class:`class_` creates bindings for a C++ *class* or *struct*-style data
+``py::class_`` creates bindings for a C++ *class* or *struct*-style data
 structure. :func:`init` is a convenience function that takes the types of a
 constructor's parameters as template arguments and wraps the corresponding
-constructor (see the :ref:`custom_constructors` section for details). An
-interactive Python session demonstrating this example is shown below:
+constructor (see the :ref:`custom_constructors` section for details).
+
+.. note::
+
+    Starting with pybind11v3, it is recommended to include `py::smart_holder`
+    in most situations for safety, especially if you plan to support conversions
+    to C++ smart pointers. See :ref:`smart_holder` for more information.
+
+An interactive Python session demonstrating this example is shown below:
 
 .. code-block:: pycon
 
@@ -258,7 +265,7 @@ inheritance relationship:
 
 There are two different ways of indicating a hierarchical relationship to
 pybind11: the first specifies the C++ base class as an extra template
-parameter of the :class:`class_`:
+parameter of the ``py::class_``:
 
 .. code-block:: cpp
 
@@ -272,7 +279,7 @@ parameter of the :class:`class_`:
         .def("bark", &Dog::bark);
 
 Alternatively, we can also assign a name to the previously bound ``Pet``
-:class:`class_` object and reference it when binding the ``Dog`` class:
+``py::class_`` object and reference it when binding the ``Dog`` class:
 
 .. code-block:: cpp
 
@@ -498,7 +505,7 @@ The binding code for this example looks as follows:
 
 
 To ensure that the nested types ``Kind`` and ``Attributes`` are created within the scope of ``Pet``, the
-``pet`` :class:`class_` instance must be supplied to the :class:`enum_` and :class:`class_`
+``pet`` ``py::class_`` instance must be supplied to the :class:`enum_` and ``py::class_``
 constructor. The :func:`enum_::export_values` function exports the enum entries
 into the parent scope, which should be skipped for newer C++11-style strongly
 typed enums.

pybind11/docs/compiling.rst

@@ -18,14 +18,14 @@ A Python extension module can be created with just a few lines of code:
 
 .. code-block:: cmake
 
-    cmake_minimum_required(VERSION 3.15...3.29)
+    cmake_minimum_required(VERSION 3.15...3.30)
     project(example LANGUAGES CXX)
 
     set(PYBIND11_FINDPYTHON ON)
     find_package(pybind11 CONFIG REQUIRED)
 
     pybind11_add_module(example example.cpp)
-    install(TARGET example DESTINATION .)
+    install(TARGETS example DESTINATION .)
 
 (You use the ``add_subdirectory`` instead, see the example in :ref:`cmake`.) In
 this example, the code is located in a file named :file:`example.cpp`.  Either
@@ -319,11 +319,11 @@ Building with CMake
 
 For C++ codebases that have an existing CMake-based build system, a Python
 extension module can be created with just a few lines of code, as seen above in
-the module section. Pybind11 currently supports a lower minimum if you don't
-use the modern FindPython, though be aware that CMake 3.27 removed the old
-mechanism, so pybind11 will automatically switch if the old mechanism is not
-available. Please opt into the new mechanism if at all possible. Our default
-may change in future versions. This is the minimum required:
+the module section. Pybind11 currently defaults to the old mechanism, though be
+aware that CMake 3.27 removed the old mechanism, so pybind11 will automatically
+switch if the old mechanism is not available. Please opt into the new mechanism
+if at all possible. Our default may change in future versions. This is the
+minimum required:
 
 
 
@@ -333,6 +333,9 @@ may change in future versions. This is the minimum required:
 .. versionchanged:: 2.11
    CMake 3.5+ is required.
 
+.. versionchanged:: 2.14
+   CMake 3.15+ is required.
+
 
 Further information can be found at :doc:`cmake/index`.
 
@@ -388,7 +391,7 @@ that will be respected instead of the built-in flag search.
 
 The ``OPT_SIZE`` flag enables size-based optimization equivalent to the
 standard ``/Os`` or ``-Os`` compiler flags and the ``MinSizeRel`` build type,
-which avoid optimizations that that can substantially increase the size of the
+which avoid optimizations that can substantially increase the size of the
 resulting binary. This flag is particularly useful in projects that are split
 into performance-critical parts and associated bindings. In this case, we can
 compile the project in release mode (and hence, optimize performance globally),
@@ -426,7 +429,7 @@ with ``PYTHON_EXECUTABLE``.  For example:
 
 .. code-block:: bash
 
-    cmake -DPYBIND11_PYTHON_VERSION=3.7 ..
+    cmake -DPYBIND11_PYTHON_VERSION=3.8 ..
 
     # Another method:
     cmake -DPYTHON_EXECUTABLE=/path/to/python ..
@@ -444,7 +447,7 @@ See the `Config file`_ docstring for details of relevant CMake variables.
 
 .. code-block:: cmake
 
-    cmake_minimum_required(VERSION 3.4...3.18)
+    cmake_minimum_required(VERSION 3.15...3.30)
     project(example LANGUAGES CXX)
 
     find_package(pybind11 REQUIRED)
@@ -483,17 +486,16 @@ can refer to the same [cmake_example]_ repository for a full sample project
 FindPython mode
 ---------------
 
-CMake 3.12+ (3.15+ recommended, 3.18.2+ ideal) added a new module called
-FindPython that had a highly improved search algorithm and modern targets
-and tools. If you use FindPython, pybind11 will detect this and use the
-existing targets instead:
+Modern CMake (3.18.2+ ideal) added a new module called FindPython that had a
+highly improved search algorithm and modern targets and tools. If you use
+FindPython, pybind11 will detect this and use the existing targets instead:
 
 .. code-block:: cmake
 
-    cmake_minimum_required(VERSION 3.15...3.22)
+    cmake_minimum_required(VERSION 3.15...3.30)
     project(example LANGUAGES CXX)
 
-    find_package(Python 3.7 COMPONENTS Interpreter Development REQUIRED)
+    find_package(Python 3.8 COMPONENTS Interpreter Development REQUIRED)
     find_package(pybind11 CONFIG REQUIRED)
     # or add_subdirectory(pybind11)
 
@@ -541,7 +543,7 @@ available in all modes. The targets provided are:
      Just the "linking" part of pybind11:module
 
    ``pybind11::module``
-     Everything for extension modules - ``pybind11::pybind11`` + ``Python::Module`` (FindPython CMake 3.15+) or ``pybind11::python_link_helper``
+     Everything for extension modules - ``pybind11::pybind11`` + ``Python::Module`` (FindPython) or ``pybind11::python_link_helper``
 
    ``pybind11::embed``
      Everything for embedding the Python interpreter - ``pybind11::pybind11`` + ``Python::Python`` (FindPython) or Python libs
@@ -568,7 +570,7 @@ You can use these targets to build complex applications. For example, the
 
 .. code-block:: cmake
 
-    cmake_minimum_required(VERSION 3.5...3.29)
+    cmake_minimum_required(VERSION 3.15...3.30)
     project(example LANGUAGES CXX)
 
     find_package(pybind11 REQUIRED)  # or add_subdirectory(pybind11)
@@ -626,7 +628,7 @@ information about usage in C++, see :doc:`/advanced/embedding`.
 
 .. code-block:: cmake
 
-    cmake_minimum_required(VERSION 3.5...3.29)
+    cmake_minimum_required(VERSION 3.15...3.30)
     project(example LANGUAGES CXX)
 
     find_package(pybind11 REQUIRED)  # or add_subdirectory(pybind11)
@@ -719,7 +721,7 @@ customizable pybind11-based wrappers by parsing C++ header files.
 
 [litgen]_ is an automatic python bindings generator with a focus on generating
 documented and discoverable bindings: bindings will nicely reproduce the documentation
-found in headers. It is is based on srcML (srcml.org), a highly scalable, multi-language
+found in headers. It is based on srcML (srcml.org), a highly scalable, multi-language
 parsing tool with a developer centric approach. The API that you want to expose to python
 must be C++14 compatible (but your implementation can use more modern constructs).
 

pybind11/docs/faq.rst

@@ -247,6 +247,50 @@ been received, you must either explicitly interrupt execution by throwing
         });
     }
 
+What is a highly conclusive and simple way to find memory leaks (e.g. in pybind11 bindings)?
+============================================================================================
+
+Use ``while True`` & ``top`` (Linux, macOS).
+
+For example, locally change tests/test_type_caster_pyobject_ptr.py like this:
+
+.. code-block:: diff
+
+     def test_return_list_pyobject_ptr_reference():
+    +  while True:
+         vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder)
+         assert [e.value for e in vec_obj] == [93, 186]
+         # Commenting out the next `assert` will leak the Python references.
+         # An easy way to see evidence of the leaks:
+         # Insert `while True:` as the first line of this function and monitor the
+         # process RES (Resident Memory Size) with the Unix top command.
+    -    assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2
+    +    # assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2
+
+Then run the test as you would normally do, which will go into the infinite loop.
+
+**In another shell, but on the same machine** run:
+
+.. code-block:: bash
+
+    top
+
+This will show:
+
+.. code-block::
+
+        PID USER      PR  NI    VIRT    RES    SHR S  %CPU  %MEM     TIME+ COMMAND
+    1266095 rwgk      20   0 5207496 611372  45696 R 100.0   0.3   0:08.01 test_type_caste
+
+Look for the number under ``RES`` there. You'll see it going up very quickly.
+
+**Don't forget to Ctrl-C the test command** before your machine becomes
+unresponsive due to swapping.
+
+This method only takes a couple minutes of effort and is very conclusive.
+What you want to see is that the ``RES`` number is stable after a couple
+seconds.
+
 CMake doesn't detect the right Python version
 =============================================
 
@@ -258,9 +302,9 @@ CMake configure line. (Replace ``$(which python)`` with a path to python if
 your prefer.)
 
 You can alternatively try ``-DPYBIND11_FINDPYTHON=ON``, which will activate the
-new CMake FindPython support instead of pybind11's custom search. Requires
-CMake 3.12+, and 3.15+ or 3.18.2+ are even better. You can set this in your
-``CMakeLists.txt`` before adding or finding pybind11, as well.
+new CMake FindPython support instead of pybind11's custom search. Newer CMake,
+like, 3.18.2+, is recommended. You can set this in your ``CMakeLists.txt``
+before adding or finding pybind11, as well.
 
 Inconsistent detection of Python version in CMake and pybind11
 ==============================================================
@@ -281,11 +325,11 @@ There are three possible solutions:
    from CMake and rely on pybind11 in detecting Python version. If this is not
    possible, the CMake machinery should be called *before* including pybind11.
 2. Set ``PYBIND11_FINDPYTHON`` to ``True`` or use ``find_package(Python
-   COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better,
-   3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead
-   of the old, deprecated search tools, and these modules are much better at
-   finding the correct Python. If FindPythonLibs/Interp are not available
-   (CMake 3.27+), then this will be ignored and FindPython will be used.
+   COMPONENTS Interpreter Development)`` on modern CMake ( 3.18.2+ best).
+   Pybind11 in these cases uses the new CMake FindPython instead of the old,
+   deprecated search tools, and these modules are much better at finding the
+   correct Python. If FindPythonLibs/Interp are not available (CMake 3.27+),
+   then this will be ignored and FindPython will be used.
 3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python.
    However, you will have to use the target-based system, and do more setup
    yourself, because it does not know about or include things that depend on

pybind11/docs/limitations.rst

@@ -50,10 +50,6 @@ clean, well written patch would likely be accepted to solve them.
   One consequence is that containers of ``char *`` are currently not supported.
   `#2245 <https://github.com/pybind/pybind11/issues/2245>`_
 
-- The ``cpptest`` does not run on Windows with Python 3.8 or newer, due to DLL
-  loader changes. User code that is correctly installed should not be affected.
-  `#2560 <https://github.com/pybind/pybind11/issue/2560>`_
-
 Python 3.9.0 warning
 ^^^^^^^^^^^^^^^^^^^^
 

pybind11/docs/reference.rst

@@ -68,8 +68,8 @@ Convenience functions converting to Python types
 
 .. _extras:
 
-Passing extra arguments to ``def`` or ``class_``
-================================================
+Passing extra arguments to ``def`` or ``py::class_``
+====================================================
 
 .. doxygengroup:: annotations
     :members:

pybind11/docs/requirements.txt

@@ -16,9 +16,9 @@ breathe==4.35.0 \
     --hash=sha256:5165541c3c67b6c7adde8b3ecfe895c6f7844783c4076b6d8d287e4f33d62386 \
     --hash=sha256:52c581f42ca4310737f9e435e3851c3d1f15446205a85fbc272f1f97ed74f5be
     # via -r requirements.in
-certifi==2024.2.2 \
-    --hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \
-    --hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1
+certifi==2024.7.4 \
+    --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \
+    --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90
     # via requests
 charset-normalizer==3.3.2 \
     --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \
@@ -130,9 +130,9 @@ imagesize==1.4.1 \
     --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \
     --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a
     # via sphinx
-jinja2==3.1.4 \
-    --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
-    --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
+jinja2==3.1.6 \
+    --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \
+    --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
     # via sphinx
 markupsafe==2.1.5 \
     --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \

pybind11/docs/upgrade.rst

@@ -24,7 +24,8 @@ changes are that:
   function is not available anymore.
 
 Due to NumPy changes, you may experience difficulties updating to NumPy 2.
-Please see the [NumPy 2 migration guide](https://numpy.org/devdocs/numpy_2_0_migration_guide.html) for details.
+Please see the `NumPy 2 migration guide <https://numpy.org/devdocs/numpy_2_0_migration_guide.html>`_
+for details.
 For example, a more direct change could be that the default integer ``"int_"``
 (and ``"uint"``) is now ``ssize_t`` and not ``long`` (affects 64bit windows).
 

pybind11/include/pybind11/attr.h

@@ -81,6 +81,10 @@ struct dynamic_attr {};
 /// Annotation which enables the buffer protocol for a type
 struct buffer_protocol {};
 
+/// Annotation which enables releasing the GIL before calling the C++ destructor of wrapped
+/// instances (pybind/pybind11#1446).
+struct release_gil_before_calling_cpp_dtor {};
+
 /// Annotation which requests that a special metaclass is created for a type
 struct metaclass {
     handle value;
@@ -272,7 +276,7 @@ struct function_record {
 struct type_record {
     PYBIND11_NOINLINE type_record()
         : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false),
-          default_holder(true), module_local(false), is_final(false) {}
+          module_local(false), is_final(false), release_gil_before_calling_cpp_dtor(false) {}
 
     /// Handle to the parent scope
     handle scope;
@@ -322,15 +326,17 @@ struct type_record {
     /// Does the class implement the buffer protocol?
     bool buffer_protocol : 1;
 
-    /// Is the default (unique_ptr) holder type used?
-    bool default_holder : 1;
-
     /// Is the class definition local to the module shared object?
     bool module_local : 1;
 
     /// Is the class inheritable from python classes?
     bool is_final : 1;
 
+    /// Solves pybind/pybind11#1446
+    bool release_gil_before_calling_cpp_dtor : 1;
+
+    holder_enum_t holder_enum_v = holder_enum_t::undefined;
+
     PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) {
         auto *base_info = detail::get_type_info(base, false);
         if (!base_info) {
@@ -340,18 +346,22 @@ struct type_record {
                           + "\" referenced unknown base type \"" + tname + "\"");
         }
 
-        if (default_holder != base_info->default_holder) {
+        // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks.
+        bool this_has_unique_ptr_holder = (holder_enum_v == holder_enum_t::std_unique_ptr);
+        bool base_has_unique_ptr_holder
+            = (base_info->holder_enum_v == holder_enum_t::std_unique_ptr);
+        if (this_has_unique_ptr_holder != base_has_unique_ptr_holder) {
             std::string tname(base.name());
             detail::clean_type_id(tname);
             pybind11_fail("generic_type: type \"" + std::string(name) + "\" "
-                          + (default_holder ? "does not have" : "has")
+                          + (this_has_unique_ptr_holder ? "does not have" : "has")
                           + " a non-default holder type while its base \"" + tname + "\" "
-                          + (base_info->default_holder ? "does not" : "does"));
+                          + (base_has_unique_ptr_holder ? "does not" : "does"));
         }
 
         bases.append((PyObject *) base_info->type);
 
-#if PY_VERSION_HEX < 0x030B0000
+#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET
         dynamic_attr |= base_info->type->tp_dictoffset != 0;
 #else
         dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0;
@@ -603,6 +613,14 @@ struct process_attribute<module_local> : process_attribute_default<module_local>
     static void init(const module_local &l, type_record *r) { r->module_local = l.value; }
 };
 
+template <>
+struct process_attribute<release_gil_before_calling_cpp_dtor>
+    : process_attribute_default<release_gil_before_calling_cpp_dtor> {
+    static void init(const release_gil_before_calling_cpp_dtor &, type_record *r) {
+        r->release_gil_before_calling_cpp_dtor = true;
+    }
+};
+
 /// Process a 'prepend' attribute, putting this at the beginning of the overload chain
 template <>
 struct process_attribute<prepend> : process_attribute_default<prepend> {

pybind11/include/pybind11/cast.h

@@ -158,7 +158,7 @@ public:
         } else {
             handle src_or_index = src;
             // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls.
-#if PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION)
+#if defined(PYPY_VERSION)
             object index;
             if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr())
                 index = reinterpret_steal<object>(PyNumber_Index(src.ptr()));
@@ -343,7 +343,7 @@ public:
 #else
             // Alternate approach for CPython: this does the same as the above, but optimized
             // using the CPython API so as to avoid an unneeded attribute lookup.
-            else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) {
+            else if (auto *tp_as_number = Py_TYPE(src.ptr())->tp_as_number) {
                 if (PYBIND11_NB_BOOL(tp_as_number)) {
                     res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr());
                 }
@@ -740,6 +740,13 @@ class type_caster<std::pair<T1, T2>> : public tuple_caster<std::pair, T1, T2> {}
 template <typename... Ts>
 class type_caster<std::tuple<Ts...>> : public tuple_caster<std::tuple, Ts...> {};
 
+template <>
+class type_caster<std::tuple<>> : public tuple_caster<std::tuple> {
+public:
+    // PEP 484 specifies this syntax for an empty tuple
+    static constexpr auto name = const_name("tuple[()]");
+};
+
 /// Helper class which abstracts away certain actions. Users can provide specializations for
 /// custom holders, but it's only necessary if the type has a non-standard interface.
 template <typename T>
@@ -747,6 +754,7 @@ struct holder_helper {
     static auto get(const T &p) -> decltype(p.get()) { return p.get(); }
 };
 
+// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to shared_ptr specialization.
 /// Type caster for holder types like std::shared_ptr, etc.
 /// The SFINAE hook is provided to help work around the current lack of support
 /// for smart-pointer interoperability. Please consider it an implementation
@@ -782,16 +790,19 @@ public:
 protected:
     friend class type_caster_generic;
     void check_holder_compat() {
-        if (typeinfo->default_holder) {
+        // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks.
+        bool inst_has_unique_ptr_holder
+            = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr);
+        if (inst_has_unique_ptr_holder) {
             throw cast_error("Unable to load a custom holder type from a default-holder instance");
         }
     }
 
-    bool load_value(value_and_holder &&v_h) {
+    void load_value(value_and_holder &&v_h) {
         if (v_h.holder_constructed()) {
             value = v_h.value_ptr();
             holder = v_h.template holder<holder_type>();
-            return true;
+            return;
         }
         throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) "
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
@@ -828,10 +839,144 @@ protected:
     holder_type holder;
 };
 
+template <typename, typename SFINAE = void>
+struct copyable_holder_caster_shared_ptr_with_smart_holder_support_enabled : std::true_type {};
+
+// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor copyable_holder_caster to reduce code duplication.
+template <typename type>
+struct copyable_holder_caster<
+    type,
+    std::shared_ptr<type>,
+    enable_if_t<copyable_holder_caster_shared_ptr_with_smart_holder_support_enabled<type>::value>>
+    : public type_caster_base<type> {
+public:
+    using base = type_caster_base<type>;
+    static_assert(std::is_base_of<base, type_caster<type>>::value,
+                  "Holder classes are only supported for custom types");
+    using base::base;
+    using base::cast;
+    using base::typeinfo;
+    using base::value;
+
+    bool load(handle src, bool convert) {
+        if (base::template load_impl<copyable_holder_caster<type, std::shared_ptr<type>>>(
+                src, convert)) {
+            sh_load_helper.maybe_set_python_instance_is_alias(src);
+            return true;
+        }
+        return false;
+    }
+
+    explicit operator std::shared_ptr<type> *() {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            pybind11_fail("Passing `std::shared_ptr<T> *` from Python to C++ is not supported "
+                          "(inherently unsafe).");
+        }
+        return std::addressof(shared_ptr_storage);
+    }
+
+    explicit operator std::shared_ptr<type> &() {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            shared_ptr_storage = sh_load_helper.load_as_shared_ptr(value);
+        }
+        return shared_ptr_storage;
+    }
+
+    static handle
+    cast(const std::shared_ptr<type> &src, return_value_policy policy, handle parent) {
+        const auto *ptr = src.get();
+        auto st = type_caster_base<type>::src_and_type(ptr);
+        if (st.second == nullptr) {
+            return handle(); // no type info: error will be set already
+        }
+        if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            return smart_holder_type_caster_support::smart_holder_from_shared_ptr(
+                src, policy, parent, st);
+        }
+        return type_caster_base<type>::cast_holder(ptr, &src);
+    }
+
+    // This function will succeed even if the `responsible_parent` does not own the
+    // wrapped C++ object directly.
+    // It is the responsibility of the caller to ensure that the `responsible_parent`
+    // has a `keep_alive` relationship with the owner of the wrapped C++ object, or
+    // that the wrapped C++ object lives for the duration of the process.
+    static std::shared_ptr<type> shared_ptr_with_responsible_parent(handle responsible_parent) {
+        copyable_holder_caster loader;
+        loader.load(responsible_parent, /*convert=*/false);
+        assert(loader.typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder);
+        return loader.sh_load_helper.load_as_shared_ptr(loader.value, responsible_parent);
+    }
+
+protected:
+    friend class type_caster_generic;
+    void check_holder_compat() {
+        // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks.
+        bool inst_has_unique_ptr_holder
+            = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr);
+        if (inst_has_unique_ptr_holder) {
+            throw cast_error("Unable to load a custom holder type from a default-holder instance");
+        }
+    }
+
+    void load_value(value_and_holder &&v_h) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            sh_load_helper.loaded_v_h = v_h;
+            sh_load_helper.was_populated = true;
+            value = sh_load_helper.get_void_ptr_or_nullptr();
+            return;
+        }
+        if (v_h.holder_constructed()) {
+            value = v_h.value_ptr();
+            shared_ptr_storage = v_h.template holder<std::shared_ptr<type>>();
+            return;
+        }
+        throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) "
+#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
+                         "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for "
+                         "type information)");
+#else
+                         "of type '"
+                         + type_id<std::shared_ptr<type>>() + "''");
+#endif
+    }
+
+    template <typename T = std::shared_ptr<type>,
+              detail::enable_if_t<!std::is_constructible<T, const T &, type *>::value, int> = 0>
+    bool try_implicit_casts(handle, bool) {
+        return false;
+    }
+
+    template <typename T = std::shared_ptr<type>,
+              detail::enable_if_t<std::is_constructible<T, const T &, type *>::value, int> = 0>
+    bool try_implicit_casts(handle src, bool convert) {
+        for (auto &cast : typeinfo->implicit_casts) {
+            copyable_holder_caster sub_caster(*cast.first);
+            if (sub_caster.load(src, convert)) {
+                value = cast.second(sub_caster.value);
+                if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+                    sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h;
+                } else {
+                    shared_ptr_storage
+                        = std::shared_ptr<type>(sub_caster.shared_ptr_storage, (type *) value);
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+
+    static bool try_direct_conversions(handle) { return false; }
+
+    smart_holder_type_caster_support::load_helper<remove_cv_t<type>> sh_load_helper; // Const2Mutbl
+    std::shared_ptr<type> shared_ptr_storage;
+};
+
 /// Specialize for the common std::shared_ptr, so users don't need to
 template <typename T>
 class type_caster<std::shared_ptr<T>> : public copyable_holder_caster<T, std::shared_ptr<T>> {};
 
+// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to unique_ptr specialization.
 /// Type caster for holder types like std::unique_ptr.
 /// Please consider the SFINAE hook an implementation detail, as explained
 /// in the comment for the copyable_holder_caster.
@@ -847,6 +992,143 @@ struct move_only_holder_caster {
     static constexpr auto name = type_caster_base<type>::name;
 };
 
+template <typename, typename SFINAE = void>
+struct move_only_holder_caster_unique_ptr_with_smart_holder_support_enabled : std::true_type {};
+
+// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor move_only_holder_caster to reduce code duplication.
+template <typename type, typename deleter>
+struct move_only_holder_caster<
+    type,
+    std::unique_ptr<type, deleter>,
+    enable_if_t<move_only_holder_caster_unique_ptr_with_smart_holder_support_enabled<type>::value>>
+    : public type_caster_base<type> {
+public:
+    using base = type_caster_base<type>;
+    static_assert(std::is_base_of<base, type_caster<type>>::value,
+                  "Holder classes are only supported for custom types");
+    using base::base;
+    using base::cast;
+    using base::typeinfo;
+    using base::value;
+
+    static handle
+    cast(std::unique_ptr<type, deleter> &&src, return_value_policy policy, handle parent) {
+        auto *ptr = src.get();
+        auto st = type_caster_base<type>::src_and_type(ptr);
+        if (st.second == nullptr) {
+            return handle(); // no type info: error will be set already
+        }
+        if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            return smart_holder_type_caster_support::smart_holder_from_unique_ptr(
+                std::move(src), policy, parent, st);
+        }
+        return type_caster_generic::cast(st.first,
+                                         return_value_policy::take_ownership,
+                                         {},
+                                         st.second,
+                                         nullptr,
+                                         nullptr,
+                                         std::addressof(src));
+    }
+
+    static handle
+    cast(const std::unique_ptr<type, deleter> &src, return_value_policy policy, handle parent) {
+        if (!src) {
+            return none().release();
+        }
+        if (policy == return_value_policy::automatic) {
+            policy = return_value_policy::reference_internal;
+        }
+        if (policy != return_value_policy::reference_internal) {
+            throw cast_error("Invalid return_value_policy for const unique_ptr&");
+        }
+        return type_caster_base<type>::cast(src.get(), policy, parent);
+    }
+
+    bool load(handle src, bool convert) {
+        if (base::template load_impl<
+                move_only_holder_caster<type, std::unique_ptr<type, deleter>>>(src, convert)) {
+            sh_load_helper.maybe_set_python_instance_is_alias(src);
+            return true;
+        }
+        return false;
+    }
+
+    void load_value(value_and_holder &&v_h) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            sh_load_helper.loaded_v_h = v_h;
+            sh_load_helper.loaded_v_h.type = typeinfo;
+            sh_load_helper.was_populated = true;
+            value = sh_load_helper.get_void_ptr_or_nullptr();
+            return;
+        }
+        pybind11_fail("Passing `std::unique_ptr<T>` from Python to C++ requires `py::class_<T, "
+                      "py::smart_holder>` (with T = "
+                      + clean_type_id(typeinfo->cpptype->name()) + ")");
+    }
+
+    template <typename T_>
+    using cast_op_type
+        = conditional_t<std::is_same<typename std::remove_volatile<T_>::type,
+                                     const std::unique_ptr<type, deleter> &>::value
+                            || std::is_same<typename std::remove_volatile<T_>::type,
+                                            const std::unique_ptr<const type, deleter> &>::value,
+                        const std::unique_ptr<type, deleter> &,
+                        std::unique_ptr<type, deleter>>;
+
+    explicit operator std::unique_ptr<type, deleter>() {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            return sh_load_helper.template load_as_unique_ptr<deleter>(value);
+        }
+        pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__));
+    }
+
+    explicit operator const std::unique_ptr<type, deleter> &() {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            // Get shared_ptr to ensure that the Python object is not disowned elsewhere.
+            shared_ptr_storage = sh_load_helper.load_as_shared_ptr(value);
+            // Build a temporary unique_ptr that is meant to never expire.
+            unique_ptr_storage = std::shared_ptr<std::unique_ptr<type, deleter>>(
+                new std::unique_ptr<type, deleter>{
+                    sh_load_helper.template load_as_const_unique_ptr<deleter>(
+                        shared_ptr_storage.get())},
+                [](std::unique_ptr<type, deleter> *ptr) {
+                    if (!ptr) {
+                        pybind11_fail("FATAL: `const std::unique_ptr<T, D> &` was disowned "
+                                      "(EXPECT UNDEFINED BEHAVIOR).");
+                    }
+                    (void) ptr->release();
+                    delete ptr;
+                });
+            return *unique_ptr_storage;
+        }
+        pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__));
+    }
+
+    bool try_implicit_casts(handle src, bool convert) {
+        for (auto &cast : typeinfo->implicit_casts) {
+            move_only_holder_caster sub_caster(*cast.first);
+            if (sub_caster.load(src, convert)) {
+                value = cast.second(sub_caster.value);
+                if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+                    sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h;
+                } else {
+                    pybind11_fail("Expected to be UNREACHABLE: " __FILE__
+                                  ":" PYBIND11_TOSTRING(__LINE__));
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+
+    static bool try_direct_conversions(handle) { return false; }
+
+    smart_holder_type_caster_support::load_helper<remove_cv_t<type>> sh_load_helper; // Const2Mutbl
+    std::shared_ptr<type> shared_ptr_storage; // Serves as a pseudo lock.
+    std::shared_ptr<std::unique_ptr<type, deleter>> unique_ptr_storage;
+};
+
 template <typename type, typename deleter>
 class type_caster<std::unique_ptr<type, deleter>>
     : public move_only_holder_caster<type, std::unique_ptr<type, deleter>> {};
@@ -856,18 +1138,20 @@ using type_caster_holder = conditional_t<is_copy_constructible<holder_type>::val
                                          copyable_holder_caster<type, holder_type>,
                                          move_only_holder_caster<type, holder_type>>;
 
-template <typename T, bool Value = false>
-struct always_construct_holder {
+template <bool Value = false>
+struct always_construct_holder_value {
     static constexpr bool value = Value;
 };
 
+template <typename T, bool Value = false>
+struct always_construct_holder : always_construct_holder_value<Value> {};
+
 /// Create a specialization for custom holder types (silently ignores std::shared_ptr)
 #define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...)                                      \
     PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)                                                  \
     namespace detail {                                                                            \
     template <typename type>                                                                      \
-    struct always_construct_holder<holder_type> : always_construct_holder<void, ##__VA_ARGS__> {  \
-    };                                                                                            \
+    struct always_construct_holder<holder_type> : always_construct_holder_value<__VA_ARGS__> {};  \
     template <typename type>                                                                      \
     class type_caster<holder_type, enable_if_t<!is_shared_ptr<holder_type>::value>>               \
         : public type_caster_holder<type, holder_type> {};                                        \
@@ -878,10 +1162,14 @@ struct always_construct_holder {
 template <typename base, typename holder>
 struct is_holder_type
     : std::is_base_of<detail::type_caster_holder<base, holder>, detail::type_caster<holder>> {};
-// Specialization for always-supported unique_ptr holders:
+
+// Specializations for always-supported holders:
 template <typename base, typename deleter>
 struct is_holder_type<base, std::unique_ptr<base, deleter>> : std::true_type {};
 
+template <typename base>
+struct is_holder_type<base, smart_holder> : std::true_type {};
+
 #ifdef PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION // See PR #4888
 
 // This leads to compilation errors if a specialization is missing.
@@ -1005,10 +1293,18 @@ template <>
 struct handle_type_name<args> {
     static constexpr auto name = const_name("*args");
 };
+template <typename T>
+struct handle_type_name<Args<T>> {
+    static constexpr auto name = const_name("*args: ") + make_caster<T>::name;
+};
 template <>
 struct handle_type_name<kwargs> {
     static constexpr auto name = const_name("**kwargs");
 };
+template <typename T>
+struct handle_type_name<KWArgs<T>> {
+    static constexpr auto name = const_name("**kwargs: ") + make_caster<T>::name;
+};
 template <>
 struct handle_type_name<obj_attr_accessor> {
     static constexpr auto name = const_name<obj_attr_accessor>();
@@ -1314,6 +1610,31 @@ object object_or_cast(T &&o) {
     return pybind11::cast(std::forward<T>(o));
 }
 
+// Declared in pytypes.h:
+// Implemented here so that make_caster<T> can be used.
+template <typename D>
+template <typename T>
+str_attr_accessor object_api<D>::attr_with_type_hint(const char *key) const {
+#if !defined(__cpp_inline_variables)
+    static_assert(always_false<T>::value,
+                  "C++17 feature __cpp_inline_variables not available: "
+                  "https://en.cppreference.com/w/cpp/language/static#Static_data_members");
+#endif
+    object ann = annotations();
+    if (ann.contains(key)) {
+        throw std::runtime_error("__annotations__[\"" + std::string(key) + "\"] was set already.");
+    }
+    ann[key] = make_caster<T>::name.text;
+    return {derived(), key};
+}
+
+template <typename D>
+template <typename T>
+obj_attr_accessor object_api<D>::attr_with_type_hint(handle key) const {
+    (void) attr_with_type_hint<T>(key.cast<std::string>().c_str());
+    return {derived(), reinterpret_borrow<object>(key)};
+}
+
 // Placeholder type for the unneeded (and dead code) static variable in the
 // PYBIND11_OVERRIDE_OVERRIDE macro
 struct override_unused {};
@@ -1496,7 +1817,7 @@ struct kw_only {};
 
 /// \ingroup annotations
 /// Annotation indicating that all previous arguments are positional-only; the is the equivalent of
-/// an unnamed '/' argument (in Python 3.8)
+/// an unnamed '/' argument
 struct pos_only {};
 
 template <typename T>
@@ -1557,15 +1878,24 @@ struct function_call {
     handle init_self;
 };
 
+// See PR #5396 for the discussion that led to this
+template <typename Base, typename Derived, typename = void>
+struct is_same_or_base_of : std::is_same<Base, Derived> {};
+
+// Only evaluate is_base_of if Derived is complete.
+// is_base_of raises a compiler error if Derived is incomplete.
+template <typename Base, typename Derived>
+struct is_same_or_base_of<Base, Derived, decltype(void(sizeof(Derived)))>
+    : any_of<std::is_same<Base, Derived>, std::is_base_of<Base, Derived>> {};
+
 /// Helper class which loads arguments for C++ functions called from Python
 template <typename... Args>
 class argument_loader {
     using indices = make_index_sequence<sizeof...(Args)>;
-
     template <typename Arg>
-    using argument_is_args = std::is_same<intrinsic_t<Arg>, args>;
+    using argument_is_args = is_same_or_base_of<args, intrinsic_t<Arg>>;
     template <typename Arg>
-    using argument_is_kwargs = std::is_same<intrinsic_t<Arg>, kwargs>;
+    using argument_is_kwargs = is_same_or_base_of<kwargs, intrinsic_t<Arg>>;
     // Get kwargs argument position, or -1 if not present:
     static constexpr auto kwargs_pos = constexpr_last<argument_is_kwargs, Args...>();
 

pybind11/include/pybind11/conduit/README.txt

@@ -0,0 +1,15 @@
+NOTE
+----
+
+The C++ code here
+
+** only depends on <Python.h> **
+
+and nothing else.
+
+DO NOT ADD CODE WITH OTHER EXTERNAL DEPENDENCIES TO THIS DIRECTORY.
+
+Read on:
+
+pybind11_conduit_v1.h — Type-safe interoperability between different
+                        independent Python/C++ bindings systems.

pybind11/include/pybind11/conduit/pybind11_conduit_v1.h

@@ -0,0 +1,111 @@
+// Copyright (c) 2024 The pybind Community.
+
+/* The pybind11_conduit_v1 feature enables type-safe interoperability between
+
+* different independent Python/C++ bindings systems,
+
+* including pybind11 versions with different PYBIND11_INTERNALS_VERSION's.
+
+The naming of the feature is a bit misleading:
+
+* The feature is in no way tied to pybind11 internals.
+
+* It just happens to originate from pybind11 and currently still lives there.
+
+* The only external dependency is <Python.h>.
+
+The implementation is a VERY light-weight dependency. It is designed to be
+compatible with any ISO C++11 (or higher) compiler, and does NOT require
+C++ Exception Handling to be enabled.
+
+Please see https://github.com/pybind/pybind11/pull/5296 for more background.
+
+The implementation involves a
+
+def _pybind11_conduit_v1_(
+    self,
+    pybind11_platform_abi_id: bytes,
+    cpp_type_info_capsule: capsule,
+    pointer_kind: bytes) -> capsule
+
+method that is meant to be added to Python objects wrapping C++ objects
+(e.g. pybind11::class_-wrapped types).
+
+The design of the _pybind11_conduit_v1_ feature provides two layers of
+protection against C++ ABI mismatches:
+
+* The first and most important layer is that the pybind11_platform_abi_id's
+  must match between extensions. — This will never be perfect, but is the same
+  pragmatic approach used in pybind11 since 2017
+  (https://github.com/pybind/pybind11/commit/96997a4b9d4ec3d389a570604394af5d5eee2557,
+  PYBIND11_INTERNALS_ID).
+
+* The second layer is that the typeid(std::type_info).name()'s must match
+  between extensions.
+
+The implementation below (which is shorter than this comment!), serves as a
+battle-tested specification. The main API is this one function:
+
+auto *cpp_pointer = pybind11_conduit_v1::get_type_pointer_ephemeral<YourType>(py_obj);
+
+It is meant to be a minimalistic reference implementation, intentionally
+without comprehensive error reporting. It is expected that major bindings
+systems will roll their own, compatible implementations, potentially with
+system-specific error reporting. The essential specifications all bindings
+systems need to agree on are merely:
+
+* PYBIND11_PLATFORM_ABI_ID (const char* literal).
+
+* The cpp_type_info capsule (see below: a void *ptr and a const char *name).
+
+* The cpp_conduit capsule (see below: a void *ptr and a const char *name).
+
+* "raw_pointer_ephemeral" means: the lifetime of the pointer is the lifetime
+  of the py_obj.
+
+*/
+
+// THIS MUST STAY AT THE TOP!
+#include "pybind11_platform_abi_id.h"
+
+#include <Python.h>
+#include <typeinfo>
+
+namespace pybind11_conduit_v1 {
+
+inline void *get_raw_pointer_ephemeral(PyObject *py_obj, const std::type_info *cpp_type_info) {
+    PyObject *cpp_type_info_capsule
+        = PyCapsule_New(const_cast<void *>(static_cast<const void *>(cpp_type_info)),
+                        typeid(std::type_info).name(),
+                        nullptr);
+    if (cpp_type_info_capsule == nullptr) {
+        return nullptr;
+    }
+    PyObject *cpp_conduit = PyObject_CallMethod(py_obj,
+                                                "_pybind11_conduit_v1_",
+                                                "yOy",
+                                                PYBIND11_PLATFORM_ABI_ID,
+                                                cpp_type_info_capsule,
+                                                "raw_pointer_ephemeral");
+    Py_DECREF(cpp_type_info_capsule);
+    if (cpp_conduit == nullptr) {
+        return nullptr;
+    }
+    void *raw_ptr = PyCapsule_GetPointer(cpp_conduit, cpp_type_info->name());
+    Py_DECREF(cpp_conduit);
+    if (PyErr_Occurred()) {
+        return nullptr;
+    }
+    return raw_ptr;
+}
+
+template <typename T>
+T *get_type_pointer_ephemeral(PyObject *py_obj) {
+    void *raw_ptr = get_raw_pointer_ephemeral(py_obj, &typeid(T));
+    if (raw_ptr == nullptr) {
+        return nullptr;
+    }
+    return static_cast<T *>(raw_ptr);
+}
+
+} // namespace pybind11_conduit_v1

pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h

@@ -0,0 +1,87 @@
+#pragma once
+
+// Copyright (c) 2024 The pybind Community.
+
+// To maximize reusability:
+// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING.
+
+#include "wrap_include_python_h.h"
+
+// Implementation details. DO NOT USE ELSEWHERE. (Unfortunately we cannot #undef them.)
+// This is duplicated here to maximize portability.
+#define PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x) #x
+#define PYBIND11_PLATFORM_ABI_ID_TOSTRING(x) PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x)
+
+#ifdef PYBIND11_COMPILER_TYPE
+//   // To maintain backward compatibility (see PR #5439).
+#    define PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE ""
+#else
+#    define PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE "_"
+#    if defined(__MINGW32__)
+#        define PYBIND11_COMPILER_TYPE "mingw"
+#    elif defined(__CYGWIN__)
+#        define PYBIND11_COMPILER_TYPE "gcc_cygwin"
+#    elif defined(_MSC_VER)
+#        define PYBIND11_COMPILER_TYPE "msvc"
+#    elif defined(__clang__) || defined(__GNUC__)
+#        define PYBIND11_COMPILER_TYPE "system" // Assumed compatible with system compiler.
+#    else
+#        error "Unknown PYBIND11_COMPILER_TYPE: PLEASE REVISE THIS CODE."
+#    endif
+#endif
+
+// PR #5439 made this macro obsolete. However, there are many manipulations of this macro in the
+// wild. Therefore, to maintain backward compatibility, it is kept around.
+#ifndef PYBIND11_STDLIB
+#    define PYBIND11_STDLIB ""
+#endif
+
+#ifndef PYBIND11_BUILD_ABI
+#    if defined(_MSC_VER)                 // See PR #4953.
+#        if defined(_MT) && defined(_DLL) // Corresponding to CL command line options /MD or /MDd.
+#            if (_MSC_VER) / 100 == 19
+#                define PYBIND11_BUILD_ABI "_md_mscver19"
+#            else
+#                error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE."
+#            endif
+#        elif defined(_MT) // Corresponding to CL command line options /MT or /MTd.
+#            define PYBIND11_BUILD_ABI "_mt_mscver" PYBIND11_PLATFORM_ABI_ID_TOSTRING(_MSC_VER)
+#        else
+#            if (_MSC_VER) / 100 == 19
+#                define PYBIND11_BUILD_ABI "_none_mscver19"
+#            else
+#                error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE."
+#            endif
+#        endif
+#    elif defined(_LIBCPP_ABI_VERSION) // https://libcxx.llvm.org/DesignDocs/ABIVersioning.html
+#        define PYBIND11_BUILD_ABI                                                                \
+            "_libcpp_abi" PYBIND11_PLATFORM_ABI_ID_TOSTRING(_LIBCPP_ABI_VERSION)
+#    elif defined(_GLIBCXX_USE_CXX11_ABI) // See PR #5439.
+#        if defined(__NVCOMPILER)
+//           // Assume that NVHPC is in the 1xxx ABI family.
+//           // THIS ASSUMPTION IS NOT FUTURE PROOF but apparently the best we can do.
+//           // Please let us know if there is a way to validate the assumption here.
+#        elif !defined(__GXX_ABI_VERSION)
+#            error                                                                                \
+                "Unknown platform or compiler (_GLIBCXX_USE_CXX11_ABI): PLEASE REVISE THIS CODE."
+#        endif
+#        if defined(__GXX_ABI_VERSION) && __GXX_ABI_VERSION < 1002 || __GXX_ABI_VERSION >= 2000
+#            error "Unknown platform or compiler (__GXX_ABI_VERSION): PLEASE REVISE THIS CODE."
+#        endif
+#        define PYBIND11_BUILD_ABI                                                                \
+            "_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_" PYBIND11_PLATFORM_ABI_ID_TOSTRING(           \
+                _GLIBCXX_USE_CXX11_ABI)
+#    else
+#        error "Unknown platform or compiler: PLEASE REVISE THIS CODE."
+#    endif
+#endif
+
+// On MSVC, debug and release builds are not ABI-compatible!
+#if defined(_MSC_VER) && defined(_DEBUG)
+#    define PYBIND11_BUILD_TYPE "_debug"
+#else
+#    define PYBIND11_BUILD_TYPE ""
+#endif
+
+#define PYBIND11_PLATFORM_ABI_ID                                                                  \
+    PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE

pybind11/include/pybind11/conduit/wrap_include_python_h.h

@@ -0,0 +1,72 @@
+#pragma once
+
+// Copyright (c) 2024 The pybind Community.
+
+// STRONG REQUIREMENT:
+//   This header is a wrapper around `#include <Python.h>`, therefore it
+//   MUST BE INCLUDED BEFORE ANY STANDARD HEADERS are included.
+// See also:
+//   https://docs.python.org/3/c-api/intro.html#include-files
+// Quoting from there:
+//   Note: Since Python may define some pre-processor definitions which affect
+//   the standard headers on some systems, you must include Python.h before
+//   any standard headers are included.
+
+// To maximize reusability:
+// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING.
+
+// Disable linking to pythonX_d.lib on Windows in debug mode.
+#if defined(_MSC_VER) && defined(_DEBUG) && !defined(Py_DEBUG)
+// Workaround for a VS 2022 issue.
+// See https://github.com/pybind/pybind11/pull/3497 for full context.
+// NOTE: This workaround knowingly violates the Python.h include order
+//       requirement (see above).
+#    include <yvals.h>
+#    if _MSVC_STL_VERSION >= 143
+#        include <crtdefs.h>
+#    endif
+#    define PYBIND11_DEBUG_MARKER
+#    undef _DEBUG
+#endif
+
+// Don't let Python.h #define (v)snprintf as macro because they are implemented
+// properly in Visual Studio since 2015.
+#if defined(_MSC_VER)
+#    define HAVE_SNPRINTF 1
+#endif
+
+#if defined(_MSC_VER)
+#    pragma warning(push)
+#    pragma warning(disable : 4505)
+// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed
+#endif
+
+#include <Python.h>
+#include <frameobject.h>
+#include <pythread.h>
+
+#if defined(_MSC_VER)
+#    pragma warning(pop)
+#endif
+
+#if defined(PYBIND11_DEBUG_MARKER)
+#    define _DEBUG
+#    undef PYBIND11_DEBUG_MARKER
+#endif
+
+// Python #defines overrides on all sorts of core functions, which
+// tends to wreak havok in C++ codebases that expect these to work
+// like regular functions (potentially with several overloads).
+#if defined(isalnum)
+#    undef isalnum
+#    undef isalpha
+#    undef islower
+#    undef isspace
+#    undef isupper
+#    undef tolower
+#    undef toupper
+#endif
+
+#if defined(copysign)
+#    undef copysign
+#endif

pybind11/include/pybind11/detail/class.h

@@ -9,8 +9,10 @@
 
 #pragma once
 
-#include "../attr.h"
-#include "../options.h"
+#include <pybind11/attr.h>
+#include <pybind11/options.h>
+
+#include "exception_translation.h"
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
@@ -310,7 +312,31 @@ inline void traverse_offset_bases(void *valueptr,
     }
 }
 
+#ifdef Py_GIL_DISABLED
+inline void enable_try_inc_ref(PyObject *obj) {
+    // TODO: Replace with PyUnstable_Object_EnableTryIncRef when available.
+    // See https://github.com/python/cpython/issues/128844
+    if (_Py_IsImmortal(obj)) {
+        return;
+    }
+    for (;;) {
+        Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared);
+        if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) {
+            // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states.
+            return;
+        }
+        if (_Py_atomic_compare_exchange_ssize(
+                &obj->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) {
+            return;
+        }
+    }
+}
+#endif
+
 inline bool register_instance_impl(void *ptr, instance *self) {
+#ifdef Py_GIL_DISABLED
+    enable_try_inc_ref(reinterpret_cast<PyObject *>(self));
+#endif
     with_instance_map(ptr, [&](instance_map &instances) { instances.emplace(ptr, self); });
     return true; // unused, but gives the same signature as the deregister func
 }
@@ -431,6 +457,8 @@ inline void clear_instance(PyObject *self) {
             if (instance->owned || v_h.holder_constructed()) {
                 v_h.type->dealloc(v_h);
             }
+        } else if (v_h.holder_constructed()) {
+            v_h.type->dealloc(v_h); // Disowned instance.
         }
     }
     // Deallocate the value/holder layout internals:
@@ -466,19 +494,9 @@ extern "C" inline void pybind11_object_dealloc(PyObject *self) {
 
     type->tp_free(self);
 
-#if PY_VERSION_HEX < 0x03080000
-    // `type->tp_dealloc != pybind11_object_dealloc` means that we're being called
-    // as part of a derived type's dealloc, in which case we're not allowed to decref
-    // the type here. For cross-module compatibility, we shouldn't compare directly
-    // with `pybind11_object_dealloc`, but with the common one stashed in internals.
-    auto pybind11_object_type = (PyTypeObject *) get_internals().instance_base;
-    if (type->tp_dealloc == pybind11_object_type->tp_dealloc)
-        Py_DECREF(type);
-#else
     // This was not needed before Python 3.8 (Python issue 35810)
     // https://github.com/pybind/pybind11/issues/1946
     Py_DECREF(type);
-#endif
 }
 
 std::string error_string();
@@ -558,7 +576,7 @@ extern "C" inline int pybind11_clear(PyObject *self) {
 inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) {
     auto *type = &heap_type->ht_type;
     type->tp_flags |= Py_TPFLAGS_HAVE_GC;
-#if PY_VERSION_HEX < 0x030B0000
+#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET
     type->tp_dictoffset = type->tp_basicsize;           // place dict at the end
     type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it
 #else
@@ -591,31 +609,85 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla
         return -1;
     }
     std::memset(view, 0, sizeof(Py_buffer));
-    buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data);
+    std::unique_ptr<buffer_info> info = nullptr;
+    try {
+        info.reset(tinfo->get_buffer(obj, tinfo->get_buffer_data));
+    } catch (...) {
+        try_translate_exceptions();
+        raise_from(PyExc_BufferError, "Error getting buffer");
+        return -1;
+    }
+    if (info == nullptr) {
+        pybind11_fail("FATAL UNEXPECTED SITUATION: tinfo->get_buffer() returned nullptr.");
+    }
+
     if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) {
-        delete info;
         // view->obj = nullptr;  // Was just memset to 0, so not necessary
         set_error(PyExc_BufferError, "Writable buffer requested for readonly storage");
         return -1;
     }
-    view->obj = obj;
-    view->ndim = 1;
-    view->internal = info;
-    view->buf = info->ptr;
+
+    // Fill in all the information, and then downgrade as requested by the caller, or raise an
+    // error if that's not possible.
     view->itemsize = info->itemsize;
     view->len = view->itemsize;
     for (auto s : info->shape) {
         view->len *= s;
     }
+    view->ndim = static_cast<int>(info->ndim);
+    view->shape = info->shape.data();
+    view->strides = info->strides.data();
     view->readonly = static_cast<int>(info->readonly);
     if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
         view->format = const_cast<char *>(info->format.c_str());
     }
-    if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) {
-        view->ndim = (int) info->ndim;
-        view->strides = info->strides.data();
-        view->shape = info->shape.data();
+
+    // Note, all contiguity flags imply PyBUF_STRIDES and lower.
+    if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) {
+        if (PyBuffer_IsContiguous(view, 'C') == 0) {
+            std::memset(view, 0, sizeof(Py_buffer));
+            set_error(PyExc_BufferError,
+                      "C-contiguous buffer requested for discontiguous storage");
+            return -1;
+        }
+    } else if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) {
+        if (PyBuffer_IsContiguous(view, 'F') == 0) {
+            std::memset(view, 0, sizeof(Py_buffer));
+            set_error(PyExc_BufferError,
+                      "Fortran-contiguous buffer requested for discontiguous storage");
+            return -1;
+        }
+    } else if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS) {
+        if (PyBuffer_IsContiguous(view, 'A') == 0) {
+            std::memset(view, 0, sizeof(Py_buffer));
+            set_error(PyExc_BufferError, "Contiguous buffer requested for discontiguous storage");
+            return -1;
+        }
+
+    } else if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES) {
+        // If no strides are requested, the buffer must be C-contiguous.
+        // https://docs.python.org/3/c-api/buffer.html#contiguity-requests
+        if (PyBuffer_IsContiguous(view, 'C') == 0) {
+            std::memset(view, 0, sizeof(Py_buffer));
+            set_error(PyExc_BufferError,
+                      "C-contiguous buffer requested for discontiguous storage");
+            return -1;
+        }
+
+        view->strides = nullptr;
+
+        // Since this is a contiguous buffer, it can also pretend to be 1D.
+        if ((flags & PyBUF_ND) != PyBUF_ND) {
+            view->shape = nullptr;
+            view->ndim = 0;
+        }
     }
+
+    // Set these after all checks so they don't leak out into the caller, and can be automatically
+    // cleaned up on error.
+    view->buf = info->ptr;
+    view->internal = info.release();
+    view->obj = obj;
     Py_INCREF(view->obj);
     return 0;
 }

pybind11/include/pybind11/detail/common.h

@@ -9,13 +9,18 @@
 
 #pragma once
 
-#define PYBIND11_VERSION_MAJOR 2
-#define PYBIND11_VERSION_MINOR 13
-#define PYBIND11_VERSION_PATCH 1
+#include <pybind11/conduit/wrap_include_python_h.h>
+#if PY_VERSION_HEX < 0x03080000
+#    error "PYTHON < 3.8 IS UNSUPPORTED. pybind11 v2.13 was the last to support Python 3.7."
+#endif
+
+#define PYBIND11_VERSION_MAJOR 3
+#define PYBIND11_VERSION_MINOR 0
+#define PYBIND11_VERSION_PATCH 0.dev1
 
 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html
 // Additional convention: 0xD = dev
-#define PYBIND11_VERSION_HEX 0x020D0100
+#define PYBIND11_VERSION_HEX 0x030000D1
 
 // Define some generic pybind11 helper macros for warning management.
 //
@@ -41,7 +46,7 @@
 #    define PYBIND11_COMPILER_CLANG
 #    define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__)
 #    define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push)
-#    define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push)
+#    define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic pop)
 #elif defined(__GNUC__)
 #    define PYBIND11_COMPILER_GCC
 #    define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__)
@@ -164,14 +169,6 @@
 #    endif
 #endif
 
-#if !defined(PYBIND11_EXPORT_EXCEPTION)
-#    if defined(__apple_build_version__)
-#        define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT
-#    else
-#        define PYBIND11_EXPORT_EXCEPTION
-#    endif
-#endif
-
 // For CUDA, GCC7, GCC8:
 // PYBIND11_NOINLINE_FORCED is incompatible with `-Wattributes -Werror`.
 // When defining PYBIND11_NOINLINE_FORCED, it is best to also use `-Wno-attributes`.
@@ -212,31 +209,6 @@
 #    define PYBIND11_MAYBE_UNUSED __attribute__((__unused__))
 #endif
 
-/* Don't let Python.h #define (v)snprintf as macro because they are implemented
-   properly in Visual Studio since 2015. */
-#if defined(_MSC_VER)
-#    define HAVE_SNPRINTF 1
-#endif
-
-/// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode
-#if defined(_MSC_VER)
-PYBIND11_WARNING_PUSH
-PYBIND11_WARNING_DISABLE_MSVC(4505)
-// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only)
-#    if defined(_DEBUG) && !defined(Py_DEBUG)
-// Workaround for a VS 2022 issue.
-// NOTE: This workaround knowingly violates the Python.h include order requirement:
-// https://docs.python.org/3/c-api/intro.html#include-files
-// See https://github.com/pybind/pybind11/pull/3497 for full context.
-#        include <yvals.h>
-#        if _MSVC_STL_VERSION >= 143
-#            include <crtdefs.h>
-#        endif
-#        define PYBIND11_DEBUG_MARKER
-#        undef _DEBUG
-#    endif
-#endif
-
 // https://en.cppreference.com/w/c/chrono/localtime
 #if defined(__STDC_LIB_EXT1__) && !defined(__STDC_WANT_LIB_EXT1__)
 #    define __STDC_WANT_LIB_EXT1__
@@ -271,46 +243,14 @@ PYBIND11_WARNING_DISABLE_MSVC(4505)
 #    endif
 #endif
 
-#include <Python.h>
-#if PY_VERSION_HEX < 0x03070000
-#    error "PYTHON < 3.7 IS UNSUPPORTED. pybind11 v2.12 was the last to support Python 3.6."
-#endif
-#include <frameobject.h>
-#include <pythread.h>
-
-/* Python #defines overrides on all sorts of core functions, which
-   tends to weak havok in C++ codebases that expect these to work
-   like regular functions (potentially with several overloads) */
-#if defined(isalnum)
-#    undef isalnum
-#    undef isalpha
-#    undef islower
-#    undef isspace
-#    undef isupper
-#    undef tolower
-#    undef toupper
-#endif
-
-#if defined(copysign)
-#    undef copysign
-#endif
-
 #if defined(PYBIND11_NUMPY_1_ONLY)
 #    define PYBIND11_INTERNAL_NUMPY_1_ONLY_DETECTED
 #endif
 
-#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+#if (defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
 #    define PYBIND11_SIMPLE_GIL_MANAGEMENT
 #endif
 
-#if defined(_MSC_VER)
-#    if defined(PYBIND11_DEBUG_MARKER)
-#        define _DEBUG
-#        undef PYBIND11_DEBUG_MARKER
-#    endif
-PYBIND11_WARNING_POP
-#endif
-
 #include <cstddef>
 #include <cstring>
 #include <exception>
@@ -329,6 +269,17 @@ PYBIND11_WARNING_POP
 #    endif
 #endif
 
+// For libc++, the exceptions should be exported,
+// otherwise, the exception translation would be incorrect.
+// IMPORTANT: This code block must stay BELOW the #include <exception> above (see PR #5390).
+#if !defined(PYBIND11_EXPORT_EXCEPTION)
+#    if defined(_LIBCPP_EXCEPTION)
+#        define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT
+#    else
+#        define PYBIND11_EXPORT_EXCEPTION
+#    endif
+#endif
+
 // Must be after including <version> or one of the other headers specified by the standard
 #if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L
 #    define PYBIND11_HAS_U8STRING
@@ -387,6 +338,20 @@ PYBIND11_WARNING_POP
 #define PYBIND11_CONCAT(first, second) first##second
 #define PYBIND11_ENSURE_INTERNALS_READY pybind11::detail::get_internals();
 
+#if !defined(GRAALVM_PYTHON)
+#    define PYBIND11_PYCFUNCTION_GET_DOC(func) ((func)->m_ml->ml_doc)
+#    define PYBIND11_PYCFUNCTION_SET_DOC(func, doc)                                               \
+        do {                                                                                      \
+            (func)->m_ml->ml_doc = (doc);                                                         \
+        } while (0)
+#else
+#    define PYBIND11_PYCFUNCTION_GET_DOC(func) (GraalPyCFunction_GetDoc((PyObject *) (func)))
+#    define PYBIND11_PYCFUNCTION_SET_DOC(func, doc)                                               \
+        do {                                                                                      \
+            GraalPyCFunction_SetDoc((PyObject *) (func), (doc));                                  \
+        } while (0)
+#endif
+
 #define PYBIND11_CHECK_PYTHON_VERSION                                                             \
     {                                                                                             \
         const char *compiled_ver                                                                  \
@@ -462,7 +427,25 @@ PYBIND11_WARNING_POP
                 return "Hello, World!";
             });
         }
+
+    The third macro argument is optional (available since 2.13.0), and can be used to
+    mark the extension module as safe to run without the GIL under a free-threaded CPython
+    interpreter. Passing this argument has no effect on other interpreters.
+
+    .. code-block:: cpp
+
+        PYBIND11_MODULE(example, m, py::mod_gil_not_used()) {
+            m.doc() = "pybind11 example module safe to run without the GIL";
+
+            // Add bindings here
+            m.def("foo", []() {
+                return "Hello, Free-threaded World!";
+            });
+        }
+
 \endrst */
+PYBIND11_WARNING_PUSH
+PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments")
 #define PYBIND11_MODULE(name, variable, ...)                                                      \
     static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name)            \
         PYBIND11_MAYBE_UNUSED;                                                                    \
@@ -483,6 +466,7 @@ PYBIND11_WARNING_POP
         PYBIND11_CATCH_INIT_EXCEPTIONS                                                            \
     }                                                                                             \
     void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ & (variable))
+PYBIND11_WARNING_POP
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
@@ -538,7 +522,7 @@ enum class return_value_policy : uint8_t {
         object without taking ownership similar to the above
         return_value_policy::reference policy. In contrast to that policy, the
         function or property's implicit this argument (called the parent) is
-        considered to be the the owner of the return value (the child).
+        considered to be the owner of the return value (the child).
         pybind11 then couples the lifetime of the parent to the child via a
         reference relationship that ensures that the parent cannot be garbage
         collected while Python is still using the child. More advanced
@@ -621,6 +605,8 @@ struct instance {
     bool simple_instance_registered : 1;
     /// If true, get_internals().patients has an entry for this object
     bool has_patients : 1;
+    /// If true, this Python object needs to be kept alive for the lifetime of the C++ value.
+    bool is_alias : 1;
 
     /// Initializes all of the above type/values/holders data (but not the instance values
     /// themselves)
@@ -643,6 +629,14 @@ struct instance {
 static_assert(std::is_standard_layout<instance>::value,
               "Internal error: `pybind11::detail::instance` is not standard layout!");
 
+// Some older compilers (e.g. gcc 9.4.0) require
+//     static_assert(always_false<T>::value, "...");
+// instead of
+//     static_assert(false, "...");
+// to trigger the static_assert() in a template only if it is actually instantiated.
+template <typename>
+struct always_false : std::false_type {};
+
 /// from __cpp_future__ import (convenient aliases from C++14/17)
 #if defined(PYBIND11_CPP14)
 using std::conditional_t;
@@ -1109,14 +1103,14 @@ struct overload_cast_impl {
     }
 
     template <typename Return, typename Class>
-    constexpr auto operator()(Return (Class::*pmf)(Args...),
-                              std::false_type = {}) const noexcept -> decltype(pmf) {
+    constexpr auto operator()(Return (Class::*pmf)(Args...), std::false_type = {}) const noexcept
+        -> decltype(pmf) {
         return pmf;
     }
 
     template <typename Return, typename Class>
-    constexpr auto operator()(Return (Class::*pmf)(Args...) const,
-                              std::true_type) const noexcept -> decltype(pmf) {
+    constexpr auto operator()(Return (Class::*pmf)(Args...) const, std::true_type) const noexcept
+        -> decltype(pmf) {
         return pmf;
     }
 };
@@ -1264,5 +1258,10 @@ constexpr
 #    define PYBIND11_DETAILED_ERROR_MESSAGES
 #endif
 
+// CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508.
+#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION)
+#    define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET
+#endif
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/detail/cpp_conduit.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2024 The pybind Community.
+
+#pragma once
+
+#include <pybind11/pytypes.h>
+
+#include "common.h"
+#include "internals.h"
+
+#include <typeinfo>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+// Forward declaration needed here: Refactoring opportunity.
+extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *);
+
+inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) {
+#if defined(PYPY_VERSION)
+    auto &internals = get_internals();
+    return bool(internals.registered_types_py.find(type_obj)
+                != internals.registered_types_py.end());
+#else
+    return bool(type_obj->tp_new == pybind11_object_new);
+#endif
+}
+
+inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) {
+    PyObject *descr = _PyType_Lookup(type_obj, attr_name);
+    return bool((descr != nullptr) && PyInstanceMethod_Check(descr));
+}
+
+inline object try_get_cpp_conduit_method(PyObject *obj) {
+    if (PyType_Check(obj)) {
+        return object();
+    }
+    PyTypeObject *type_obj = Py_TYPE(obj);
+    str attr_name("_pybind11_conduit_v1_");
+    bool assumed_to_be_callable = false;
+    if (type_is_managed_by_our_internals(type_obj)) {
+        if (!is_instance_method_of_type(type_obj, attr_name.ptr())) {
+            return object();
+        }
+        assumed_to_be_callable = true;
+    }
+    PyObject *method = PyObject_GetAttr(obj, attr_name.ptr());
+    if (method == nullptr) {
+        PyErr_Clear();
+        return object();
+    }
+    if (!assumed_to_be_callable && PyCallable_Check(method) == 0) {
+        Py_DECREF(method);
+        return object();
+    }
+    return reinterpret_steal<object>(method);
+}
+
+inline void *try_raw_pointer_ephemeral_from_cpp_conduit(handle src,
+                                                        const std::type_info *cpp_type_info) {
+    object method = try_get_cpp_conduit_method(src.ptr());
+    if (method) {
+        capsule cpp_type_info_capsule(const_cast<void *>(static_cast<const void *>(cpp_type_info)),
+                                      typeid(std::type_info).name());
+        object cpp_conduit = method(bytes(PYBIND11_PLATFORM_ABI_ID),
+                                    cpp_type_info_capsule,
+                                    bytes("raw_pointer_ephemeral"));
+        if (isinstance<capsule>(cpp_conduit)) {
+            return reinterpret_borrow<capsule>(cpp_conduit).get_pointer();
+        }
+    }
+    return nullptr;
+}
+
+#define PYBIND11_HAS_CPP_CONDUIT 1
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/detail/descr.h

@@ -99,6 +99,13 @@ constexpr descr<1, Type> const_name() {
     return {'%'};
 }
 
+// Use a different name based on whether the parameter is used as input or output
+template <size_t N1, size_t N2>
+constexpr descr<N1 + N2 + 1> io_name(char const (&text1)[N1], char const (&text2)[N2]) {
+    return const_name("@") + const_name(text1) + const_name("@") + const_name(text2)
+           + const_name("@");
+}
+
 // If "_" is defined as a macro, py::detail::_ cannot be provided.
 // It is therefore best to use py::detail::const_name universally.
 // This block is for backward compatibility only.
@@ -156,9 +163,8 @@ constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) {
 }
 #else
 template <size_t N, typename... Ts, typename... Args>
-constexpr auto concat(const descr<N, Ts...> &d,
-                      const Args &...args) -> decltype(std::declval<descr<N + 2, Ts...>>()
-                                                       + concat(args...)) {
+constexpr auto concat(const descr<N, Ts...> &d, const Args &...args)
+    -> decltype(std::declval<descr<N + 2, Ts...>>() + concat(args...)) {
     return d + const_name(", ") + concat(args...);
 }
 #endif
@@ -168,5 +174,15 @@ constexpr descr<N + 2, Ts...> type_descr(const descr<N, Ts...> &descr) {
     return const_name("{") + descr + const_name("}");
 }
 
+template <size_t N, typename... Ts>
+constexpr descr<N + 4, Ts...> arg_descr(const descr<N, Ts...> &descr) {
+    return const_name("@^") + descr + const_name("@!");
+}
+
+template <size_t N, typename... Ts>
+constexpr descr<N + 4, Ts...> return_descr(const descr<N, Ts...> &descr) {
+    return const_name("@$") + descr + const_name("@!");
+}
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h

@@ -0,0 +1,39 @@
+// Copyright (c) 2021 The Pybind Development Team.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "common.h"
+
+#include <type_traits>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <typename To, typename From, typename SFINAE = void>
+struct dynamic_raw_ptr_cast_is_possible : std::false_type {};
+
+template <typename To, typename From>
+struct dynamic_raw_ptr_cast_is_possible<
+    To,
+    From,
+    detail::enable_if_t<!std::is_same<To, void>::value && std::is_polymorphic<From>::value>>
+    : std::true_type {};
+
+template <typename To,
+          typename From,
+          detail::enable_if_t<!dynamic_raw_ptr_cast_is_possible<To, From>::value, int> = 0>
+To *dynamic_raw_ptr_cast_if_possible(From * /*ptr*/) {
+    return nullptr;
+}
+
+template <typename To,
+          typename From,
+          detail::enable_if_t<dynamic_raw_ptr_cast_is_possible<To, From>::value, int> = 0>
+To *dynamic_raw_ptr_cast_if_possible(From *ptr) {
+    return dynamic_cast<To *>(ptr);
+}
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/detail/exception_translation.h

@@ -0,0 +1,71 @@
+/*
+    pybind11/detail/exception_translation.h: means to translate C++ exceptions to Python exceptions
+
+    Copyright (c) 2024 The Pybind Development Team.
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "common.h"
+#include "internals.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+// Apply all the extensions translators from a list
+// Return true if one of the translators completed without raising an exception
+// itself. Return of false indicates that if there are other translators
+// available, they should be tried.
+inline bool apply_exception_translators(std::forward_list<ExceptionTranslator> &translators) {
+    auto last_exception = std::current_exception();
+
+    for (auto &translator : translators) {
+        try {
+            translator(last_exception);
+            return true;
+        } catch (...) {
+            last_exception = std::current_exception();
+        }
+    }
+    return false;
+}
+
+inline void try_translate_exceptions() {
+    /* When an exception is caught, give each registered exception
+        translator a chance to translate it to a Python exception. First
+        all module-local translators will be tried in reverse order of
+        registration. If none of the module-locale translators handle
+        the exception (or there are no module-locale translators) then
+        the global translators will be tried, also in reverse order of
+        registration.
+
+        A translator may choose to do one of the following:
+
+        - catch the exception and call py::set_error()
+            to set a standard (or custom) Python exception, or
+        - do nothing and let the exception fall through to the next translator, or
+        - delegate translation to the next translator by throwing a new type of exception.
+        */
+
+    bool handled = with_exception_translators(
+        [&](std::forward_list<ExceptionTranslator> &exception_translators,
+            std::forward_list<ExceptionTranslator> &local_exception_translators) {
+            if (detail::apply_exception_translators(local_exception_translators)) {
+                return true;
+            }
+            if (detail::apply_exception_translators(exception_translators)) {
+                return true;
+            }
+            return false;
+        });
+
+    if (!handled) {
+        set_error(PyExc_SystemError, "Exception escaped from default exception translator!");
+    }
+}
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/detail/init.h

@@ -10,6 +10,7 @@
 #pragma once
 
 #include "class.h"
+#include "using_smart_holder.h"
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
@@ -128,11 +129,13 @@ void construct(value_and_holder &v_h, Cpp<Class> *ptr, bool need_alias) {
         // the holder and destruction happens when we leave the C++ scope, and the holder
         // class gets to handle the destruction however it likes.
         v_h.value_ptr() = ptr;
-        v_h.set_instance_registered(true);          // To prevent init_instance from registering it
-        v_h.type->init_instance(v_h.inst, nullptr); // Set up the holder
+        v_h.set_instance_registered(true); // Trick to prevent init_instance from registering it
+        // DANGER ZONE BEGIN: exceptions will leave v_h in an invalid state.
+        v_h.type->init_instance(v_h.inst, nullptr);                        // Set up the holder
         Holder<Class> temp_holder(std::move(v_h.holder<Holder<Class>>())); // Steal the holder
         v_h.type->dealloc(v_h); // Destroys the moved-out holder remains, resets value ptr to null
         v_h.set_instance_registered(false);
+        // DANGER ZONE END.
 
         construct_alias_from_cpp<Class>(is_alias_constructible<Class>{}, v_h, std::move(*ptr));
     } else {
@@ -153,7 +156,7 @@ void construct(value_and_holder &v_h, Alias<Class> *alias_ptr, bool) {
 // holder.  This also handles types like std::shared_ptr<T> and std::unique_ptr<T> where T is a
 // derived type (through those holder's implicit conversion from derived class holder
 // constructors).
-template <typename Class>
+template <typename Class, detail::enable_if_t<!is_smart_holder<Holder<Class>>::value, int> = 0>
 void construct(value_and_holder &v_h, Holder<Class> holder, bool need_alias) {
     PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias);
     auto *ptr = holder_helper<Holder<Class>>::get(holder);
@@ -195,6 +198,74 @@ void construct(value_and_holder &v_h, Alias<Class> &&result, bool) {
     v_h.value_ptr() = new Alias<Class>(std::move(result));
 }
 
+template <typename T, typename D>
+smart_holder init_smart_holder_from_unique_ptr(std::unique_ptr<T, D> &&unq_ptr,
+                                               bool void_cast_raw_ptr) {
+    void *void_ptr = void_cast_raw_ptr ? static_cast<void *>(unq_ptr.get()) : nullptr;
+    return smart_holder::from_unique_ptr(std::move(unq_ptr), void_ptr);
+}
+
+template <typename Class,
+          typename D = std::default_delete<Cpp<Class>>,
+          detail::enable_if_t<is_smart_holder<Holder<Class>>::value, int> = 0>
+void construct(value_and_holder &v_h, std::unique_ptr<Cpp<Class>, D> &&unq_ptr, bool need_alias) {
+    PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias);
+    auto *ptr = unq_ptr.get();
+    no_nullptr(ptr);
+    if (Class::has_alias && need_alias && !is_alias<Class>(ptr)) {
+        throw type_error("pybind11::init(): construction failed: returned std::unique_ptr pointee "
+                         "is not an alias instance");
+    }
+    // Here and below: if the new object is a trampoline, the shared_from_this mechanism needs
+    // to be prevented from accessing the smart_holder vptr, because it does not keep the
+    // trampoline Python object alive. For types that don't inherit from enable_shared_from_this
+    // it does not matter if void_cast_raw_ptr is true or false, therefore it's not necessary
+    // to also inspect the type.
+    auto smhldr = init_smart_holder_from_unique_ptr(
+        std::move(unq_ptr), /*void_cast_raw_ptr*/ Class::has_alias && is_alias<Class>(ptr));
+    v_h.value_ptr() = ptr;
+    v_h.type->init_instance(v_h.inst, &smhldr);
+}
+
+template <typename Class,
+          typename D = std::default_delete<Alias<Class>>,
+          detail::enable_if_t<is_smart_holder<Holder<Class>>::value, int> = 0>
+void construct(value_and_holder &v_h,
+               std::unique_ptr<Alias<Class>, D> &&unq_ptr,
+               bool /*need_alias*/) {
+    auto *ptr = unq_ptr.get();
+    no_nullptr(ptr);
+    auto smhldr
+        = init_smart_holder_from_unique_ptr(std::move(unq_ptr), /*void_cast_raw_ptr*/ true);
+    v_h.value_ptr() = ptr;
+    v_h.type->init_instance(v_h.inst, &smhldr);
+}
+
+template <typename Class, detail::enable_if_t<is_smart_holder<Holder<Class>>::value, int> = 0>
+void construct(value_and_holder &v_h, std::shared_ptr<Cpp<Class>> &&shd_ptr, bool need_alias) {
+    PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias);
+    auto *ptr = shd_ptr.get();
+    no_nullptr(ptr);
+    if (Class::has_alias && need_alias && !is_alias<Class>(ptr)) {
+        throw type_error("pybind11::init(): construction failed: returned std::shared_ptr pointee "
+                         "is not an alias instance");
+    }
+    auto smhldr = smart_holder::from_shared_ptr(shd_ptr);
+    v_h.value_ptr() = ptr;
+    v_h.type->init_instance(v_h.inst, &smhldr);
+}
+
+template <typename Class, detail::enable_if_t<is_smart_holder<Holder<Class>>::value, int> = 0>
+void construct(value_and_holder &v_h,
+               std::shared_ptr<Alias<Class>> &&shd_ptr,
+               bool /*need_alias*/) {
+    auto *ptr = shd_ptr.get();
+    no_nullptr(ptr);
+    auto smhldr = smart_holder::from_shared_ptr(shd_ptr);
+    v_h.value_ptr() = ptr;
+    v_h.type->init_instance(v_h.inst, &smhldr);
+}
+
 // Implementing class for py::init<...>()
 template <typename... Args>
 struct constructor {
@@ -408,7 +479,7 @@ struct pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)> {
 
     template <typename Class, typename... Extra>
     void execute(Class &cl, const Extra &...extra) && {
-        cl.def("__getstate__", std::move(get));
+        cl.def("__getstate__", std::move(get), pos_only());
 
 #if defined(PYBIND11_CPP14)
         cl.def(

pybind11/include/pybind11/detail/internals.h

@@ -12,10 +12,11 @@
 #include "common.h"
 
 #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
-#    include "../gil.h"
+#    include <pybind11/gil.h>
 #endif
 
-#include "../pytypes.h"
+#include <pybind11/conduit/pybind11_platform_abi_id.h>
+#include <pybind11/pytypes.h>
 
 #include <exception>
 #include <mutex>
@@ -36,18 +37,12 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
-#    if PY_VERSION_HEX >= 0x030C0000 || defined(_MSC_VER)
-// Version bump for Python 3.12+, before first 3.12 beta release.
-// Version bump for MSVC piggy-backed on PR #4779. See comments there.
-#        define PYBIND11_INTERNALS_VERSION 5
-#    else
-#        define PYBIND11_INTERNALS_VERSION 4
-#    endif
+#    define PYBIND11_INTERNALS_VERSION 7
 #endif
 
-// This requirement is mainly to reduce the support burden (see PR #4570).
-static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5,
-              "pybind11 ABI version 5 is the minimum for Python 3.12+");
+#if PYBIND11_INTERNALS_VERSION < 7
+#    error "PYBIND11_INTERNALS_VERSION 7 is the minimum for all platforms for pybind11v3."
+#endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
@@ -66,40 +61,29 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass);
 // Thread Specific Storage (TSS) API.
 // Avoid unnecessary allocation of `Py_tss_t`, since we cannot use
 // `Py_LIMITED_API` anyway.
-#if PYBIND11_INTERNALS_VERSION > 4
-#    define PYBIND11_TLS_KEY_REF Py_tss_t &
-#    if defined(__clang__)
-#        define PYBIND11_TLS_KEY_INIT(var)                                                        \
-            _Pragma("clang diagnostic push")                                         /**/         \
-                _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/         \
-                Py_tss_t var                                                                      \
-                = Py_tss_NEEDS_INIT;                                                              \
-            _Pragma("clang diagnostic pop")
-#    elif defined(__GNUC__) && !defined(__INTEL_COMPILER)
-#        define PYBIND11_TLS_KEY_INIT(var)                                                        \
-            _Pragma("GCC diagnostic push")                                         /**/           \
-                _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/           \
-                Py_tss_t var                                                                      \
-                = Py_tss_NEEDS_INIT;                                                              \
-            _Pragma("GCC diagnostic pop")
-#    else
-#        define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT;
-#    endif
-#    define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0)
-#    define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key))
-#    define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value))
-#    define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr)
-#    define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key))
+#define PYBIND11_TLS_KEY_REF Py_tss_t &
+#if defined(__clang__)
+#    define PYBIND11_TLS_KEY_INIT(var)                                                            \
+        _Pragma("clang diagnostic push")                                         /**/             \
+            _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/             \
+            Py_tss_t var                                                                          \
+            = Py_tss_NEEDS_INIT;                                                                  \
+        _Pragma("clang diagnostic pop")
+#elif defined(__GNUC__) && !defined(__INTEL_COMPILER)
+#    define PYBIND11_TLS_KEY_INIT(var)                                                            \
+        _Pragma("GCC diagnostic push")                                         /**/               \
+            _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/               \
+            Py_tss_t var                                                                          \
+            = Py_tss_NEEDS_INIT;                                                                  \
+        _Pragma("GCC diagnostic pop")
 #else
-#    define PYBIND11_TLS_KEY_REF Py_tss_t *
-#    define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr;
-#    define PYBIND11_TLS_KEY_CREATE(var)                                                          \
-        (((var) = PyThread_tss_alloc()) != nullptr && (PyThread_tss_create((var)) == 0))
-#    define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key))
-#    define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value))
-#    define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr)
-#    define PYBIND11_TLS_FREE(key) PyThread_tss_free(key)
+#    define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT;
 #endif
+#define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0)
+#define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key))
+#define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value))
+#define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr)
+#define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key))
 
 // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly
 // other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module
@@ -107,8 +91,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass);
 // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name,
 // which works.  If not under a known-good stl, provide our own name-based hash and equality
 // functions that use the type name.
-#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__))                                     \
-    || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION))
+#if !defined(_LIBCPP_VERSION)
 inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; }
 using type_hash = std::hash<std::type_index>;
 using type_equal_to = std::equal_to<std::type_index>;
@@ -148,20 +131,36 @@ struct override_hash {
 
 using instance_map = std::unordered_multimap<const void *, instance *>;
 
+#ifdef Py_GIL_DISABLED
+// Wrapper around PyMutex to provide BasicLockable semantics
+class pymutex {
+    PyMutex mutex;
+
+public:
+    pymutex() : mutex({}) {}
+    void lock() { PyMutex_Lock(&mutex); }
+    void unlock() { PyMutex_Unlock(&mutex); }
+};
+
 // Instance map shards are used to reduce mutex contention in free-threaded Python.
 struct instance_map_shard {
-    std::mutex mutex;
     instance_map registered_instances;
+    pymutex mutex;
     // alignas(64) would be better, but causes compile errors in macOS before 10.14 (see #5200)
-    char padding[64 - (sizeof(std::mutex) + sizeof(instance_map)) % 64];
+    char padding[64 - (sizeof(instance_map) + sizeof(pymutex)) % 64];
 };
 
+static_assert(sizeof(instance_map_shard) % 64 == 0,
+              "instance_map_shard size is not a multiple of 64 bytes");
+#endif
+
 /// Internal data structure used to track registered instances and types.
 /// Whenever binary incompatible changes are made to this structure,
 /// `PYBIND11_INTERNALS_VERSION` must be incremented.
 struct internals {
 #ifdef Py_GIL_DISABLED
-    std::mutex mutex;
+    pymutex mutex;
+    pymutex exception_translator_mutex;
 #endif
     // std::type_index -> pybind11's type information
     type_map<type_info *> registered_types_cpp;
@@ -180,35 +179,26 @@ struct internals {
     std::forward_list<ExceptionTranslator> registered_exception_translators;
     std::unordered_map<std::string, void *> shared_data; // Custom data to be shared across
                                                          // extensions
-#if PYBIND11_INTERNALS_VERSION == 4
-    std::vector<PyObject *> unused_loader_patient_stack_remove_at_v5;
-#endif
-    std::forward_list<std::string> static_strings; // Stores the std::strings backing
-                                                   // detail::c_str()
+    std::forward_list<std::string> static_strings;       // Stores the std::strings backing
+                                                         // detail::c_str()
     PyTypeObject *static_property_type;
     PyTypeObject *default_metaclass;
     PyObject *instance_base;
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PYBIND11_TLS_KEY_INIT(tstate)
-#if PYBIND11_INTERNALS_VERSION > 4
     PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
-#endif // PYBIND11_INTERNALS_VERSION > 4
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PyInterpreterState *istate = nullptr;
 
-#if PYBIND11_INTERNALS_VERSION > 4
     // Note that we have to use a std::string to allocate memory to ensure a unique address
     // We want unique addresses since we use pointer equality to compare function records
     std::string function_record_capsule_name = internals_function_record_capsule_name;
-#endif
 
     internals() = default;
     internals(const internals &other) = delete;
     internals &operator=(const internals &other) = delete;
     ~internals() {
-#if PYBIND11_INTERNALS_VERSION > 4
         PYBIND11_TLS_FREE(loader_life_support_tls_key);
-#endif // PYBIND11_INTERNALS_VERSION > 4
 
         // This destructor is called *after* Py_Finalize() in finalize_interpreter().
         // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is
@@ -221,6 +211,17 @@ struct internals {
     }
 };
 
+// For backwards compatibility (i.e. #ifdef guards):
+#define PYBIND11_HAS_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+
+enum class holder_enum_t : uint8_t {
+    undefined,
+    std_unique_ptr, // Default, lacking interop with std::shared_ptr.
+    std_shared_ptr, // Lacking interop with std::unique_ptr.
+    smart_holder,   // Full std::unique_ptr / std::shared_ptr interop.
+    custom_holder,
+};
+
 /// Additional type information which does not fit into the PyTypeObject.
 /// Changes to this struct also require bumping `PYBIND11_INTERNALS_VERSION`.
 struct type_info {
@@ -236,6 +237,7 @@ struct type_info {
     buffer_info *(*get_buffer)(PyObject *, void *) = nullptr;
     void *get_buffer_data = nullptr;
     void *(*module_local_load)(PyObject *, const type_info *) = nullptr;
+    holder_enum_t holder_enum_v = holder_enum_t::undefined;
     /* A simple type never occurs as a (direct or indirect) parent
      * of a class that makes use of multiple inheritance.
      * A type can be simple even if it has non-simple ancestors as long as it has no descendants.
@@ -243,78 +245,17 @@ struct type_info {
     bool simple_type : 1;
     /* True if there is no multiple inheritance in this type's inheritance tree */
     bool simple_ancestors : 1;
-    /* for base vs derived holder_type checks */
-    bool default_holder : 1;
     /* true if this is a type registered with py::module_local */
     bool module_local : 1;
 };
 
-/// On MSVC, debug and release builds are not ABI-compatible!
-#if defined(_MSC_VER) && defined(_DEBUG)
-#    define PYBIND11_BUILD_TYPE "_debug"
-#else
-#    define PYBIND11_BUILD_TYPE ""
-#endif
-
-/// Let's assume that different compilers are ABI-incompatible.
-/// A user can manually set this string if they know their
-/// compiler is compatible.
-#ifndef PYBIND11_COMPILER_TYPE
-#    if defined(_MSC_VER)
-#        define PYBIND11_COMPILER_TYPE "_msvc"
-#    elif defined(__INTEL_COMPILER)
-#        define PYBIND11_COMPILER_TYPE "_icc"
-#    elif defined(__clang__)
-#        define PYBIND11_COMPILER_TYPE "_clang"
-#    elif defined(__PGI)
-#        define PYBIND11_COMPILER_TYPE "_pgi"
-#    elif defined(__MINGW32__)
-#        define PYBIND11_COMPILER_TYPE "_mingw"
-#    elif defined(__CYGWIN__)
-#        define PYBIND11_COMPILER_TYPE "_gcc_cygwin"
-#    elif defined(__GNUC__)
-#        define PYBIND11_COMPILER_TYPE "_gcc"
-#    else
-#        define PYBIND11_COMPILER_TYPE "_unknown"
-#    endif
-#endif
-
-/// Also standard libs
-#ifndef PYBIND11_STDLIB
-#    if defined(_LIBCPP_VERSION)
-#        define PYBIND11_STDLIB "_libcpp"
-#    elif defined(__GLIBCXX__) || defined(__GLIBCPP__)
-#        define PYBIND11_STDLIB "_libstdcpp"
-#    else
-#        define PYBIND11_STDLIB ""
-#    endif
-#endif
-
-/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility.
-/// On MSVC, changes in _MSC_VER may indicate ABI incompatibility (#2898).
-#ifndef PYBIND11_BUILD_ABI
-#    if defined(__GXX_ABI_VERSION)
-#        define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION)
-#    elif defined(_MSC_VER)
-#        define PYBIND11_BUILD_ABI "_mscver" PYBIND11_TOSTRING(_MSC_VER)
-#    else
-#        define PYBIND11_BUILD_ABI ""
-#    endif
-#endif
-
-#ifndef PYBIND11_INTERNALS_KIND
-#    define PYBIND11_INTERNALS_KIND ""
-#endif
-
 #define PYBIND11_INTERNALS_ID                                                                     \
     "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION)                        \
-        PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB                            \
-            PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__"
+        PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__"
 
 #define PYBIND11_MODULE_LOCAL_ID                                                                  \
     "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION)                     \
-        PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB                            \
-            PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__"
+        PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__"
 
 /// Each module locally stores a pointer to the `internals` data. The data
 /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`.
@@ -432,7 +373,7 @@ inline void translate_local_exception(std::exception_ptr p) {
 
 inline object get_python_state_dict() {
     object state_dict;
-#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION)
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
     state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins());
 #else
 #    if PY_VERSION_HEX < 0x03090000
@@ -530,15 +471,14 @@ PYBIND11_NOINLINE internals &get_internals() {
         }
         PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate);
 
-#if PYBIND11_INTERNALS_VERSION > 4
         // NOLINTNEXTLINE(bugprone-assignment-in-if-condition)
         if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) {
             pybind11_fail("get_internals: could not successfully initialize the "
                           "loader_life_support TSS key!");
         }
-#endif
+
         internals_ptr->istate = tstate->interp;
-        state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp);
+        state_dict[PYBIND11_INTERNALS_ID] = capsule(reinterpret_cast<void *>(internals_pp));
         internals_ptr->registered_exception_translators.push_front(&translate_exception);
         internals_ptr->static_property_type = make_static_property_type();
         internals_ptr->default_metaclass = make_default_metaclass();
@@ -566,40 +506,6 @@ PYBIND11_NOINLINE internals &get_internals() {
 struct local_internals {
     type_map<type_info *> registered_types_cpp;
     std::forward_list<ExceptionTranslator> registered_exception_translators;
-#if PYBIND11_INTERNALS_VERSION == 4
-
-    // For ABI compatibility, we can't store the loader_life_support TLS key in
-    // the `internals` struct directly.  Instead, we store it in `shared_data` and
-    // cache a copy in `local_internals`.  If we allocated a separate TLS key for
-    // each instance of `local_internals`, we could end up allocating hundreds of
-    // TLS keys if hundreds of different pybind11 modules are loaded (which is a
-    // plausible number).
-    PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
-
-    // Holds the shared TLS key for the loader_life_support stack.
-    struct shared_loader_life_support_data {
-        PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
-        shared_loader_life_support_data() {
-            // NOLINTNEXTLINE(bugprone-assignment-in-if-condition)
-            if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) {
-                pybind11_fail("local_internals: could not successfully initialize the "
-                              "loader_life_support TLS key!");
-            }
-        }
-        // We can't help but leak the TLS key, because Python never unloads extension modules.
-    };
-
-    local_internals() {
-        auto &internals = get_internals();
-        // Get or create the `loader_life_support_stack_key`.
-        auto &ptr = internals.shared_data["_life_support"];
-        if (!ptr) {
-            ptr = new shared_loader_life_support_data;
-        }
-        loader_life_support_tls_key
-            = static_cast<shared_loader_life_support_data *>(ptr)->loader_life_support_tls_key;
-    }
-#endif //  PYBIND11_INTERNALS_VERSION == 4
 };
 
 /// Works like `get_internals`, but for things which are locally registered.
@@ -614,7 +520,7 @@ inline local_internals &get_local_internals() {
 }
 
 #ifdef Py_GIL_DISABLED
-#    define PYBIND11_LOCK_INTERNALS(internals) std::unique_lock<std::mutex> lock((internals).mutex)
+#    define PYBIND11_LOCK_INTERNALS(internals) std::unique_lock<pymutex> lock((internals).mutex)
 #else
 #    define PYBIND11_LOCK_INTERNALS(internals)
 #endif
@@ -626,6 +532,19 @@ inline auto with_internals(const F &cb) -> decltype(cb(get_internals())) {
     return cb(internals);
 }
 
+template <typename F>
+inline auto with_exception_translators(const F &cb)
+    -> decltype(cb(get_internals().registered_exception_translators,
+                   get_local_internals().registered_exception_translators)) {
+    auto &internals = get_internals();
+#ifdef Py_GIL_DISABLED
+    std::unique_lock<pymutex> lock((internals).exception_translator_mutex);
+#endif
+    auto &local_internals = get_local_internals();
+    return cb(internals.registered_exception_translators,
+              local_internals.registered_exception_translators);
+}
+
 inline std::uint64_t mix64(std::uint64_t z) {
     // David Stafford's variant 13 of the MurmurHash3 finalizer popularized
     // by the SplitMix PRNG.
@@ -636,8 +555,8 @@ inline std::uint64_t mix64(std::uint64_t z) {
 }
 
 template <typename F>
-inline auto with_instance_map(const void *ptr,
-                              const F &cb) -> decltype(cb(std::declval<instance_map &>())) {
+inline auto with_instance_map(const void *ptr, const F &cb)
+    -> decltype(cb(std::declval<instance_map &>())) {
     auto &internals = get_internals();
 
 #ifdef Py_GIL_DISABLED
@@ -651,7 +570,7 @@ inline auto with_instance_map(const void *ptr,
     auto idx = static_cast<size_t>(hash & internals.instance_shards_mask);
 
     auto &shard = internals.instance_shards[idx];
-    std::unique_lock<std::mutex> lock(shard.mutex);
+    std::unique_lock<pymutex> lock(shard.mutex);
     return cb(shard.registered_instances);
 #else
     (void) ptr;
@@ -667,7 +586,7 @@ inline size_t num_registered_instances() {
     size_t count = 0;
     for (size_t i = 0; i <= internals.instance_shards_mask; ++i) {
         auto &shard = internals.instance_shards[i];
-        std::unique_lock<std::mutex> lock(shard.mutex);
+        std::unique_lock<pymutex> lock(shard.mutex);
         count += shard.registered_instances.size();
     }
     return count;
@@ -692,7 +611,8 @@ const char *c_str(Args &&...args) {
 }
 
 inline const char *get_function_record_capsule_name() {
-#if PYBIND11_INTERNALS_VERSION > 4
+    // On GraalPy, pointer equality of the names is currently not guaranteed
+#if !defined(GRAALVM_PYTHON)
     return get_internals().function_record_capsule_name.c_str();
 #else
     return nullptr;

pybind11/include/pybind11/detail/struct_smart_holder.h

@@ -0,0 +1,349 @@
+// Copyright (c) 2020-2024 The Pybind Development Team.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/* Proof-of-Concept for smart pointer interoperability.
+
+High-level aspects:
+
+* Support all `unique_ptr`, `shared_ptr` interops that are feasible.
+
+* Cleanly and clearly report all interops that are infeasible.
+
+* Meant to fit into a `PyObject`, as a holder for C++ objects.
+
+* Support a system design that makes it impossible to trigger
+  C++ Undefined Behavior, especially from Python.
+
+* Support a system design with clean runtime inheritance casting. From this
+  it follows that the `smart_holder` needs to be type-erased (`void*`).
+
+* Handling of RTTI for the type-erased held pointer is NOT implemented here.
+  It is the responsibility of the caller to ensure that `static_cast<T *>`
+  is well-formed when calling `as_*` member functions. Inheritance casting
+  needs to be handled in a different layer (similar to the code organization
+  in boost/python/object/inheritance.hpp).
+
+Details:
+
+* The "root holder" chosen here is a `shared_ptr<void>` (named `vptr` in this
+  implementation). This choice is practically inevitable because `shared_ptr`
+  has only very limited support for inspecting and accessing its deleter.
+
+* If created from a raw pointer, or a `unique_ptr` without a custom deleter,
+  `vptr` always uses a custom deleter, to support `unique_ptr`-like disowning.
+  The custom deleters could be extended to included life-time management for
+  external objects (e.g. `PyObject`).
+
+* If created from an external `shared_ptr`, or a `unique_ptr` with a custom
+  deleter, including life-time management for external objects is infeasible.
+
+* By choice, the smart_holder is movable but not copyable, to keep the design
+  simple, and to guard against accidental copying overhead.
+
+* The `void_cast_raw_ptr` option is needed to make the `smart_holder` `vptr`
+  member invisible to the `shared_from_this` mechanism, in case the lifetime
+  of a `PyObject` is tied to the pointee.
+*/
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+#include <typeinfo>
+#include <utility>
+
+// pybindit = Python Bindings Innovation Track.
+// Currently not in pybind11 namespace to signal that this POC does not depend
+// on any existing pybind11 functionality.
+namespace pybindit {
+namespace memory {
+
+static constexpr bool type_has_shared_from_this(...) { return false; }
+
+template <typename T>
+static constexpr bool type_has_shared_from_this(const std::enable_shared_from_this<T> *) {
+    return true;
+}
+
+struct guarded_delete {
+    std::weak_ptr<void> released_ptr;    // Trick to keep the smart_holder memory footprint small.
+    std::function<void(void *)> del_fun; // Rare case.
+    void (*del_ptr)(void *);             // Common case.
+    bool use_del_fun;
+    bool armed_flag;
+    guarded_delete(std::function<void(void *)> &&del_fun, bool armed_flag)
+        : del_fun{std::move(del_fun)}, del_ptr{nullptr}, use_del_fun{true},
+          armed_flag{armed_flag} {}
+    guarded_delete(void (*del_ptr)(void *), bool armed_flag)
+        : del_ptr{del_ptr}, use_del_fun{false}, armed_flag{armed_flag} {}
+    void operator()(void *raw_ptr) const {
+        if (armed_flag) {
+            if (use_del_fun) {
+                del_fun(raw_ptr);
+            } else {
+                del_ptr(raw_ptr);
+            }
+        }
+    }
+};
+
+template <typename T, typename std::enable_if<std::is_destructible<T>::value, int>::type = 0>
+inline void builtin_delete_if_destructible(void *raw_ptr) {
+    std::default_delete<T>{}(static_cast<T *>(raw_ptr));
+}
+
+template <typename T, typename std::enable_if<!std::is_destructible<T>::value, int>::type = 0>
+inline void builtin_delete_if_destructible(void *) {
+    // This noop operator is needed to avoid a compilation error (for `delete raw_ptr;`), but
+    // throwing an exception from a destructor will std::terminate the process. Therefore the
+    // runtime check for lifetime-management correctness is implemented elsewhere (in
+    // ensure_pointee_is_destructible()).
+}
+
+template <typename T>
+guarded_delete make_guarded_builtin_delete(bool armed_flag) {
+    return guarded_delete(builtin_delete_if_destructible<T>, armed_flag);
+}
+
+template <typename T, typename D>
+struct custom_deleter {
+    D deleter;
+    explicit custom_deleter(D &&deleter) : deleter{std::forward<D>(deleter)} {}
+    void operator()(void *raw_ptr) { deleter(static_cast<T *>(raw_ptr)); }
+};
+
+template <typename T, typename D>
+guarded_delete make_guarded_custom_deleter(D &&uqp_del, bool armed_flag) {
+    return guarded_delete(
+        std::function<void(void *)>(custom_deleter<T, D>(std::forward<D>(uqp_del))), armed_flag);
+}
+
+template <typename T>
+inline bool is_std_default_delete(const std::type_info &rtti_deleter) {
+    return rtti_deleter == typeid(std::default_delete<T>)
+           || rtti_deleter == typeid(std::default_delete<T const>);
+}
+
+struct smart_holder {
+    const std::type_info *rtti_uqp_del = nullptr;
+    std::shared_ptr<void> vptr;
+    bool vptr_is_using_noop_deleter : 1;
+    bool vptr_is_using_builtin_delete : 1;
+    bool vptr_is_external_shared_ptr : 1;
+    bool is_populated : 1;
+    bool is_disowned : 1;
+
+    // Design choice: smart_holder is movable but not copyable.
+    smart_holder(smart_holder &&) = default;
+    smart_holder(const smart_holder &) = delete;
+    smart_holder &operator=(smart_holder &&) = delete;
+    smart_holder &operator=(const smart_holder &) = delete;
+
+    smart_holder()
+        : vptr_is_using_noop_deleter{false}, vptr_is_using_builtin_delete{false},
+          vptr_is_external_shared_ptr{false}, is_populated{false}, is_disowned{false} {}
+
+    bool has_pointee() const { return vptr != nullptr; }
+
+    template <typename T>
+    static void ensure_pointee_is_destructible(const char *context) {
+        if (!std::is_destructible<T>::value) {
+            throw std::invalid_argument(std::string("Pointee is not destructible (") + context
+                                        + ").");
+        }
+    }
+
+    void ensure_is_populated(const char *context) const {
+        if (!is_populated) {
+            throw std::runtime_error(std::string("Unpopulated holder (") + context + ").");
+        }
+    }
+    void ensure_is_not_disowned(const char *context) const {
+        if (is_disowned) {
+            throw std::runtime_error(std::string("Holder was disowned already (") + context
+                                     + ").");
+        }
+    }
+
+    void ensure_vptr_is_using_builtin_delete(const char *context) const {
+        if (vptr_is_external_shared_ptr) {
+            throw std::invalid_argument(std::string("Cannot disown external shared_ptr (")
+                                        + context + ").");
+        }
+        if (vptr_is_using_noop_deleter) {
+            throw std::invalid_argument(std::string("Cannot disown non-owning holder (") + context
+                                        + ").");
+        }
+        if (!vptr_is_using_builtin_delete) {
+            throw std::invalid_argument(std::string("Cannot disown custom deleter (") + context
+                                        + ").");
+        }
+    }
+
+    template <typename T, typename D>
+    void ensure_compatible_rtti_uqp_del(const char *context) const {
+        const std::type_info *rtti_requested = &typeid(D);
+        if (!rtti_uqp_del) {
+            if (!is_std_default_delete<T>(*rtti_requested)) {
+                throw std::invalid_argument(std::string("Missing unique_ptr deleter (") + context
+                                            + ").");
+            }
+            ensure_vptr_is_using_builtin_delete(context);
+        } else if (!(*rtti_requested == *rtti_uqp_del)
+                   && !(vptr_is_using_builtin_delete
+                        && is_std_default_delete<T>(*rtti_requested))) {
+            throw std::invalid_argument(std::string("Incompatible unique_ptr deleter (") + context
+                                        + ").");
+        }
+    }
+
+    void ensure_has_pointee(const char *context) const {
+        if (!has_pointee()) {
+            throw std::invalid_argument(std::string("Disowned holder (") + context + ").");
+        }
+    }
+
+    void ensure_use_count_1(const char *context) const {
+        if (vptr == nullptr) {
+            throw std::invalid_argument(std::string("Cannot disown nullptr (") + context + ").");
+        }
+        // In multithreaded environments accessing use_count can lead to
+        // race conditions, but in the context of Python it is a bug (elsewhere)
+        // if the Global Interpreter Lock (GIL) is not being held when this code
+        // is reached.
+        // PYBIND11:REMINDER: This may need to be protected by a mutex in free-threaded Python.
+        if (vptr.use_count() != 1) {
+            throw std::invalid_argument(std::string("Cannot disown use_count != 1 (") + context
+                                        + ").");
+        }
+    }
+
+    void reset_vptr_deleter_armed_flag(bool armed_flag) const {
+        auto *vptr_del_ptr = std::get_deleter<guarded_delete>(vptr);
+        if (vptr_del_ptr == nullptr) {
+            throw std::runtime_error(
+                "smart_holder::reset_vptr_deleter_armed_flag() called in an invalid context.");
+        }
+        vptr_del_ptr->armed_flag = armed_flag;
+    }
+
+    // Caller is responsible for precondition: ensure_compatible_rtti_uqp_del<T, D>() must succeed.
+    template <typename T, typename D>
+    std::unique_ptr<D> extract_deleter(const char *context) const {
+        const auto *gd = std::get_deleter<guarded_delete>(vptr);
+        if (gd && gd->use_del_fun) {
+            const auto &custom_deleter_ptr = gd->del_fun.template target<custom_deleter<T, D>>();
+            if (custom_deleter_ptr == nullptr) {
+                throw std::runtime_error(
+                    std::string("smart_holder::extract_deleter() precondition failure (") + context
+                    + ").");
+            }
+            static_assert(std::is_copy_constructible<D>::value,
+                          "Required for compatibility with smart_holder functionality.");
+            return std::unique_ptr<D>(new D(custom_deleter_ptr->deleter));
+        }
+        return nullptr;
+    }
+
+    static smart_holder from_raw_ptr_unowned(void *raw_ptr) {
+        smart_holder hld;
+        hld.vptr.reset(raw_ptr, [](void *) {});
+        hld.vptr_is_using_noop_deleter = true;
+        hld.is_populated = true;
+        return hld;
+    }
+
+    template <typename T>
+    T *as_raw_ptr_unowned() const {
+        return static_cast<T *>(vptr.get());
+    }
+
+    template <typename T>
+    static smart_holder from_raw_ptr_take_ownership(T *raw_ptr, bool void_cast_raw_ptr = false) {
+        ensure_pointee_is_destructible<T>("from_raw_ptr_take_ownership");
+        smart_holder hld;
+        auto gd = make_guarded_builtin_delete<T>(true);
+        if (void_cast_raw_ptr) {
+            hld.vptr.reset(static_cast<void *>(raw_ptr), std::move(gd));
+        } else {
+            hld.vptr.reset(raw_ptr, std::move(gd));
+        }
+        hld.vptr_is_using_builtin_delete = true;
+        hld.is_populated = true;
+        return hld;
+    }
+
+    // Caller is responsible for ensuring the complex preconditions
+    // (see `smart_holder_type_caster_support::load_helper`).
+    void disown() {
+        reset_vptr_deleter_armed_flag(false);
+        is_disowned = true;
+    }
+
+    // Caller is responsible for ensuring the complex preconditions
+    // (see `smart_holder_type_caster_support::load_helper`).
+    void reclaim_disowned() {
+        reset_vptr_deleter_armed_flag(true);
+        is_disowned = false;
+    }
+
+    // Caller is responsible for ensuring the complex preconditions
+    // (see `smart_holder_type_caster_support::load_helper`).
+    void release_disowned() { vptr.reset(); }
+
+    void ensure_can_release_ownership(const char *context = "ensure_can_release_ownership") const {
+        ensure_is_not_disowned(context);
+        ensure_vptr_is_using_builtin_delete(context);
+        ensure_use_count_1(context);
+    }
+
+    // Caller is responsible for ensuring the complex preconditions
+    // (see `smart_holder_type_caster_support::load_helper`).
+    void release_ownership() {
+        reset_vptr_deleter_armed_flag(false);
+        release_disowned();
+    }
+
+    template <typename T, typename D>
+    static smart_holder from_unique_ptr(std::unique_ptr<T, D> &&unq_ptr,
+                                        void *void_ptr = nullptr) {
+        smart_holder hld;
+        hld.rtti_uqp_del = &typeid(D);
+        hld.vptr_is_using_builtin_delete = is_std_default_delete<T>(*hld.rtti_uqp_del);
+        guarded_delete gd{nullptr, false};
+        if (hld.vptr_is_using_builtin_delete) {
+            gd = make_guarded_builtin_delete<T>(true);
+        } else {
+            gd = make_guarded_custom_deleter<T, D>(std::move(unq_ptr.get_deleter()), true);
+        }
+        if (void_ptr != nullptr) {
+            hld.vptr.reset(void_ptr, std::move(gd));
+        } else {
+            hld.vptr.reset(unq_ptr.get(), std::move(gd));
+        }
+        (void) unq_ptr.release();
+        hld.is_populated = true;
+        return hld;
+    }
+
+    template <typename T>
+    static smart_holder from_shared_ptr(std::shared_ptr<T> shd_ptr) {
+        smart_holder hld;
+        hld.vptr = std::static_pointer_cast<void>(shd_ptr);
+        hld.vptr_is_external_shared_ptr = true;
+        hld.is_populated = true;
+        return hld;
+    }
+
+    template <typename T>
+    std::shared_ptr<T> as_shared_ptr() const {
+        return std::static_pointer_cast<T>(vptr);
+    }
+};
+
+} // namespace memory
+} // namespace pybindit

pybind11/include/pybind11/detail/type_caster_base.h

@@ -9,15 +9,24 @@
 
 #pragma once
 
-#include "../pytypes.h"
+#include <pybind11/gil.h>
+#include <pybind11/pytypes.h>
+#include <pybind11/trampoline_self_life_support.h>
+
 #include "common.h"
+#include "cpp_conduit.h"
 #include "descr.h"
+#include "dynamic_raw_ptr_cast_if_possible.h"
 #include "internals.h"
 #include "typeid.h"
+#include "using_smart_holder.h"
+#include "value_and_holder.h"
 
 #include <cstdint>
+#include <cstring>
 #include <iterator>
 #include <new>
+#include <stdexcept>
 #include <string>
 #include <type_traits>
 #include <typeindex>
@@ -38,11 +47,7 @@ private:
 
     // Store stack pointer in thread-local storage.
     static PYBIND11_TLS_KEY_REF get_stack_tls_key() {
-#if PYBIND11_INTERNALS_VERSION == 4
-        return get_local_internals().loader_life_support_tls_key;
-#else
         return get_internals().loader_life_support_tls_key;
-#endif
     }
     static loader_life_support *get_stack_top() {
         return static_cast<loader_life_support *>(PYBIND11_TLS_GET_VALUE(get_stack_tls_key()));
@@ -112,7 +117,6 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector<type_
     for (handle parent : reinterpret_borrow<tuple>(t->tp_bases)) {
         check.push_back((PyTypeObject *) parent.ptr());
     }
-
     auto const &type_dict = get_internals().registered_types_py;
     for (size_t i = 0; i < check.size(); i++) {
         auto *type = check[i];
@@ -171,13 +175,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector<type_
  * The value is cached for the lifetime of the Python type.
  */
 inline const std::vector<detail::type_info *> &all_type_info(PyTypeObject *type) {
-    auto ins = all_type_info_get_cache(type);
-    if (ins.second) {
-        // New cache entry: populate it
-        all_type_info_populate(type, ins.first->second);
-    }
-
-    return ins.first->second;
+    return all_type_info_get_cache(type).first->second;
 }
 
 /**
@@ -243,6 +241,49 @@ PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, bool throw_if
     return handle(type_info ? ((PyObject *) type_info->type) : nullptr);
 }
 
+inline bool try_incref(PyObject *obj) {
+    // Tries to increment the reference count of an object if it's not zero.
+    // TODO: Use PyUnstable_TryIncref when available.
+    // See https://github.com/python/cpython/issues/128844
+#ifdef Py_GIL_DISABLED
+    // See
+    // https://github.com/python/cpython/blob/d05140f9f77d7dfc753dd1e5ac3a5962aaa03eff/Include/internal/pycore_object.h#L761
+    uint32_t local = _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local);
+    local += 1;
+    if (local == 0) {
+        // immortal
+        return true;
+    }
+    if (_Py_IsOwnedByCurrentThread(obj)) {
+        _Py_atomic_store_uint32_relaxed(&obj->ob_ref_local, local);
+#    ifdef Py_REF_DEBUG
+        _Py_INCREF_IncRefTotal();
+#    endif
+        return true;
+    }
+    Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared);
+    for (;;) {
+        // If the shared refcount is zero and the object is either merged
+        // or may not have weak references, then we cannot incref it.
+        if (shared == 0 || shared == _Py_REF_MERGED) {
+            return false;
+        }
+
+        if (_Py_atomic_compare_exchange_ssize(
+                &obj->ob_ref_shared, &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) {
+#    ifdef Py_REF_DEBUG
+            _Py_INCREF_IncRefTotal();
+#    endif
+            return true;
+        }
+    }
+#else
+    assert(Py_REFCNT(obj) > 0);
+    Py_INCREF(obj);
+    return true;
+#endif
+}
+
 // Searches the inheritance graph for a registered Python instance, using all_type_info().
 PYBIND11_NOINLINE handle find_registered_python_instance(void *src,
                                                          const detail::type_info *tinfo) {
@@ -251,7 +292,10 @@ PYBIND11_NOINLINE handle find_registered_python_instance(void *src,
         for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) {
             for (auto *instance_type : detail::all_type_info(Py_TYPE(it_i->second))) {
                 if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) {
-                    return handle((PyObject *) it_i->second).inc_ref();
+                    auto *wrapper = reinterpret_cast<PyObject *>(it_i->second);
+                    if (try_incref(wrapper)) {
+                        return handle(wrapper);
+                    }
                 }
             }
         }
@@ -259,67 +303,6 @@ PYBIND11_NOINLINE handle find_registered_python_instance(void *src,
     });
 }
 
-struct value_and_holder {
-    instance *inst = nullptr;
-    size_t index = 0u;
-    const detail::type_info *type = nullptr;
-    void **vh = nullptr;
-
-    // Main constructor for a found value/holder:
-    value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index)
-        : inst{i}, index{index}, type{type},
-          vh{inst->simple_layout ? inst->simple_value_holder
-                                 : &inst->nonsimple.values_and_holders[vpos]} {}
-
-    // Default constructor (used to signal a value-and-holder not found by get_value_and_holder())
-    value_and_holder() = default;
-
-    // Used for past-the-end iterator
-    explicit value_and_holder(size_t index) : index{index} {}
-
-    template <typename V = void>
-    V *&value_ptr() const {
-        return reinterpret_cast<V *&>(vh[0]);
-    }
-    // True if this `value_and_holder` has a non-null value pointer
-    explicit operator bool() const { return value_ptr() != nullptr; }
-
-    template <typename H>
-    H &holder() const {
-        return reinterpret_cast<H &>(vh[1]);
-    }
-    bool holder_constructed() const {
-        return inst->simple_layout
-                   ? inst->simple_holder_constructed
-                   : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u;
-    }
-    // NOLINTNEXTLINE(readability-make-member-function-const)
-    void set_holder_constructed(bool v = true) {
-        if (inst->simple_layout) {
-            inst->simple_holder_constructed = v;
-        } else if (v) {
-            inst->nonsimple.status[index] |= instance::status_holder_constructed;
-        } else {
-            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed;
-        }
-    }
-    bool instance_registered() const {
-        return inst->simple_layout
-                   ? inst->simple_instance_registered
-                   : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0);
-    }
-    // NOLINTNEXTLINE(readability-make-member-function-const)
-    void set_instance_registered(bool v = true) {
-        if (inst->simple_layout) {
-            inst->simple_instance_registered = v;
-        } else if (v) {
-            inst->nonsimple.status[index] |= instance::status_instance_registered;
-        } else {
-            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered;
-        }
-    }
-};
-
 // Container for accessing and iterating over an instance's values/holders
 struct values_and_holders {
 private:
@@ -488,7 +471,7 @@ PYBIND11_NOINLINE void instance::allocate_layout() {
 // NOLINTNEXTLINE(readability-make-member-function-const)
 PYBIND11_NOINLINE void instance::deallocate_layout() {
     if (!simple_layout) {
-        PyMem_Free(nonsimple.values_and_holders);
+        PyMem_Free(reinterpret_cast<void *>(nonsimple.values_and_holders));
     }
 }
 
@@ -515,7 +498,7 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i
 }
 
 inline PyThreadState *get_thread_state_unchecked() {
-#if defined(PYPY_VERSION)
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
     return PyThreadState_GET();
 #elif PY_VERSION_HEX < 0x030D0000
     return _PyThreadState_UncheckedGet();
@@ -528,6 +511,361 @@ inline PyThreadState *get_thread_state_unchecked() {
 void keep_alive_impl(handle nurse, handle patient);
 inline PyObject *make_new_instance(PyTypeObject *type);
 
+// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code.
+inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo);
+
+PYBIND11_NAMESPACE_BEGIN(smart_holder_type_caster_support)
+
+struct value_and_holder_helper {
+    value_and_holder loaded_v_h;
+
+    bool have_holder() const {
+        return loaded_v_h.vh != nullptr && loaded_v_h.holder_constructed();
+    }
+
+    smart_holder &holder() const { return loaded_v_h.holder<smart_holder>(); }
+
+    void throw_if_uninitialized_or_disowned_holder(const char *typeid_name) const {
+        static const std::string missing_value_msg = "Missing value for wrapped C++ type `";
+        if (!holder().is_populated) {
+            throw value_error(missing_value_msg + clean_type_id(typeid_name)
+                              + "`: Python instance is uninitialized.");
+        }
+        if (!holder().has_pointee()) {
+            throw value_error(missing_value_msg + clean_type_id(typeid_name)
+                              + "`: Python instance was disowned.");
+        }
+    }
+
+    void throw_if_uninitialized_or_disowned_holder(const std::type_info &type_info) const {
+        throw_if_uninitialized_or_disowned_holder(type_info.name());
+    }
+
+    // have_holder() must be true or this function will fail.
+    void throw_if_instance_is_currently_owned_by_shared_ptr() const {
+        auto *vptr_gd_ptr = std::get_deleter<pybindit::memory::guarded_delete>(holder().vptr);
+        if (vptr_gd_ptr != nullptr && !vptr_gd_ptr->released_ptr.expired()) {
+            throw value_error("Python instance is currently owned by a std::shared_ptr.");
+        }
+    }
+
+    void *get_void_ptr_or_nullptr() const {
+        if (have_holder()) {
+            auto &hld = holder();
+            if (hld.is_populated && hld.has_pointee()) {
+                return hld.template as_raw_ptr_unowned<void>();
+            }
+        }
+        return nullptr;
+    }
+};
+
+template <typename T, typename D>
+handle smart_holder_from_unique_ptr(std::unique_ptr<T, D> &&src,
+                                    return_value_policy policy,
+                                    handle parent,
+                                    const std::pair<const void *, const type_info *> &st) {
+    if (policy == return_value_policy::copy) {
+        throw cast_error("return_value_policy::copy is invalid for unique_ptr.");
+    }
+    if (!src) {
+        return none().release();
+    }
+    void *src_raw_void_ptr = const_cast<void *>(st.first);
+    assert(st.second != nullptr);
+    const detail::type_info *tinfo = st.second;
+    if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) {
+        auto *self_life_support
+            = dynamic_raw_ptr_cast_if_possible<trampoline_self_life_support>(src.get());
+        if (self_life_support != nullptr) {
+            value_and_holder &v_h = self_life_support->v_h;
+            if (v_h.inst != nullptr && v_h.vh != nullptr) {
+                auto &holder = v_h.holder<smart_holder>();
+                if (!holder.is_disowned) {
+                    pybind11_fail("smart_holder_from_unique_ptr: unexpected "
+                                  "smart_holder.is_disowned failure.");
+                }
+                // Critical transfer-of-ownership section. This must stay together.
+                self_life_support->deactivate_life_support();
+                holder.reclaim_disowned();
+                (void) src.release();
+                // Critical section end.
+                return existing_inst;
+            }
+        }
+        throw cast_error("Invalid unique_ptr: another instance owns this pointer already.");
+    }
+
+    auto inst = reinterpret_steal<object>(make_new_instance(tinfo->type));
+    auto *inst_raw_ptr = reinterpret_cast<instance *>(inst.ptr());
+    inst_raw_ptr->owned = true;
+    void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr();
+    valueptr = src_raw_void_ptr;
+
+    if (static_cast<void *>(src.get()) == src_raw_void_ptr) {
+        // This is a multiple-inheritance situation that is incompatible with the current
+        // shared_from_this handling (see PR #3023). Is there a better solution?
+        src_raw_void_ptr = nullptr;
+    }
+    auto smhldr = smart_holder::from_unique_ptr(std::move(src), src_raw_void_ptr);
+    tinfo->init_instance(inst_raw_ptr, static_cast<const void *>(&smhldr));
+
+    if (policy == return_value_policy::reference_internal) {
+        keep_alive_impl(inst, parent);
+    }
+
+    return inst.release();
+}
+
+template <typename T, typename D>
+handle smart_holder_from_unique_ptr(std::unique_ptr<T const, D> &&src,
+                                    return_value_policy policy,
+                                    handle parent,
+                                    const std::pair<const void *, const type_info *> &st) {
+    return smart_holder_from_unique_ptr(
+        std::unique_ptr<T, D>(const_cast<T *>(src.release()),
+                              std::move(src.get_deleter())), // Const2Mutbl
+        policy,
+        parent,
+        st);
+}
+
+template <typename T>
+handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
+                                    return_value_policy policy,
+                                    handle parent,
+                                    const std::pair<const void *, const type_info *> &st) {
+    switch (policy) {
+        case return_value_policy::automatic:
+        case return_value_policy::automatic_reference:
+            break;
+        case return_value_policy::take_ownership:
+            throw cast_error("Invalid return_value_policy for shared_ptr (take_ownership).");
+        case return_value_policy::copy:
+        case return_value_policy::move:
+            break;
+        case return_value_policy::reference:
+            throw cast_error("Invalid return_value_policy for shared_ptr (reference).");
+        case return_value_policy::reference_internal:
+            break;
+    }
+    if (!src) {
+        return none().release();
+    }
+
+    auto src_raw_ptr = src.get();
+    assert(st.second != nullptr);
+    void *src_raw_void_ptr = static_cast<void *>(src_raw_ptr);
+    const detail::type_info *tinfo = st.second;
+    if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) {
+        // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing smart_holder.
+        // PYBIND11:REMINDER: MISSING: keep_alive.
+        return existing_inst;
+    }
+
+    auto inst = reinterpret_steal<object>(make_new_instance(tinfo->type));
+    auto *inst_raw_ptr = reinterpret_cast<instance *>(inst.ptr());
+    inst_raw_ptr->owned = true;
+    void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr();
+    valueptr = src_raw_void_ptr;
+
+    auto smhldr
+        = smart_holder::from_shared_ptr(std::shared_ptr<void>(src, const_cast<void *>(st.first)));
+    tinfo->init_instance(inst_raw_ptr, static_cast<const void *>(&smhldr));
+
+    if (policy == return_value_policy::reference_internal) {
+        keep_alive_impl(inst, parent);
+    }
+
+    return inst.release();
+}
+
+template <typename T>
+handle smart_holder_from_shared_ptr(const std::shared_ptr<T const> &src,
+                                    return_value_policy policy,
+                                    handle parent,
+                                    const std::pair<const void *, const type_info *> &st) {
+    return smart_holder_from_shared_ptr(std::const_pointer_cast<T>(src), // Const2Mutbl
+                                        policy,
+                                        parent,
+                                        st);
+}
+
+struct shared_ptr_parent_life_support {
+    PyObject *parent;
+    explicit shared_ptr_parent_life_support(PyObject *parent) : parent{parent} {
+        Py_INCREF(parent);
+    }
+    // NOLINTNEXTLINE(readability-make-member-function-const)
+    void operator()(void *) {
+        gil_scoped_acquire gil;
+        Py_DECREF(parent);
+    }
+};
+
+struct shared_ptr_trampoline_self_life_support {
+    PyObject *self;
+    explicit shared_ptr_trampoline_self_life_support(instance *inst)
+        : self{reinterpret_cast<PyObject *>(inst)} {
+        gil_scoped_acquire gil;
+        Py_INCREF(self);
+    }
+    // NOLINTNEXTLINE(readability-make-member-function-const)
+    void operator()(void *) {
+        gil_scoped_acquire gil;
+        Py_DECREF(self);
+    }
+};
+
+template <typename T,
+          typename D,
+          typename std::enable_if<std::is_default_constructible<D>::value, int>::type = 0>
+inline std::unique_ptr<T, D> unique_with_deleter(T *raw_ptr, std::unique_ptr<D> &&deleter) {
+    if (deleter == nullptr) {
+        return std::unique_ptr<T, D>(raw_ptr);
+    }
+    return std::unique_ptr<T, D>(raw_ptr, std::move(*deleter));
+}
+
+template <typename T,
+          typename D,
+          typename std::enable_if<!std::is_default_constructible<D>::value, int>::type = 0>
+inline std::unique_ptr<T, D> unique_with_deleter(T *raw_ptr, std::unique_ptr<D> &&deleter) {
+    if (deleter == nullptr) {
+        pybind11_fail("smart_holder_type_casters: deleter is not default constructible and no"
+                      " instance available to return.");
+    }
+    return std::unique_ptr<T, D>(raw_ptr, std::move(*deleter));
+}
+
+template <typename T>
+struct load_helper : value_and_holder_helper {
+    bool was_populated = false;
+    bool python_instance_is_alias = false;
+
+    void maybe_set_python_instance_is_alias(handle src) {
+        if (was_populated) {
+            python_instance_is_alias = reinterpret_cast<instance *>(src.ptr())->is_alias;
+        }
+    }
+
+    static std::shared_ptr<T> make_shared_ptr_with_responsible_parent(T *raw_ptr, handle parent) {
+        return std::shared_ptr<T>(raw_ptr, shared_ptr_parent_life_support(parent.ptr()));
+    }
+
+    std::shared_ptr<T> load_as_shared_ptr(void *void_raw_ptr,
+                                          handle responsible_parent = nullptr) const {
+        if (!have_holder()) {
+            return nullptr;
+        }
+        throw_if_uninitialized_or_disowned_holder(typeid(T));
+        smart_holder &hld = holder();
+        hld.ensure_is_not_disowned("load_as_shared_ptr");
+        if (hld.vptr_is_using_noop_deleter) {
+            if (responsible_parent) {
+                return make_shared_ptr_with_responsible_parent(static_cast<T *>(void_raw_ptr),
+                                                               responsible_parent);
+            }
+            throw std::runtime_error("Non-owning holder (load_as_shared_ptr).");
+        }
+        auto *type_raw_ptr = static_cast<T *>(void_raw_ptr);
+        if (python_instance_is_alias) {
+            auto *vptr_gd_ptr = std::get_deleter<pybindit::memory::guarded_delete>(hld.vptr);
+            if (vptr_gd_ptr != nullptr) {
+                std::shared_ptr<void> released_ptr = vptr_gd_ptr->released_ptr.lock();
+                if (released_ptr) {
+                    return std::shared_ptr<T>(released_ptr, type_raw_ptr);
+                }
+                std::shared_ptr<T> to_be_released(
+                    type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst));
+                vptr_gd_ptr->released_ptr = to_be_released;
+                return to_be_released;
+            }
+            auto *sptsls_ptr = std::get_deleter<shared_ptr_trampoline_self_life_support>(hld.vptr);
+            if (sptsls_ptr != nullptr) {
+                // This code is reachable only if there are multiple registered_instances for the
+                // same pointee.
+                if (reinterpret_cast<PyObject *>(loaded_v_h.inst) == sptsls_ptr->self) {
+                    pybind11_fail("smart_holder_type_caster_support load_as_shared_ptr failure: "
+                                  "loaded_v_h.inst == sptsls_ptr->self");
+                }
+            }
+            if (sptsls_ptr != nullptr
+                || !pybindit::memory::type_has_shared_from_this(type_raw_ptr)) {
+                return std::shared_ptr<T>(
+                    type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst));
+            }
+            if (hld.vptr_is_external_shared_ptr) {
+                pybind11_fail("smart_holder_type_casters load_as_shared_ptr failure: not "
+                              "implemented: trampoline-self-life-support for external shared_ptr "
+                              "to type inheriting from std::enable_shared_from_this.");
+            }
+            pybind11_fail(
+                "smart_holder_type_casters: load_as_shared_ptr failure: internal inconsistency.");
+        }
+        std::shared_ptr<void> void_shd_ptr = hld.template as_shared_ptr<void>();
+        return std::shared_ptr<T>(void_shd_ptr, type_raw_ptr);
+    }
+
+    template <typename D>
+    std::unique_ptr<T, D> load_as_unique_ptr(void *raw_void_ptr,
+                                             const char *context = "load_as_unique_ptr") {
+        if (!have_holder()) {
+            return unique_with_deleter<T, D>(nullptr, std::unique_ptr<D>());
+        }
+        throw_if_uninitialized_or_disowned_holder(typeid(T));
+        throw_if_instance_is_currently_owned_by_shared_ptr();
+        holder().ensure_is_not_disowned(context);
+        holder().template ensure_compatible_rtti_uqp_del<T, D>(context);
+        holder().ensure_use_count_1(context);
+
+        T *raw_type_ptr = static_cast<T *>(raw_void_ptr);
+
+        auto *self_life_support
+            = dynamic_raw_ptr_cast_if_possible<trampoline_self_life_support>(raw_type_ptr);
+        if (self_life_support == nullptr && python_instance_is_alias) {
+            throw value_error("Alias class (also known as trampoline) does not inherit from "
+                              "py::trampoline_self_life_support, therefore the ownership of this "
+                              "instance cannot safely be transferred to C++.");
+        }
+
+        std::unique_ptr<D> extracted_deleter = holder().template extract_deleter<T, D>(context);
+
+        // Critical transfer-of-ownership section. This must stay together.
+        if (self_life_support != nullptr) {
+            holder().disown();
+        } else {
+            holder().release_ownership();
+        }
+        auto result = unique_with_deleter<T, D>(raw_type_ptr, std::move(extracted_deleter));
+        if (self_life_support != nullptr) {
+            self_life_support->activate_life_support(loaded_v_h);
+        } else {
+            void *value_void_ptr = loaded_v_h.value_ptr();
+            loaded_v_h.value_ptr() = nullptr;
+            deregister_instance(loaded_v_h.inst, value_void_ptr, loaded_v_h.type);
+        }
+        // Critical section end.
+
+        return result;
+    }
+
+    // This assumes load_as_shared_ptr succeeded(), and the returned shared_ptr is still alive.
+    // The returned unique_ptr is meant to never expire (the behavior is undefined otherwise).
+    template <typename D>
+    std::unique_ptr<T, D>
+    load_as_const_unique_ptr(T *raw_type_ptr, const char *context = "load_as_const_unique_ptr") {
+        if (!have_holder()) {
+            return unique_with_deleter<T, D>(nullptr, std::unique_ptr<D>());
+        }
+        holder().template ensure_compatible_rtti_uqp_del<T, D>(context);
+        return unique_with_deleter<T, D>(
+            raw_type_ptr, std::move(holder().template extract_deleter<T, D>(context)));
+    }
+};
+
+PYBIND11_NAMESPACE_END(smart_holder_type_caster_support)
+
 class type_caster_generic {
 public:
     PYBIND11_NOINLINE explicit type_caster_generic(const std::type_info &type_info)
@@ -632,6 +970,15 @@ public:
 
     // Base methods for generic caster; there are overridden in copyable_holder_caster
     void load_value(value_and_holder &&v_h) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+            smart_holder_type_caster_support::value_and_holder_helper v_h_helper;
+            v_h_helper.loaded_v_h = v_h;
+            if (v_h_helper.have_holder()) {
+                v_h_helper.throw_if_uninitialized_or_disowned_holder(cpptype->name());
+                value = v_h_helper.holder().template as_raw_ptr_unowned<void>();
+                return;
+            }
+        }
         auto *&vptr = v_h.value_ptr();
         // Lazy allocation for unallocated values:
         if (vptr == nullptr) {
@@ -670,6 +1017,13 @@ public:
         }
         return false;
     }
+    bool try_cpp_conduit(handle src) {
+        value = try_raw_pointer_ephemeral_from_cpp_conduit(src, cpptype);
+        if (value != nullptr) {
+            return true;
+        }
+        return false;
+    }
     void check_holder_compat() {}
 
     PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) {
@@ -801,6 +1155,10 @@ public:
             return true;
         }
 
+        if (convert && cpptype && this_.try_cpp_conduit(src)) {
+            return true;
+        }
+
         return false;
     }
 
@@ -828,6 +1186,32 @@ public:
     void *value = nullptr;
 };
 
+inline object cpp_conduit_method(handle self,
+                                 const bytes &pybind11_platform_abi_id,
+                                 const capsule &cpp_type_info_capsule,
+                                 const bytes &pointer_kind) {
+#ifdef PYBIND11_HAS_STRING_VIEW
+    using cpp_str = std::string_view;
+#else
+    using cpp_str = std::string;
+#endif
+    if (cpp_str(pybind11_platform_abi_id) != PYBIND11_PLATFORM_ABI_ID) {
+        return none();
+    }
+    if (std::strcmp(cpp_type_info_capsule.name(), typeid(std::type_info).name()) != 0) {
+        return none();
+    }
+    if (cpp_str(pointer_kind) != "raw_pointer_ephemeral") {
+        throw std::runtime_error("Invalid pointer_kind: \"" + std::string(pointer_kind) + "\"");
+    }
+    const auto *cpp_type_info = cpp_type_info_capsule.get_pointer<const std::type_info>();
+    type_caster_generic caster(*cpp_type_info);
+    if (!caster.load(self, false)) {
+        return none();
+    }
+    return capsule(caster.value, cpp_type_info->name());
+}
+
 /**
  * Determine suitable casting operator for pointer-or-lvalue-casting type casters.  The type caster
  * needs to provide `operator T*()` and `operator T&()` operators.
@@ -1180,14 +1564,14 @@ protected:
        does not have a private operator new implementation. A comma operator is used in the
        decltype argument to apply SFINAE to the public copy/move constructors.*/
     template <typename T, typename = enable_if_t<is_copy_constructible<T>::value>>
-    static auto make_copy_constructor(const T *) -> decltype(new T(std::declval<const T>()),
-                                                             Constructor{}) {
+    static auto make_copy_constructor(const T *)
+        -> decltype(new T(std::declval<const T>()), Constructor{}) {
         return [](const void *arg) -> void * { return new T(*reinterpret_cast<const T *>(arg)); };
     }
 
     template <typename T, typename = enable_if_t<is_move_constructible<T>::value>>
-    static auto make_move_constructor(const T *) -> decltype(new T(std::declval<T &&>()),
-                                                             Constructor{}) {
+    static auto make_move_constructor(const T *)
+        -> decltype(new T(std::declval<T &&>()), Constructor{}) {
         return [](const void *arg) -> void * {
             return new T(std::move(*const_cast<T *>(reinterpret_cast<const T *>(arg))));
         };

pybind11/include/pybind11/detail/using_smart_holder.h

@@ -0,0 +1,22 @@
+// Copyright (c) 2024 The Pybind Development Team.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "common.h"
+#include "struct_smart_holder.h"
+
+#include <type_traits>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+using pybindit::memory::smart_holder;
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <typename H>
+using is_smart_holder = std::is_same<H, smart_holder>;
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/detail/value_and_holder.h

@@ -0,0 +1,78 @@
+// Copyright (c) 2016-2024 The Pybind Development Team.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "common.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <typeinfo>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+struct value_and_holder {
+    instance *inst = nullptr;
+    size_t index = 0u;
+    const detail::type_info *type = nullptr;
+    void **vh = nullptr;
+
+    // Main constructor for a found value/holder:
+    value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index)
+        : inst{i}, index{index}, type{type},
+          vh{inst->simple_layout ? inst->simple_value_holder
+                                 : &inst->nonsimple.values_and_holders[vpos]} {}
+
+    // Default constructor (used to signal a value-and-holder not found by get_value_and_holder())
+    value_and_holder() = default;
+
+    // Used for past-the-end iterator
+    explicit value_and_holder(size_t index) : index{index} {}
+
+    template <typename V = void>
+    V *&value_ptr() const {
+        return reinterpret_cast<V *&>(vh[0]);
+    }
+    // True if this `value_and_holder` has a non-null value pointer
+    explicit operator bool() const { return value_ptr() != nullptr; }
+
+    template <typename H>
+    H &holder() const {
+        return reinterpret_cast<H &>(vh[1]);
+    }
+    bool holder_constructed() const {
+        return inst->simple_layout
+                   ? inst->simple_holder_constructed
+                   : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u;
+    }
+    // NOLINTNEXTLINE(readability-make-member-function-const)
+    void set_holder_constructed(bool v = true) {
+        if (inst->simple_layout) {
+            inst->simple_holder_constructed = v;
+        } else if (v) {
+            inst->nonsimple.status[index] |= instance::status_holder_constructed;
+        } else {
+            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed;
+        }
+    }
+    bool instance_registered() const {
+        return inst->simple_layout
+                   ? inst->simple_instance_registered
+                   : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0);
+    }
+    // NOLINTNEXTLINE(readability-make-member-function-const)
+    void set_instance_registered(bool v = true) {
+        if (inst->simple_layout) {
+            inst->simple_instance_registered = v;
+        } else if (v) {
+            inst->nonsimple.status[index] |= instance::status_instance_registered;
+        } else {
+            inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered;
+        }
+    }
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/eigen/matrix.h

@@ -9,7 +9,8 @@
 
 #pragma once
 
-#include "../numpy.h"
+#include <pybind11/numpy.h>
+
 #include "common.h"
 
 /* HINT: To suppress warnings originating from the Eigen headers, use -isystem.
@@ -224,19 +225,22 @@ struct EigenProps {
         = !show_c_contiguous && show_order && requires_col_major;
 
     static constexpr auto descriptor
-        = const_name("numpy.ndarray[") + npy_format_descriptor<Scalar>::name + const_name("[")
+        = const_name("typing.Annotated[")
+          + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[")
+          + npy_format_descriptor<Scalar>::name + io_name("", "]") + const_name(", \"[")
           + const_name<fixed_rows>(const_name<(size_t) rows>(), const_name("m")) + const_name(", ")
-          + const_name<fixed_cols>(const_name<(size_t) cols>(), const_name("n")) + const_name("]")
-          +
+          + const_name<fixed_cols>(const_name<(size_t) cols>(), const_name("n"))
+          + const_name("]\"")
           // For a reference type (e.g. Ref<MatrixXd>) we have other constraints that might need to
           // be satisfied: writeable=True (for a mutable reference), and, depending on the map's
           // stride options, possibly f_contiguous or c_contiguous.  We include them in the
           // descriptor output to provide some hint as to why a TypeError is occurring (otherwise
-          // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and
-          // an error message that you *gave* a numpy.ndarray of the right type and dimensions.
-          const_name<show_writeable>(", flags.writeable", "")
-          + const_name<show_c_contiguous>(", flags.c_contiguous", "")
-          + const_name<show_f_contiguous>(", flags.f_contiguous", "") + const_name("]");
+          // it can be confusing to see that a function accepts a
+          // 'typing.Annotated[numpy.typing.NDArray[numpy.float64], "[3,2]"]' and an error message
+          // that you *gave* a numpy.ndarray of the right type and dimensions.
+          + const_name<show_writeable>(", \"flags.writeable\"", "")
+          + const_name<show_c_contiguous>(", \"flags.c_contiguous\"", "")
+          + const_name<show_f_contiguous>(", \"flags.f_contiguous\"", "") + const_name("]");
 };
 
 // Casts an Eigen type to numpy array.  If given a base, the numpy array references the src data,
@@ -315,8 +319,11 @@ struct type_caster<Type, enable_if_t<is_eigen_dense_plain<Type>::value>> {
             return false;
         }
 
+        PYBIND11_WARNING_PUSH
+        PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") // See PR #5516
         // Allocate the new type, then build a numpy reference into it
         value = Type(fits.rows, fits.cols);
+        PYBIND11_WARNING_POP
         auto ref = reinterpret_steal<array>(eigen_ref_array<props>(value));
         if (dims == 1) {
             ref = ref.squeeze();
@@ -437,7 +444,9 @@ public:
         }
     }
 
-    static constexpr auto name = props::descriptor;
+    // return_descr forces the use of NDArray instead of ArrayLike in args
+    // since Ref<...> args can only accept arrays.
+    static constexpr auto name = return_descr(props::descriptor);
 
     // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return
     // types but not bound arguments).  We still provide them (with an explicitly delete) so that

pybind11/include/pybind11/eigen/tensor.h

@@ -7,7 +7,8 @@
 
 #pragma once
 
-#include "../numpy.h"
+#include <pybind11/numpy.h>
+
 #include "common.h"
 
 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
@@ -123,13 +124,16 @@ struct eigen_tensor_helper<
 template <typename Type, bool ShowDetails, bool NeedsWriteable = false>
 struct get_tensor_descriptor {
     static constexpr auto details
-        = const_name<NeedsWriteable>(", flags.writeable", "")
-          + const_name<static_cast<int>(Type::Layout) == static_cast<int>(Eigen::RowMajor)>(
-              ", flags.c_contiguous", ", flags.f_contiguous");
+        = const_name<NeedsWriteable>(", \"flags.writeable\"", "") + const_name
+              < static_cast<int>(Type::Layout)
+          == static_cast<int>(Eigen::RowMajor)
+                 > (", \"flags.c_contiguous\"", ", \"flags.f_contiguous\"");
     static constexpr auto value
-        = const_name("numpy.ndarray[") + npy_format_descriptor<typename Type::Scalar>::name
-          + const_name("[") + eigen_tensor_helper<remove_cv_t<Type>>::dimensions_descriptor
-          + const_name("]") + const_name<ShowDetails>(details, const_name("")) + const_name("]");
+        = const_name("typing.Annotated[")
+          + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[")
+          + npy_format_descriptor<typename Type::Scalar>::name + io_name("", "]")
+          + const_name(", \"[") + eigen_tensor_helper<remove_cv_t<Type>>::dimensions_descriptor
+          + const_name("]\"") + const_name<ShowDetails>(details, const_name("")) + const_name("]");
 };
 
 // When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes<T, 0> does not have the begin() member
@@ -469,9 +473,6 @@ struct type_caster<Eigen::TensorMap<Type, Options>,
                 parent_object = reinterpret_borrow<object>(parent);
                 break;
 
-            case return_value_policy::take_ownership:
-                delete src;
-                // fallthrough
             default:
                 // move, take_ownership don't make any sense for a ref/map:
                 pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either "
@@ -504,7 +505,10 @@ protected:
     std::unique_ptr<MapType> value;
 
 public:
-    static constexpr auto name = get_tensor_descriptor<Type, true, needs_writeable>::value;
+    // return_descr forces the use of NDArray instead of ArrayLike since refs can only reference
+    // arrays
+    static constexpr auto name
+        = return_descr(get_tensor_descriptor<Type, true, needs_writeable>::value);
     explicit operator MapType *() { return value.get(); }
     explicit operator MapType &() { return *value; }
     explicit operator MapType &&() && { return std::move(*value); }

pybind11/include/pybind11/embed.h

@@ -104,23 +104,13 @@ inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers,
     detail::precheck_interpreter();
     Py_InitializeEx(init_signal_handlers ? 1 : 0);
 
-    // Before it was special-cased in python 3.8, passing an empty or null argv
-    // caused a segfault, so we have to reimplement the special case ourselves.
-    bool special_case = (argv == nullptr || argc <= 0);
-
-    const char *const empty_argv[]{"\0"};
-    const char *const *safe_argv = special_case ? empty_argv : argv;
-    if (special_case) {
-        argc = 1;
-    }
-
     auto argv_size = static_cast<size_t>(argc);
     // SetArgv* on python 3 takes wchar_t, so we have to convert.
     std::unique_ptr<wchar_t *[]> widened_argv(new wchar_t *[argv_size]);
     std::vector<std::unique_ptr<wchar_t[], detail::wide_char_arg_deleter>> widened_argv_entries;
     widened_argv_entries.reserve(argv_size);
     for (size_t ii = 0; ii < argv_size; ++ii) {
-        widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii]));
+        widened_argv_entries.emplace_back(detail::widen_chars(argv[ii]));
         if (!widened_argv_entries.back()) {
             // A null here indicates a character-encoding failure or the python
             // interpreter out of memory. Give up.

pybind11/include/pybind11/eval.h

@@ -19,7 +19,7 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
 
 inline void ensure_builtins_in_globals(object &global) {
-#if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x03080000
+#if defined(PYPY_VERSION)
     // Running exec and eval adds `builtins` module under `__builtins__` key to
     // globals if not yet present.  Python 3.8 made PyRun_String behave
     // similarly. Let's also do that for older versions, for consistency. This
@@ -94,18 +94,18 @@ void exec(const char (&s)[N], object global = globals(), object local = object()
     eval<eval_statements>(s, std::move(global), std::move(local));
 }
 
-#if defined(PYPY_VERSION)
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
 template <eval_mode mode = eval_statements>
 object eval_file(str, object, object) {
-    pybind11_fail("eval_file not supported in PyPy3. Use eval");
+    pybind11_fail("eval_file not supported in this interpreter. Use eval");
 }
 template <eval_mode mode = eval_statements>
 object eval_file(str, object) {
-    pybind11_fail("eval_file not supported in PyPy3. Use eval");
+    pybind11_fail("eval_file not supported in this interpreter. Use eval");
 }
 template <eval_mode mode = eval_statements>
 object eval_file(str) {
-    pybind11_fail("eval_file not supported in PyPy3. Use eval");
+    pybind11_fail("eval_file not supported in this interpreter. Use eval");
 }
 #else
 template <eval_mode mode = eval_statements>

pybind11/include/pybind11/functional.h

@@ -9,12 +9,55 @@
 
 #pragma once
 
+#define PYBIND11_HAS_TYPE_CASTER_STD_FUNCTION_SPECIALIZATIONS
+
 #include "pybind11.h"
 
 #include <functional>
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
+PYBIND11_NAMESPACE_BEGIN(type_caster_std_function_specializations)
+
+// ensure GIL is held during functor destruction
+struct func_handle {
+    function f;
+#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17))
+    // This triggers a syntax error under very special conditions (very weird indeed).
+    explicit
+#endif
+        func_handle(function &&f_) noexcept
+        : f(std::move(f_)) {
+    }
+    func_handle(const func_handle &f_) { operator=(f_); }
+    func_handle &operator=(const func_handle &f_) {
+        gil_scoped_acquire acq;
+        f = f_.f;
+        return *this;
+    }
+    ~func_handle() {
+        gil_scoped_acquire acq;
+        function kill_f(std::move(f));
+    }
+};
+
+// to emulate 'move initialization capture' in C++11
+struct func_wrapper_base {
+    func_handle hfunc;
+    explicit func_wrapper_base(func_handle &&hf) noexcept : hfunc(hf) {}
+};
+
+template <typename Return, typename... Args>
+struct func_wrapper : func_wrapper_base {
+    using func_wrapper_base::func_wrapper_base;
+    Return operator()(Args... args) const {
+        gil_scoped_acquire acq;
+        // casts the returned object as a rvalue to the return type
+        return hfunc.f(std::forward<Args>(args)...).template cast<Return>();
+    }
+};
+
+PYBIND11_NAMESPACE_END(type_caster_std_function_specializations)
 
 template <typename Return, typename... Args>
 struct type_caster<std::function<Return(Args...)>> {
@@ -77,40 +120,8 @@ public:
             // See PR #1413 for full details
         }
 
-        // ensure GIL is held during functor destruction
-        struct func_handle {
-            function f;
-#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17))
-            // This triggers a syntax error under very special conditions (very weird indeed).
-            explicit
-#endif
-                func_handle(function &&f_) noexcept
-                : f(std::move(f_)) {
-            }
-            func_handle(const func_handle &f_) { operator=(f_); }
-            func_handle &operator=(const func_handle &f_) {
-                gil_scoped_acquire acq;
-                f = f_.f;
-                return *this;
-            }
-            ~func_handle() {
-                gil_scoped_acquire acq;
-                function kill_f(std::move(f));
-            }
-        };
-
-        // to emulate 'move initialization capture' in C++11
-        struct func_wrapper {
-            func_handle hfunc;
-            explicit func_wrapper(func_handle &&hf) noexcept : hfunc(std::move(hf)) {}
-            Return operator()(Args... args) const {
-                gil_scoped_acquire acq;
-                // casts the returned object as a rvalue to the return type
-                return hfunc.f(std::forward<Args>(args)...).template cast<Return>();
-            }
-        };
-
-        value = func_wrapper(func_handle(std::move(func)));
+        value = type_caster_std_function_specializations::func_wrapper<Return, Args...>(
+            type_caster_std_function_specializations::func_handle(std::move(func)));
         return true;
     }
 

pybind11/include/pybind11/gil.h

@@ -147,9 +147,7 @@ public:
         // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer)
         tstate = PyEval_SaveThread();
         if (disassoc) {
-            // Python >= 3.7 can remove this, it's an int before 3.7
-            // NOLINTNEXTLINE(readability-qualified-auto)
-            auto key = internals.tstate;
+            auto key = internals.tstate; // NOLINT(readability-qualified-auto)
             PYBIND11_TLS_DELETE_VALUE(key);
         }
     }
@@ -173,9 +171,7 @@ public:
             PyEval_RestoreThread(tstate);
         }
         if (disassoc) {
-            // Python >= 3.7 can remove this, it's an int before 3.7
-            // NOLINTNEXTLINE(readability-qualified-auto)
-            auto key = detail::get_internals().tstate;
+            auto key = detail::get_internals().tstate; // NOLINT(readability-qualified-auto)
             PYBIND11_TLS_REPLACE_VALUE(key, tstate);
         }
     }

pybind11/include/pybind11/gil_safe_call_once.h

@@ -8,6 +8,10 @@
 #include <cassert>
 #include <mutex>
 
+#ifdef Py_GIL_DISABLED
+#    include <atomic>
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
 // Use the `gil_safe_call_once_and_store` class below instead of the naive
@@ -42,6 +46,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 // get processed only when it is the main thread's turn again and it is running
 // normal Python code. However, this will be unnoticeable for quick call-once
 // functions, which is usually the case.
+//
+// For in-depth background, see docs/advanced/deadlock.md
 template <typename T>
 class gil_safe_call_once_and_store {
 public:
@@ -82,7 +88,12 @@ public:
 private:
     alignas(T) char storage_[sizeof(T)] = {};
     std::once_flag once_flag_ = {};
-    bool is_initialized_ = false;
+#ifdef Py_GIL_DISABLED
+    std::atomic_bool
+#else
+    bool
+#endif
+        is_initialized_{false};
     // The `is_initialized_`-`storage_` pair is very similar to `std::optional`,
     // but the latter does not have the triviality properties of former,
     // therefore `std::optional` is not a viable alternative here.

pybind11/include/pybind11/numpy.h

@@ -175,7 +175,6 @@ inline numpy_internals &get_numpy_internals() {
 PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) {
     module_ numpy = module_::import("numpy");
     str version_string = numpy.attr("__version__");
-
     module_ numpy_lib = module_::import("numpy.lib");
     object numpy_version = numpy_lib.attr("NumpyVersion")(version_string);
     int major_version = numpy_version.attr("major").cast<int>();
@@ -212,6 +211,7 @@ constexpr int platform_lookup(int I, Ints... Is) {
 }
 
 struct npy_api {
+    // If you change this code, please review `normalized_dtype_num` below.
     enum constants {
         NPY_ARRAY_C_CONTIGUOUS_ = 0x0001,
         NPY_ARRAY_F_CONTIGUOUS_ = 0x0002,
@@ -384,6 +384,74 @@ private:
     }
 };
 
+// This table normalizes typenums by mapping NPY_INT_, NPY_LONG, ... to NPY_INT32_, NPY_INT64, ...
+// This is needed to correctly handle situations where multiple typenums map to the same type,
+// e.g. NPY_LONG_ may be equivalent to NPY_INT_ or NPY_LONGLONG_ despite having a different
+// typenum. The normalized typenum should always match the values used in npy_format_descriptor.
+// If you change this code, please review `enum constants` above.
+static constexpr int normalized_dtype_num[npy_api::NPY_VOID_ + 1] = {
+    // NPY_BOOL_ =>
+    npy_api::NPY_BOOL_,
+    // NPY_BYTE_ =>
+    npy_api::NPY_BYTE_,
+    // NPY_UBYTE_ =>
+    npy_api::NPY_UBYTE_,
+    // NPY_SHORT_ =>
+    npy_api::NPY_INT16_,
+    // NPY_USHORT_ =>
+    npy_api::NPY_UINT16_,
+    // NPY_INT_ =>
+    sizeof(int) == sizeof(std::int16_t)   ? npy_api::NPY_INT16_
+    : sizeof(int) == sizeof(std::int32_t) ? npy_api::NPY_INT32_
+    : sizeof(int) == sizeof(std::int64_t) ? npy_api::NPY_INT64_
+                                          : npy_api::NPY_INT_,
+    // NPY_UINT_ =>
+    sizeof(unsigned int) == sizeof(std::uint16_t)   ? npy_api::NPY_UINT16_
+    : sizeof(unsigned int) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_
+    : sizeof(unsigned int) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_
+                                                    : npy_api::NPY_UINT_,
+    // NPY_LONG_ =>
+    sizeof(long) == sizeof(std::int16_t)   ? npy_api::NPY_INT16_
+    : sizeof(long) == sizeof(std::int32_t) ? npy_api::NPY_INT32_
+    : sizeof(long) == sizeof(std::int64_t) ? npy_api::NPY_INT64_
+                                           : npy_api::NPY_LONG_,
+    // NPY_ULONG_ =>
+    sizeof(unsigned long) == sizeof(std::uint16_t)   ? npy_api::NPY_UINT16_
+    : sizeof(unsigned long) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_
+    : sizeof(unsigned long) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_
+                                                     : npy_api::NPY_ULONG_,
+    // NPY_LONGLONG_ =>
+    sizeof(long long) == sizeof(std::int16_t)   ? npy_api::NPY_INT16_
+    : sizeof(long long) == sizeof(std::int32_t) ? npy_api::NPY_INT32_
+    : sizeof(long long) == sizeof(std::int64_t) ? npy_api::NPY_INT64_
+                                                : npy_api::NPY_LONGLONG_,
+    // NPY_ULONGLONG_ =>
+    sizeof(unsigned long long) == sizeof(std::uint16_t)   ? npy_api::NPY_UINT16_
+    : sizeof(unsigned long long) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_
+    : sizeof(unsigned long long) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_
+                                                          : npy_api::NPY_ULONGLONG_,
+    // NPY_FLOAT_ =>
+    npy_api::NPY_FLOAT_,
+    // NPY_DOUBLE_ =>
+    npy_api::NPY_DOUBLE_,
+    // NPY_LONGDOUBLE_ =>
+    npy_api::NPY_LONGDOUBLE_,
+    // NPY_CFLOAT_ =>
+    npy_api::NPY_CFLOAT_,
+    // NPY_CDOUBLE_ =>
+    npy_api::NPY_CDOUBLE_,
+    // NPY_CLONGDOUBLE_ =>
+    npy_api::NPY_CLONGDOUBLE_,
+    // NPY_OBJECT_ =>
+    npy_api::NPY_OBJECT_,
+    // NPY_STRING_ =>
+    npy_api::NPY_STRING_,
+    // NPY_UNICODE_ =>
+    npy_api::NPY_UNICODE_,
+    // NPY_VOID_ =>
+    npy_api::NPY_VOID_,
+};
+
 inline PyArray_Proxy *array_proxy(void *ptr) { return reinterpret_cast<PyArray_Proxy *>(ptr); }
 
 inline const PyArray_Proxy *array_proxy(const void *ptr) {
@@ -684,6 +752,13 @@ public:
         return detail::npy_format_descriptor<typename std::remove_cv<T>::type>::dtype();
     }
 
+    /// Return the type number associated with a C++ type.
+    /// This is the constexpr equivalent of `dtype::of<T>().num()`.
+    template <typename T>
+    static constexpr int num_of() {
+        return detail::npy_format_descriptor<typename std::remove_cv<T>::type>::value;
+    }
+
     /// Size of the data type in bytes.
 #ifdef PYBIND11_NUMPY_1_ONLY
     ssize_t itemsize() const { return detail::array_descriptor_proxy(m_ptr)->elsize; }
@@ -725,7 +800,9 @@ public:
         return detail::array_descriptor_proxy(m_ptr)->type;
     }
 
-    /// type number of dtype.
+    /// Type number of dtype. Note that different values may be returned for equivalent types,
+    /// e.g. even though ``long`` may be equivalent to ``int`` or ``long long``, they still have
+    /// different type numbers. Consider using `normalized_num` to avoid this.
     int num() const {
         // Note: The signature, `dtype::num` follows the naming of NumPy's public
         // Python API (i.e., ``dtype.num``), rather than its internal
@@ -733,6 +810,17 @@ public:
         return detail::array_descriptor_proxy(m_ptr)->type_num;
     }
 
+    /// Type number of dtype, normalized to match the return value of `num_of` for equivalent
+    /// types. This function can be used to write switch statements that correctly handle
+    /// equivalent types with different type numbers.
+    int normalized_num() const {
+        int value = num();
+        if (value >= 0 && value <= detail::npy_api::NPY_VOID_) {
+            return detail::normalized_dtype_num[value];
+        }
+        return value;
+    }
+
     /// Single character for byteorder
     char byteorder() const { return detail::array_descriptor_proxy(m_ptr)->byteorder; }
 
@@ -901,7 +989,11 @@ public:
 
     template <typename T>
     array(ShapeContainer shape, StridesContainer strides, const T *ptr, handle base = handle())
-        : array(pybind11::dtype::of<T>(), std::move(shape), std::move(strides), ptr, base) {}
+        : array(pybind11::dtype::of<T>(),
+                std::move(shape),
+                std::move(strides),
+                reinterpret_cast<const void *>(ptr),
+                base) {}
 
     template <typename T>
     array(ShapeContainer shape, const T *ptr, handle base = handle())
@@ -1424,7 +1516,11 @@ public:
 };
 
 template <typename T>
-struct npy_format_descriptor<T, enable_if_t<is_same_ignoring_cvref<T, PyObject *>::value>> {
+struct npy_format_descriptor<
+    T,
+    enable_if_t<is_same_ignoring_cvref<T, PyObject *>::value
+                || ((std::is_same<T, handle>::value || std::is_same<T, object>::value)
+                    && sizeof(T) == sizeof(PyObject *))>> {
     static constexpr auto name = const_name("object");
 
     static constexpr int value = npy_api::NPY_OBJECT_;
@@ -1986,7 +2082,7 @@ private:
         // Pointers to values the function was called with; the vectorized ones set here will start
         // out as array_t<T> pointers, but they will be changed them to T pointers before we make
         // call the wrapped function.  Non-vectorized pointers are left as-is.
-        std::array<void *, N> params{{&args...}};
+        std::array<void *, N> params{{reinterpret_cast<void *>(&args)...}};
 
         // The array of `buffer_info`s of vectorized arguments:
         std::array<buffer_info, NVectorized> buffers{
@@ -2086,7 +2182,8 @@ vectorize_helper<Func, Return, Args...> vectorize_extractor(const Func &f, Retur
 template <typename T, int Flags>
 struct handle_type_name<array_t<T, Flags>> {
     static constexpr auto name
-        = const_name("numpy.ndarray[") + npy_format_descriptor<T>::name + const_name("]");
+        = io_name("typing.Annotated[numpy.typing.ArrayLike, ", "numpy.typing.NDArray[")
+          + npy_format_descriptor<T>::name + const_name("]");
 };
 
 PYBIND11_NAMESPACE_END(detail)

pybind11/include/pybind11/pybind11.h

@@ -9,9 +9,11 @@
 */
 
 #pragma once
-
 #include "detail/class.h"
+#include "detail/dynamic_raw_ptr_cast_if_possible.h"
+#include "detail/exception_translation.h"
 #include "detail/init.h"
+#include "detail/using_smart_holder.h"
 #include "attr.h"
 #include "gil.h"
 #include "gil_safe_call_once.h"
@@ -22,10 +24,17 @@
 #include <cstring>
 #include <memory>
 #include <new>
+#include <stack>
 #include <string>
 #include <utility>
 #include <vector>
 
+// See PR #5448. This warning suppression is needed for the PYBIND11_OVERRIDE macro family.
+// NOTE that this is NOT embedded in a push/pop pair because that is very difficult to achieve.
+#if defined(__clang_major__) && __clang_major__ < 14
+PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments")
+#endif
+
 #if defined(__cpp_lib_launder) && !(defined(_MSC_VER) && (_MSC_VER < 1914))
 #    define PYBIND11_STD_LAUNDER std::launder
 #    define PYBIND11_HAS_STD_LAUNDER 1
@@ -95,24 +104,6 @@ inline std::string replace_newlines_and_squash(const char *text) {
     return result.substr(str_begin, str_range);
 }
 
-// Apply all the extensions translators from a list
-// Return true if one of the translators completed without raising an exception
-// itself. Return of false indicates that if there are other translators
-// available, they should be tried.
-inline bool apply_exception_translators(std::forward_list<ExceptionTranslator> &translators) {
-    auto last_exception = std::current_exception();
-
-    for (auto &translator : translators) {
-        try {
-            translator(last_exception);
-            return true;
-        } catch (...) {
-            last_exception = std::current_exception();
-        }
-    }
-    return false;
-}
-
 #if defined(_MSC_VER)
 #    define PYBIND11_COMPAT_STRDUP _strdup
 #else
@@ -319,9 +310,20 @@ protected:
             constexpr bool has_kw_only_args = any_of<std::is_same<kw_only, Extra>...>::value,
                            has_pos_only_args = any_of<std::is_same<pos_only, Extra>...>::value,
                            has_arg_annotations = any_of<is_keyword<Extra>...>::value;
+            constexpr bool has_is_method = any_of<std::is_same<is_method, Extra>...>::value;
+            // The implicit `self` argument is not present and not counted in method definitions.
+            constexpr bool has_args = cast_in::args_pos >= 0;
+            constexpr bool is_method_with_self_arg_only = has_is_method && !has_args;
             static_assert(has_arg_annotations || !has_kw_only_args,
                           "py::kw_only requires the use of argument annotations");
-            static_assert(has_arg_annotations || !has_pos_only_args,
+            static_assert(((/* Need `py::arg("arg_name")` annotation in function/method. */
+                            has_arg_annotations)
+                           || (/* Allow methods with no arguments `def method(self, /): ...`.
+                                * A method has at least one argument `self`. There can be no
+                                * `py::arg` annotation. E.g. `class.def("method", py::pos_only())`.
+                                */
+                               is_method_with_self_arg_only))
+                              || !has_pos_only_args,
                           "py::pos_only requires the use of argument annotations (for docstrings "
                           "and aligning the annotations to the argument)");
 
@@ -441,6 +443,13 @@ protected:
         std::string signature;
         size_t type_index = 0, arg_index = 0;
         bool is_starred = false;
+        // `is_return_value.top()` is true if we are currently inside the return type of the
+        // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops
+        // back to the previous state.
+        std::stack<bool> is_return_value({false});
+        // The following characters have special meaning in the signature parsing. Literals
+        // containing these are escaped with `!`.
+        std::string special_chars("!@%{}-");
         for (const auto *pc = text; *pc != '\0'; ++pc) {
             const auto c = *pc;
 
@@ -494,7 +503,57 @@ protected:
                 } else {
                     signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name()));
                 }
+            } else if (c == '!' && special_chars.find(*(pc + 1)) != std::string::npos) {
+                // typing::Literal escapes special characters with !
+                signature += *++pc;
+            } else if (c == '@') {
+                // `@^ ... @!` and `@$ ... @!` are used to force arg/return value type (see
+                // typing::Callable/detail::arg_descr/detail::return_descr)
+                if (*(pc + 1) == '^') {
+                    is_return_value.emplace(false);
+                    ++pc;
+                    continue;
+                }
+                if (*(pc + 1) == '$') {
+                    is_return_value.emplace(true);
+                    ++pc;
+                    continue;
+                }
+                if (*(pc + 1) == '!') {
+                    is_return_value.pop();
+                    ++pc;
+                    continue;
+                }
+                // Handle types that differ depending on whether they appear
+                // in an argument or a return value position (see io_name<text1, text2>).
+                // For named arguments (py::arg()) with noconvert set, return value type is used.
+                ++pc;
+                if (!is_return_value.top()
+                    && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) {
+                    while (*pc != '\0' && *pc != '@') {
+                        signature += *pc++;
+                    }
+                    if (*pc == '@') {
+                        ++pc;
+                    }
+                    while (*pc != '\0' && *pc != '@') {
+                        ++pc;
+                    }
+                } else {
+                    while (*pc != '\0' && *pc != '@') {
+                        ++pc;
+                    }
+                    if (*pc == '@') {
+                        ++pc;
+                    }
+                    while (*pc != '\0' && *pc != '@') {
+                        signature += *pc++;
+                    }
+                }
             } else {
+                if (c == '-' && *(pc + 1) == '>') {
+                    is_return_value.emplace(true);
+                }
                 signature += c;
             }
         }
@@ -591,8 +650,7 @@ protected:
                 // chain.
                 chain_start = rec;
                 rec->next = chain;
-                auto rec_capsule
-                    = reinterpret_borrow<capsule>(((PyCFunctionObject *) m_ptr)->m_self);
+                auto rec_capsule = reinterpret_borrow<capsule>(PyCFunction_GET_SELF(m_ptr));
                 rec_capsule.set_pointer(unique_rec.release());
                 guarded_strdup.release();
             } else {
@@ -610,7 +668,8 @@ protected:
         int index = 0;
         /* Create a nice pydoc rec including all signatures and
            docstrings of the functions in the overload chain */
-        if (chain && options::show_function_signatures()) {
+        if (chain && options::show_function_signatures()
+            && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) {
             // First a generic signature
             signatures += rec->name;
             signatures += "(*args, **kwargs)\n";
@@ -619,7 +678,8 @@ protected:
         // Then specific overload signatures
         bool first_user_def = true;
         for (auto *it = chain_start; it != nullptr; it = it->next) {
-            if (options::show_function_signatures()) {
+            if (options::show_function_signatures()
+                && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) {
                 if (index > 0) {
                     signatures += '\n';
                 }
@@ -650,12 +710,11 @@ protected:
             }
         }
 
-        /* Install docstring */
         auto *func = (PyCFunctionObject *) m_ptr;
-        std::free(const_cast<char *>(func->m_ml->ml_doc));
         // Install docstring if it's non-empty (when at least one option is enabled)
-        func->m_ml->ml_doc
-            = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str());
+        auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str());
+        std::free(const_cast<char *>(PYBIND11_PYCFUNCTION_GET_DOC(func)));
+        PYBIND11_PYCFUNCTION_SET_DOC(func, doc);
 
         if (rec->is_method) {
             m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr());
@@ -1038,40 +1097,7 @@ protected:
             throw;
 #endif
         } catch (...) {
-            /* When an exception is caught, give each registered exception
-               translator a chance to translate it to a Python exception. First
-               all module-local translators will be tried in reverse order of
-               registration. If none of the module-locale translators handle
-               the exception (or there are no module-locale translators) then
-               the global translators will be tried, also in reverse order of
-               registration.
-
-               A translator may choose to do one of the following:
-
-                - catch the exception and call py::set_error()
-                  to set a standard (or custom) Python exception, or
-                - do nothing and let the exception fall through to the next translator, or
-                - delegate translation to the next translator by throwing a new type of exception.
-             */
-
-            bool handled = with_internals([&](internals &internals) {
-                auto &local_exception_translators
-                    = get_local_internals().registered_exception_translators;
-                if (detail::apply_exception_translators(local_exception_translators)) {
-                    return true;
-                }
-                auto &exception_translators = internals.registered_exception_translators;
-                if (detail::apply_exception_translators(exception_translators)) {
-                    return true;
-                }
-                return false;
-            });
-
-            if (handled) {
-                return nullptr;
-            }
-
-            set_error(PyExc_SystemError, "Exception escaped from default exception translator!");
+            try_translate_exceptions();
             return nullptr;
         }
 
@@ -1356,7 +1382,7 @@ PYBIND11_NAMESPACE_BEGIN(detail)
 
 template <>
 struct handle_type_name<module_> {
-    static constexpr auto name = const_name("module");
+    static constexpr auto name = const_name("types.ModuleType");
 };
 
 PYBIND11_NAMESPACE_END(detail)
@@ -1420,8 +1446,8 @@ protected:
         tinfo->dealloc = rec.dealloc;
         tinfo->simple_type = true;
         tinfo->simple_ancestors = true;
-        tinfo->default_holder = rec.default_holder;
         tinfo->module_local = rec.module_local;
+        tinfo->holder_enum_v = rec.holder_enum_v;
 
         with_internals([&](internals &internals) {
             auto tindex = std::type_index(*rec.type);
@@ -1431,7 +1457,17 @@ protected:
             } else {
                 internals.registered_types_cpp[tindex] = tinfo;
             }
+
+            PYBIND11_WARNING_PUSH
+#if defined(__GNUC__) && __GNUC__ == 12
+            // When using GCC 12 these warnings are disabled as they trigger
+            // false positive warnings.  Discussed here:
+            // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115824.
+            PYBIND11_WARNING_DISABLE_GCC("-Warray-bounds")
+            PYBIND11_WARNING_DISABLE_GCC("-Wstringop-overread")
+#endif
             internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo};
+            PYBIND11_WARNING_POP
         });
 
         if (rec.bases.size() > 1 || rec.multiple_inheritance) {
@@ -1584,6 +1620,239 @@ auto method_adaptor(Return (Class::*pmf)(Args...) const) -> Return (Derived::*)(
     return pmf;
 }
 
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+// Helper for the property_cpp_function static member functions below.
+// The only purpose of these functions is to support .def_readonly & .def_readwrite.
+// In this context, the PM template parameter is certain to be a Pointer to a Member.
+// The main purpose of must_be_member_function_pointer is to make this obvious, and to guard
+// against accidents. As a side-effect, it also explains why the syntactical overhead for
+// perfect forwarding is not needed.
+template <typename PM>
+using must_be_member_function_pointer = enable_if_t<std::is_member_pointer<PM>::value, int>;
+
+// Note that property_cpp_function is intentionally in the main pybind11 namespace,
+// because user-defined specializations could be useful.
+
+// Classic (non-smart_holder) implementations for .def_readonly and .def_readwrite
+// getter and setter functions.
+// WARNING: This classic implementation can lead to dangling pointers for raw pointer members.
+// See test_ptr() in tests/test_class_sh_property.py
+// However, this implementation works as-is (and safely) for smart_holder std::shared_ptr members.
+template <typename T, typename D>
+struct property_cpp_function_classic {
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function readonly(PM pm, const handle &hdl) {
+        return cpp_function([pm](const T &c) -> const D & { return c.*pm; }, is_method(hdl));
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function read(PM pm, const handle &hdl) {
+        return readonly(pm, hdl);
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function write(PM pm, const handle &hdl) {
+        return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl));
+    }
+};
+
+PYBIND11_NAMESPACE_END(detail)
+
+template <typename T, typename D, typename SFINAE = void>
+struct property_cpp_function : detail::property_cpp_function_classic<T, D> {};
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <typename T, typename D, typename SFINAE = void>
+struct both_t_and_d_use_type_caster_base : std::false_type {};
+
+// `T` is assumed to be equivalent to `intrinsic_t<T>`.
+// `D` is may or may not be equivalent to `intrinsic_t<D>`.
+template <typename T, typename D>
+struct both_t_and_d_use_type_caster_base<
+    T,
+    D,
+    enable_if_t<all_of<std::is_base_of<type_caster_base<T>, type_caster<T>>,
+                       std::is_base_of<type_caster_base<intrinsic_t<D>>, make_caster<D>>>::value>>
+    : std::true_type {};
+
+// Specialization for raw pointer members, using smart_holder if that is the class_ holder,
+// or falling back to the classic implementation if not.
+// WARNING: Like the classic implementation, this implementation can lead to dangling pointers.
+// See test_ptr() in tests/test_class_sh_property.py
+// However, the read functions return a shared_ptr to the member, emulating the PyCLIF approach:
+// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233
+// This prevents disowning of the Python object owning the raw pointer member.
+template <typename T, typename D>
+struct property_cpp_function_sh_raw_ptr_member {
+    using drp = typename std::remove_pointer<D>::type;
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function readonly(PM pm, const handle &hdl) {
+        type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true);
+        if (tinfo->holder_enum_v == holder_enum_t::smart_holder) {
+            return cpp_function(
+                [pm](handle c_hdl) -> std::shared_ptr<drp> {
+                    std::shared_ptr<T> c_sp
+                        = type_caster<std::shared_ptr<T>>::shared_ptr_with_responsible_parent(
+                            c_hdl);
+                    D ptr = (*c_sp).*pm;
+                    return std::shared_ptr<drp>(c_sp, ptr);
+                },
+                is_method(hdl));
+        }
+        return property_cpp_function_classic<T, D>::readonly(pm, hdl);
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function read(PM pm, const handle &hdl) {
+        return readonly(pm, hdl);
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function write(PM pm, const handle &hdl) {
+        type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true);
+        if (tinfo->holder_enum_v == holder_enum_t::smart_holder) {
+            return cpp_function([pm](T &c, D value) { c.*pm = std::forward<D>(std::move(value)); },
+                                is_method(hdl));
+        }
+        return property_cpp_function_classic<T, D>::write(pm, hdl);
+    }
+};
+
+// Specialization for members held by-value, using smart_holder if that is the class_ holder,
+// or falling back to the classic implementation if not.
+// The read functions return a shared_ptr to the member, emulating the PyCLIF approach:
+// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233
+// This prevents disowning of the Python object owning the member.
+template <typename T, typename D>
+struct property_cpp_function_sh_member_held_by_value {
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function readonly(PM pm, const handle &hdl) {
+        type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true);
+        if (tinfo->holder_enum_v == holder_enum_t::smart_holder) {
+            return cpp_function(
+                [pm](handle c_hdl) -> std::shared_ptr<typename std::add_const<D>::type> {
+                    std::shared_ptr<T> c_sp
+                        = type_caster<std::shared_ptr<T>>::shared_ptr_with_responsible_parent(
+                            c_hdl);
+                    return std::shared_ptr<typename std::add_const<D>::type>(c_sp,
+                                                                             &(c_sp.get()->*pm));
+                },
+                is_method(hdl));
+        }
+        return property_cpp_function_classic<T, D>::readonly(pm, hdl);
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function read(PM pm, const handle &hdl) {
+        type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true);
+        if (tinfo->holder_enum_v == holder_enum_t::smart_holder) {
+            return cpp_function(
+                [pm](handle c_hdl) -> std::shared_ptr<D> {
+                    std::shared_ptr<T> c_sp
+                        = type_caster<std::shared_ptr<T>>::shared_ptr_with_responsible_parent(
+                            c_hdl);
+                    return std::shared_ptr<D>(c_sp, &(c_sp.get()->*pm));
+                },
+                is_method(hdl));
+        }
+        return property_cpp_function_classic<T, D>::read(pm, hdl);
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function write(PM pm, const handle &hdl) {
+        type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true);
+        if (tinfo->holder_enum_v == holder_enum_t::smart_holder) {
+            return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl));
+        }
+        return property_cpp_function_classic<T, D>::write(pm, hdl);
+    }
+};
+
+// Specialization for std::unique_ptr members, using smart_holder if that is the class_ holder,
+// or falling back to the classic implementation if not.
+// read disowns the member unique_ptr.
+// write disowns the passed Python object.
+// readonly is disabled (static_assert) because there is no safe & intuitive way to make the member
+// accessible as a Python object without disowning the member unique_ptr. A .def_readonly disowning
+// the unique_ptr member is deemed highly prone to misunderstandings.
+template <typename T, typename D>
+struct property_cpp_function_sh_unique_ptr_member {
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function readonly(PM, const handle &) {
+        static_assert(!is_instantiation<std::unique_ptr, D>::value,
+                      "def_readonly cannot be used for std::unique_ptr members.");
+        return cpp_function{}; // Unreachable.
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function read(PM pm, const handle &hdl) {
+        type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true);
+        if (tinfo->holder_enum_v == holder_enum_t::smart_holder) {
+            return cpp_function(
+                [pm](handle c_hdl) -> D {
+                    std::shared_ptr<T> c_sp
+                        = type_caster<std::shared_ptr<T>>::shared_ptr_with_responsible_parent(
+                            c_hdl);
+                    return D{std::move(c_sp.get()->*pm)};
+                },
+                is_method(hdl));
+        }
+        return property_cpp_function_classic<T, D>::read(pm, hdl);
+    }
+
+    template <typename PM, must_be_member_function_pointer<PM> = 0>
+    static cpp_function write(PM pm, const handle &hdl) {
+        return cpp_function([pm](T &c, D &&value) { c.*pm = std::move(value); }, is_method(hdl));
+    }
+};
+
+PYBIND11_NAMESPACE_END(detail)
+
+template <typename T, typename D>
+struct property_cpp_function<
+    T,
+    D,
+    detail::enable_if_t<detail::all_of<std::is_pointer<D>,
+                                       detail::both_t_and_d_use_type_caster_base<T, D>>::value>>
+    : detail::property_cpp_function_sh_raw_ptr_member<T, D> {};
+
+template <typename T, typename D>
+struct property_cpp_function<T,
+                             D,
+                             detail::enable_if_t<detail::all_of<
+                                 detail::none_of<std::is_pointer<D>,
+                                                 std::is_array<D>,
+                                                 detail::is_instantiation<std::unique_ptr, D>,
+                                                 detail::is_instantiation<std::shared_ptr, D>>,
+                                 detail::both_t_and_d_use_type_caster_base<T, D>>::value>>
+    : detail::property_cpp_function_sh_member_held_by_value<T, D> {};
+
+template <typename T, typename D>
+struct property_cpp_function<
+    T,
+    D,
+    detail::enable_if_t<detail::all_of<
+        detail::is_instantiation<std::unique_ptr, D>,
+        detail::both_t_and_d_use_type_caster_base<T, typename D::element_type>>::value>>
+    : detail::property_cpp_function_sh_unique_ptr_member<T, D> {};
+
+#ifdef PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE
+// NOTE: THIS IS MEANT FOR STRESS-TESTING OR TRIAGING ONLY!
+//       Running the pybind11 unit tests with smart_holder as the default holder is to ensure
+//       that `py::smart_holder` / `py::classh` is backward-compatible with all pre-existing
+//       functionality.
+//       Be careful not to link translation units compiled with different default holders, because
+//       this will cause ODR violations (https://en.wikipedia.org/wiki/One_Definition_Rule).
+template <typename>
+using default_holder_type = smart_holder;
+#else
+template <typename T>
+using default_holder_type = std::unique_ptr<T>;
+#endif
+
 template <typename type_, typename... options>
 class class_ : public detail::generic_type {
     template <typename T>
@@ -1600,7 +1869,7 @@ public:
     using type = type_;
     using type_alias = detail::exactly_one_t<is_subtype, void, options...>;
     constexpr static bool has_alias = !std::is_void<type_alias>::value;
-    using holder_type = detail::exactly_one_t<is_holder, std::unique_ptr<type>, options...>;
+    using holder_type = detail::exactly_one_t<is_holder, default_holder_type<type>, options...>;
 
     static_assert(detail::all_of<is_valid_class_option<options>...>::value,
                   "Unknown/invalid class_ template parameters provided");
@@ -1631,8 +1900,16 @@ public:
         record.type_align = alignof(conditional_t<has_alias, type_alias, type> &);
         record.holder_size = sizeof(holder_type);
         record.init_instance = init_instance;
-        record.dealloc = dealloc;
-        record.default_holder = detail::is_instantiation<std::unique_ptr, holder_type>::value;
+
+        if (detail::is_instantiation<std::unique_ptr, holder_type>::value) {
+            record.holder_enum_v = detail::holder_enum_t::std_unique_ptr;
+        } else if (detail::is_instantiation<std::shared_ptr, holder_type>::value) {
+            record.holder_enum_v = detail::holder_enum_t::std_shared_ptr;
+        } else if (std::is_same<holder_type, smart_holder>::value) {
+            record.holder_enum_v = detail::holder_enum_t::smart_holder;
+        } else {
+            record.holder_enum_v = detail::holder_enum_t::custom_holder;
+        }
 
         set_operator_new<type>(&record);
 
@@ -1642,6 +1919,12 @@ public:
         /* Process optional arguments, if any */
         process_attributes<Extra...>::init(extra..., &record);
 
+        if (record.release_gil_before_calling_cpp_dtor) {
+            record.dealloc = dealloc_release_gil_before_calling_cpp_dtor;
+        } else {
+            record.dealloc = dealloc_without_manipulating_gil;
+        }
+
         generic_type::initialize(record);
 
         if (has_alias) {
@@ -1652,6 +1935,7 @@ public:
                     = instances[std::type_index(typeid(type))];
             });
         }
+        def("_pybind11_conduit_v1_", cpp_conduit_method);
     }
 
     template <typename Base, detail::enable_if_t<is_base<Base>::value, int> = 0>
@@ -1764,9 +2048,11 @@ public:
     class_ &def_readwrite(const char *name, D C::*pm, const Extra &...extra) {
         static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value,
                       "def_readwrite() requires a class member (or base class member)");
-        cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)),
-            fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this));
-        def_property(name, fget, fset, return_value_policy::reference_internal, extra...);
+        def_property(name,
+                     property_cpp_function<type, D>::read(pm, *this),
+                     property_cpp_function<type, D>::write(pm, *this),
+                     return_value_policy::reference_internal,
+                     extra...);
         return *this;
     }
 
@@ -1774,8 +2060,10 @@ public:
     class_ &def_readonly(const char *name, const D C::*pm, const Extra &...extra) {
         static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value,
                       "def_readonly() requires a class member (or base class member)");
-        cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this));
-        def_property_readonly(name, fget, return_value_policy::reference_internal, extra...);
+        def_property_readonly(name,
+                              property_cpp_function<type, D>::readonly(pm, *this),
+                              return_value_policy::reference_internal,
+                              extra...);
         return *this;
     }
 
@@ -1952,6 +2240,8 @@ private:
     /// instance.  Should be called as soon as the `type` value_ptr is set for an instance.  Takes
     /// an optional pointer to an existing holder to use; if not specified and the instance is
     /// `.owned`, a new holder will be constructed to manage the value pointer.
+    template <typename H = holder_type,
+              detail::enable_if_t<!detail::is_smart_holder<H>::value, int> = 0>
     static void init_instance(detail::instance *inst, const void *holder_ptr) {
         auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type)));
         if (!v_h.instance_registered()) {
@@ -1961,15 +2251,73 @@ private:
         init_holder(inst, v_h, (const holder_type *) holder_ptr, v_h.value_ptr<type>());
     }
 
-    /// Deallocates an instance; via holder, if constructed; otherwise via operator delete.
-    static void dealloc(detail::value_and_holder &v_h) {
-        // We could be deallocating because we are cleaning up after a Python exception.
-        // If so, the Python error indicator will be set. We need to clear that before
-        // running the destructor, in case the destructor code calls more Python.
-        // If we don't, the Python API will exit with an exception, and pybind11 will
-        // throw error_already_set from the C++ destructor which is forbidden and triggers
-        // std::terminate().
-        error_scope scope;
+    template <typename WrappedType>
+    static bool try_initialization_using_shared_from_this(holder_type *, WrappedType *, ...) {
+        return false;
+    }
+
+    // Adopting existing approach used by type_caster_base, although it leads to somewhat fuzzy
+    // ownership semantics: if we detected via shared_from_this that a shared_ptr exists already,
+    // it is reused, irrespective of the return_value_policy in effect.
+    // "SomeBaseOfWrappedType" is needed because std::enable_shared_from_this is not necessarily a
+    // direct base of WrappedType.
+    template <typename WrappedType, typename SomeBaseOfWrappedType>
+    static bool try_initialization_using_shared_from_this(
+        holder_type *uninitialized_location,
+        WrappedType *value_ptr_w_t,
+        const std::enable_shared_from_this<SomeBaseOfWrappedType> *) {
+        auto shd_ptr = std::dynamic_pointer_cast<WrappedType>(
+            detail::try_get_shared_from_this(value_ptr_w_t));
+        if (!shd_ptr) {
+            return false;
+        }
+        // Note: inst->owned ignored.
+        new (uninitialized_location) holder_type(holder_type::from_shared_ptr(shd_ptr));
+        return true;
+    }
+
+    template <typename H = holder_type,
+              detail::enable_if_t<detail::is_smart_holder<H>::value, int> = 0>
+    static void init_instance(detail::instance *inst, const void *holder_const_void_ptr) {
+        // Need for const_cast is a consequence of the type_info::init_instance type:
+        // void (*init_instance)(instance *, const void *);
+        auto *holder_void_ptr = const_cast<void *>(holder_const_void_ptr);
+
+        auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type)));
+        if (!v_h.instance_registered()) {
+            register_instance(inst, v_h.value_ptr(), v_h.type);
+            v_h.set_instance_registered();
+        }
+        auto *uninitialized_location = std::addressof(v_h.holder<holder_type>());
+        auto *value_ptr_w_t = v_h.value_ptr<type>();
+        // Try downcast from `type` to `type_alias`:
+        inst->is_alias
+            = detail::dynamic_raw_ptr_cast_if_possible<type_alias>(value_ptr_w_t) != nullptr;
+        if (holder_void_ptr) {
+            // Note: inst->owned ignored.
+            auto *holder_ptr = static_cast<holder_type *>(holder_void_ptr);
+            new (uninitialized_location) holder_type(std::move(*holder_ptr));
+        } else if (!try_initialization_using_shared_from_this(
+                       uninitialized_location, value_ptr_w_t, value_ptr_w_t)) {
+            if (inst->owned) {
+                new (uninitialized_location) holder_type(holder_type::from_raw_ptr_take_ownership(
+                    value_ptr_w_t, /*void_cast_raw_ptr*/ inst->is_alias));
+            } else {
+                new (uninitialized_location)
+                    holder_type(holder_type::from_raw_ptr_unowned(value_ptr_w_t));
+            }
+        }
+        v_h.set_holder_constructed();
+    }
+
+    // Deallocates an instance; via holder, if constructed; otherwise via operator delete.
+    // NOTE: The Python error indicator needs to cleared BEFORE this function is called.
+    // This is because we could be deallocating while cleaning up after a Python exception.
+    // If the error indicator is not cleared but the C++ destructor code makes Python C API
+    // calls, those calls are likely to generate a new exception, and pybind11 will then
+    // throw `error_already_set` from the C++ destructor. This is forbidden and will
+    // trigger std::terminate().
+    static void dealloc_impl(detail::value_and_holder &v_h) {
         if (v_h.holder_constructed()) {
             v_h.holder<holder_type>().~holder_type();
             v_h.set_holder_constructed(false);
@@ -1980,6 +2328,32 @@ private:
         v_h.value_ptr() = nullptr;
     }
 
+    static void dealloc_without_manipulating_gil(detail::value_and_holder &v_h) {
+        error_scope scope;
+        dealloc_impl(v_h);
+    }
+
+    static void dealloc_release_gil_before_calling_cpp_dtor(detail::value_and_holder &v_h) {
+        error_scope scope;
+        // Intentionally not using `gil_scoped_release` because the non-simple
+        // version unconditionally calls `get_internals()`.
+        // `Py_BEGIN_ALLOW_THREADS`, `Py_END_ALLOW_THREADS` cannot be used
+        // because those macros include `{` and `}`.
+        PyThreadState *py_ts = PyEval_SaveThread();
+        try {
+            dealloc_impl(v_h);
+        } catch (...) {
+            // This code path is expected to be unreachable unless there is a
+            // bug in pybind11 itself.
+            // An alternative would be to mark this function, or
+            // `dealloc_impl()`, with `nothrow`, but that would be a subtle
+            // behavior change and could make debugging more difficult.
+            PyEval_RestoreThread(py_ts);
+            throw;
+        }
+        PyEval_RestoreThread(py_ts);
+    }
+
     static detail::function_record *get_function_record(handle h) {
         h = detail::get_function(h);
         if (!h) {
@@ -2001,6 +2375,11 @@ private:
     }
 };
 
+// Supports easier switching between py::class_<T> and py::class_<T, py::smart_holder>:
+// users can simply replace the `_` in `class_` with `h` or vice versa.
+template <typename type_, typename... options>
+using classh = class_<type_, smart_holder, options...>;
+
 /// Binds an existing constructor taking arguments Args...
 template <typename... Args>
 detail::initimpl::constructor<Args...> init() {
@@ -2062,9 +2441,11 @@ struct enum_base {
                     .format(std::move(type_name), enum_name(arg), int_(arg));
             },
             name("__repr__"),
-            is_method(m_base));
+            is_method(m_base),
+            pos_only());
 
-        m_base.attr("name") = property(cpp_function(&enum_name, name("name"), is_method(m_base)));
+        m_base.attr("name")
+            = property(cpp_function(&enum_name, name("name"), is_method(m_base), pos_only()));
 
         m_base.attr("__str__") = cpp_function(
             [](handle arg) -> str {
@@ -2072,7 +2453,8 @@ struct enum_base {
                 return pybind11::str("{}.{}").format(std::move(type_name), enum_name(arg));
             },
             name("__str__"),
-            is_method(m_base));
+            is_method(m_base),
+            pos_only());
 
         if (options::show_enum_members_docstring()) {
             m_base.attr("__doc__") = static_property(
@@ -2127,7 +2509,8 @@ struct enum_base {
         },                                                                                        \
         name(op),                                                                                 \
         is_method(m_base),                                                                        \
-        arg("other"))
+        arg("other"),                                                                             \
+        pos_only())
 
 #define PYBIND11_ENUM_OP_CONV(op, expr)                                                           \
     m_base.attr(op) = cpp_function(                                                               \
@@ -2137,7 +2520,8 @@ struct enum_base {
         },                                                                                        \
         name(op),                                                                                 \
         is_method(m_base),                                                                        \
-        arg("other"))
+        arg("other"),                                                                             \
+        pos_only())
 
 #define PYBIND11_ENUM_OP_CONV_LHS(op, expr)                                                       \
     m_base.attr(op) = cpp_function(                                                               \
@@ -2147,7 +2531,8 @@ struct enum_base {
         },                                                                                        \
         name(op),                                                                                 \
         is_method(m_base),                                                                        \
-        arg("other"))
+        arg("other"),                                                                             \
+        pos_only())
 
         if (is_convertible) {
             PYBIND11_ENUM_OP_CONV_LHS("__eq__", !b.is_none() && a.equal(b));
@@ -2167,7 +2552,8 @@ struct enum_base {
                 m_base.attr("__invert__")
                     = cpp_function([](const object &arg) { return ~(int_(arg)); },
                                    name("__invert__"),
-                                   is_method(m_base));
+                                   is_method(m_base),
+                                   pos_only());
             }
         } else {
             PYBIND11_ENUM_OP_STRICT("__eq__", int_(a).equal(int_(b)), return false);
@@ -2187,11 +2573,15 @@ struct enum_base {
 #undef PYBIND11_ENUM_OP_CONV
 #undef PYBIND11_ENUM_OP_STRICT
 
-        m_base.attr("__getstate__") = cpp_function(
-            [](const object &arg) { return int_(arg); }, name("__getstate__"), is_method(m_base));
+        m_base.attr("__getstate__") = cpp_function([](const object &arg) { return int_(arg); },
+                                                   name("__getstate__"),
+                                                   is_method(m_base),
+                                                   pos_only());
 
-        m_base.attr("__hash__") = cpp_function(
-            [](const object &arg) { return int_(arg); }, name("__hash__"), is_method(m_base));
+        m_base.attr("__hash__") = cpp_function([](const object &arg) { return int_(arg); },
+                                               name("__hash__"),
+                                               is_method(m_base),
+                                               pos_only());
     }
 
     PYBIND11_NOINLINE void value(char const *name_, object value, const char *doc = nullptr) {
@@ -2283,9 +2673,9 @@ public:
         m_base.init(is_arithmetic, is_convertible);
 
         def(init([](Scalar i) { return static_cast<Type>(i); }), arg("value"));
-        def_property_readonly("value", [](Type value) { return (Scalar) value; });
-        def("__int__", [](Type value) { return (Scalar) value; });
-        def("__index__", [](Type value) { return (Scalar) value; });
+        def_property_readonly("value", [](Type value) { return (Scalar) value; }, pos_only());
+        def("__int__", [](Type value) { return (Scalar) value; }, pos_only());
+        def("__index__", [](Type value) { return (Scalar) value; }, pos_only());
         attr("__setstate__") = cpp_function(
             [](detail::value_and_holder &v_h, Scalar arg) {
                 detail::initimpl::setstate<Base>(
@@ -2294,7 +2684,8 @@ public:
             detail::is_new_style_constructor(),
             pybind11::name("__setstate__"),
             is_method(*this),
-            arg("state"));
+            arg("state"),
+            pos_only());
     }
 
     /// Export enumeration entries into the parent scope
@@ -2366,13 +2757,20 @@ keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) {
 inline std::pair<decltype(internals::registered_types_py)::iterator, bool>
 all_type_info_get_cache(PyTypeObject *type) {
     auto res = with_internals([type](internals &internals) {
-        return internals
-            .registered_types_py
+        auto ins = internals
+                       .registered_types_py
 #ifdef __cpp_lib_unordered_map_try_emplace
-            .try_emplace(type);
+                       .try_emplace(type);
 #else
-            .emplace(type, std::vector<detail::type_info *>());
+                       .emplace(type, std::vector<detail::type_info *>());
 #endif
+        if (ins.second) {
+            // For free-threading mode, this call must be under
+            // the with_internals() mutex lock, to avoid that other threads
+            // continue running with the empty ins.first->second.
+            all_type_info_populate(type, ins.first->second);
+        }
+        return ins;
     });
     if (res.second) {
         // New cache entry created; set up a weak reference to automatically remove it if the type
@@ -2473,7 +2871,8 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) {
 
     if (!detail::get_type_info(typeid(state), false)) {
         class_<state>(handle(), "iterator", pybind11::module_local())
-            .def("__iter__", [](state &s) -> state & { return s; })
+            .def(
+                "__iter__", [](state &s) -> state & { return s; }, pos_only())
             .def(
                 "__next__",
                 [](state &s) -> ValueType {
@@ -2490,6 +2889,7 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) {
                     // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
                 },
                 std::forward<Extra>(extra)...,
+                pos_only(),
                 Policy);
     }
 
@@ -2624,10 +3024,12 @@ void implicitly_convertible() {
 }
 
 inline void register_exception_translator(ExceptionTranslator &&translator) {
-    detail::with_internals([&](detail::internals &internals) {
-        internals.registered_exception_translators.push_front(
-            std::forward<ExceptionTranslator>(translator));
-    });
+    detail::with_exception_translators(
+        [&](std::forward_list<ExceptionTranslator> &exception_translators,
+            std::forward_list<ExceptionTranslator> &local_exception_translators) {
+            (void) local_exception_translators;
+            exception_translators.push_front(std::forward<ExceptionTranslator>(translator));
+        });
 }
 
 /**
@@ -2637,11 +3039,12 @@ inline void register_exception_translator(ExceptionTranslator &&translator) {
  * the exception.
  */
 inline void register_local_exception_translator(ExceptionTranslator &&translator) {
-    detail::with_internals([&](detail::internals &internals) {
-        (void) internals;
-        detail::get_local_internals().registered_exception_translators.push_front(
-            std::forward<ExceptionTranslator>(translator));
-    });
+    detail::with_exception_translators(
+        [&](std::forward_list<ExceptionTranslator> &exception_translators,
+            std::forward_list<ExceptionTranslator> &local_exception_translators) {
+            (void) exception_translators;
+            local_exception_translators.push_front(std::forward<ExceptionTranslator>(translator));
+        });
 }
 
 /**
@@ -2815,8 +3218,8 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
     }
 
     /* Don't call dispatch code if invoked from overridden function.
-       Unfortunately this doesn't work on PyPy. */
-#if !defined(PYPY_VERSION)
+       Unfortunately this doesn't work on PyPy and GraalPy. */
+#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON)
 #    if PY_VERSION_HEX >= 0x03090000
     PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
     if (frame != nullptr) {

pybind11/include/pybind11/pytypes.h

@@ -113,6 +113,17 @@ public:
     /// See above (the only difference is that the key is provided as a string literal)
     str_attr_accessor attr(const char *key) const;
 
+    /** \rst
+         Similar to the above attr functions with the difference that the templated Type
+         is used to set the `__annotations__` dict value to the corresponding key. Worth noting
+         that attr_with_type_hint is implemented in cast.h.
+    \endrst */
+    template <typename T>
+    obj_attr_accessor attr_with_type_hint(handle key) const;
+    /// See above (the only difference is that the key is provided as a string literal)
+    template <typename T>
+    str_attr_accessor attr_with_type_hint(const char *key) const;
+
     /** \rst
         Matches * unpacking in Python, e.g. to unpack arguments out of a ``tuple``
         or ``list`` for a function call. Applying another * to the result yields
@@ -182,6 +193,9 @@ public:
     /// Get or set the object's docstring, i.e. ``obj.__doc__``.
     str_attr_accessor doc() const;
 
+    /// Get or set the object's annotations, i.e. ``obj.__annotations__``.
+    object annotations() const;
+
     /// Return the object's current reference count
     ssize_t ref_count() const {
 #ifdef PYPY_VERSION
@@ -643,7 +657,7 @@ struct error_fetch_and_normalize {
 
         bool have_trace = false;
         if (m_trace) {
-#if !defined(PYPY_VERSION)
+#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON)
             auto *tb = reinterpret_cast<PyTracebackObject *>(m_trace.ptr());
 
             // Get the deepest trace possible.
@@ -1259,6 +1273,7 @@ protected:
     using pointer = arrow_proxy<const handle>;
 
     sequence_fast_readonly(handle obj, ssize_t n) : ptr(PySequence_Fast_ITEMS(obj.ptr()) + n) {}
+    sequence_fast_readonly() = default;
 
     // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
     reference dereference() const { return *ptr; }
@@ -1281,6 +1296,7 @@ protected:
     using pointer = arrow_proxy<const sequence_accessor>;
 
     sequence_slow_readwrite(handle obj, ssize_t index) : obj(obj), index(index) {}
+    sequence_slow_readwrite() = default;
 
     reference dereference() const { return {obj, static_cast<size_t>(index)}; }
     void increment() { ++index; }
@@ -1354,7 +1370,7 @@ inline bool PyUnicode_Check_Permissive(PyObject *o) {
 #    define PYBIND11_STR_CHECK_FUN PyUnicode_Check
 #endif
 
-inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; }
+inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == &PyStaticMethod_Type; }
 
 class kwargs_proxy : public handle {
 public:
@@ -1468,11 +1484,17 @@ public:
     PYBIND11_OBJECT_DEFAULT(iterator, object, PyIter_Check)
 
     iterator &operator++() {
+        init();
         advance();
         return *this;
     }
 
     iterator operator++(int) {
+        // Note: We must call init() first so that rv.value is
+        // the same as this->value just before calling advance().
+        // Otherwise, dereferencing the returned iterator may call
+        // advance() again and return the 3rd item instead of the 1st.
+        init();
         auto rv = *this;
         advance();
         return rv;
@@ -1480,15 +1502,12 @@ public:
 
     // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
     reference operator*() const {
-        if (m_ptr && !value.ptr()) {
-            auto &self = const_cast<iterator &>(*this);
-            self.advance();
-        }
+        init();
         return value;
     }
 
     pointer operator->() const {
-        operator*();
+        init();
         return &value;
     }
 
@@ -1511,6 +1530,13 @@ public:
     friend bool operator!=(const iterator &a, const iterator &b) { return a->ptr() != b->ptr(); }
 
 private:
+    void init() const {
+        if (m_ptr && !value.ptr()) {
+            auto &self = const_cast<iterator &>(*this);
+            self.advance();
+        }
+    }
+
     void advance() {
         value = reinterpret_steal<object>(PyIter_Next(m_ptr));
         if (value.ptr() == nullptr && PyErr_Occurred()) {
@@ -2214,6 +2240,18 @@ class kwargs : public dict {
     PYBIND11_OBJECT_DEFAULT(kwargs, dict, PyDict_Check)
 };
 
+// Subclasses of args and kwargs to support type hinting
+// as defined in PEP 484. See #5357 for more info.
+template <typename T>
+class Args : public args {
+    using args::args;
+};
+
+template <typename T>
+class KWArgs : public kwargs {
+    using kwargs::kwargs;
+};
+
 class anyset : public object {
 public:
     PYBIND11_OBJECT(anyset, object, PyAnySet_Check)
@@ -2534,6 +2572,19 @@ str_attr_accessor object_api<D>::doc() const {
     return attr("__doc__");
 }
 
+template <typename D>
+object object_api<D>::annotations() const {
+#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION <= 9
+    // https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older
+    if (!hasattr(derived(), "__annotations__")) {
+        setattr(derived(), "__annotations__", dict());
+    }
+    return attr("__annotations__");
+#else
+    return getattr(derived(), "__annotations__", dict());
+#endif
+}
+
 template <typename D>
 handle object_api<D>::get_type() const {
     return type::handle_of(derived());

pybind11/include/pybind11/stl.h

@@ -11,10 +11,14 @@
 
 #include "pybind11.h"
 #include "detail/common.h"
+#include "detail/descr.h"
+#include "detail/type_caster_base.h"
 
 #include <deque>
+#include <initializer_list>
 #include <list>
 #include <map>
+#include <memory>
 #include <ostream>
 #include <set>
 #include <unordered_map>
@@ -35,6 +39,89 @@
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
 
+//
+// Begin: Equivalent of
+//        https://github.com/google/clif/blob/ae4eee1de07cdf115c0c9bf9fec9ff28efce6f6c/clif/python/runtime.cc#L388-L438
+/*
+The three `PyObjectTypeIsConvertibleTo*()` functions below are
+the result of converging the behaviors of pybind11 and PyCLIF
+(http://github.com/google/clif).
+
+Originally PyCLIF was extremely far on the permissive side of the spectrum,
+while pybind11 was very far on the strict side. Originally PyCLIF accepted any
+Python iterable as input for a C++ `vector`/`set`/`map` argument, as long as
+the elements were convertible. The obvious (in hindsight) problem was that
+any empty Python iterable could be passed to any of these C++ types, e.g. `{}`
+was accepted for C++ `vector`/`set` arguments, or `[]` for C++ `map` arguments.
+
+The functions below strike a practical permissive-vs-strict compromise,
+informed by tens of thousands of use cases in the wild. A main objective is
+to prevent accidents and improve readability:
+
+- Python literals must match the C++ types.
+
+- For C++ `set`: The potentially reducing conversion from a Python sequence
+  (e.g. Python `list` or `tuple`) to a C++ `set` must be explicit, by going
+  through a Python `set`.
+
+- However, a Python `set` can still be passed to a C++ `vector`. The rationale
+  is that this conversion is not reducing. Implicit conversions of this kind
+  are also fairly commonly used, therefore enforcing explicit conversions
+  would have an unfavorable cost : benefit ratio; more sloppily speaking,
+  such an enforcement would be more annoying than helpful.
+*/
+
+inline bool PyObjectIsInstanceWithOneOfTpNames(PyObject *obj,
+                                               std::initializer_list<const char *> tp_names) {
+    if (PyType_Check(obj)) {
+        return false;
+    }
+    const char *obj_tp_name = Py_TYPE(obj)->tp_name;
+    for (const auto *tp_name : tp_names) {
+        if (std::strcmp(obj_tp_name, tp_name) == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+inline bool PyObjectTypeIsConvertibleToStdVector(PyObject *obj) {
+    if (PySequence_Check(obj) != 0) {
+        return !PyUnicode_Check(obj) && !PyBytes_Check(obj);
+    }
+    return (PyGen_Check(obj) != 0) || (PyAnySet_Check(obj) != 0)
+           || PyObjectIsInstanceWithOneOfTpNames(
+               obj, {"dict_keys", "dict_values", "dict_items", "map", "zip"});
+}
+
+inline bool PyObjectTypeIsConvertibleToStdSet(PyObject *obj) {
+    return (PyAnySet_Check(obj) != 0) || PyObjectIsInstanceWithOneOfTpNames(obj, {"dict_keys"});
+}
+
+inline bool PyObjectTypeIsConvertibleToStdMap(PyObject *obj) {
+    if (PyDict_Check(obj)) {
+        return true;
+    }
+    // Implicit requirement in the conditions below:
+    // A type with `.__getitem__()` & `.items()` methods must implement these
+    // to be compatible with https://docs.python.org/3/c-api/mapping.html
+    if (PyMapping_Check(obj) == 0) {
+        return false;
+    }
+    PyObject *items = PyObject_GetAttrString(obj, "items");
+    if (items == nullptr) {
+        PyErr_Clear();
+        return false;
+    }
+    bool is_convertible = (PyCallable_Check(items) != 0);
+    Py_DECREF(items);
+    return is_convertible;
+}
+
+//
+// End: Equivalent of clif/python/runtime.cc
+//
+
 /// Extracts an const lvalue reference or rvalue reference for U based on the type of T (e.g. for
 /// forwarding a container element).  Typically used indirect via forwarded_type(), below.
 template <typename T, typename U>
@@ -66,17 +153,10 @@ private:
     }
     void reserve_maybe(const anyset &, void *) {}
 
-public:
-    bool load(handle src, bool convert) {
-        if (!isinstance<anyset>(src)) {
-            return false;
-        }
-        auto s = reinterpret_borrow<anyset>(src);
-        value.clear();
-        reserve_maybe(s, &value);
-        for (auto entry : s) {
+    bool convert_iterable(const iterable &itbl, bool convert) {
+        for (const auto &it : itbl) {
             key_conv conv;
-            if (!conv.load(entry, convert)) {
+            if (!conv.load(it, convert)) {
                 return false;
             }
             value.insert(cast_op<Key &&>(std::move(conv)));
@@ -84,6 +164,29 @@ public:
         return true;
     }
 
+    bool convert_anyset(anyset s, bool convert) {
+        value.clear();
+        reserve_maybe(s, &value);
+        return convert_iterable(s, convert);
+    }
+
+public:
+    bool load(handle src, bool convert) {
+        if (!PyObjectTypeIsConvertibleToStdSet(src.ptr())) {
+            return false;
+        }
+        if (isinstance<anyset>(src)) {
+            value.clear();
+            return convert_anyset(reinterpret_borrow<anyset>(src), convert);
+        }
+        if (!convert) {
+            return false;
+        }
+        assert(isinstance<iterable>(src));
+        value.clear();
+        return convert_iterable(reinterpret_borrow<iterable>(src), convert);
+    }
+
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         if (!std::is_lvalue_reference<T>::value) {
@@ -115,15 +218,10 @@ private:
     }
     void reserve_maybe(const dict &, void *) {}
 
-public:
-    bool load(handle src, bool convert) {
-        if (!isinstance<dict>(src)) {
-            return false;
-        }
-        auto d = reinterpret_borrow<dict>(src);
+    bool convert_elements(const dict &d, bool convert) {
         value.clear();
         reserve_maybe(d, &value);
-        for (auto it : d) {
+        for (const auto &it : d) {
             key_conv kconv;
             value_conv vconv;
             if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) {
@@ -134,6 +232,25 @@ public:
         return true;
     }
 
+public:
+    bool load(handle src, bool convert) {
+        if (!PyObjectTypeIsConvertibleToStdMap(src.ptr())) {
+            return false;
+        }
+        if (isinstance<dict>(src)) {
+            return convert_elements(reinterpret_borrow<dict>(src), convert);
+        }
+        if (!convert) {
+            return false;
+        }
+        auto items = reinterpret_steal<object>(PyMapping_Items(src.ptr()));
+        if (!items) {
+            throw error_already_set();
+        }
+        assert(isinstance<iterable>(items));
+        return convert_elements(dict(reinterpret_borrow<iterable>(items)), convert);
+    }
+
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         dict d;
@@ -166,20 +283,21 @@ struct list_caster {
     using value_conv = make_caster<Value>;
 
     bool load(handle src, bool convert) {
-        if (!isinstance<sequence>(src) || isinstance<bytes>(src) || isinstance<str>(src)) {
+        if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) {
             return false;
         }
-        auto s = reinterpret_borrow<sequence>(src);
-        value.clear();
-        reserve_maybe(s, &value);
-        for (const auto &it : s) {
-            value_conv conv;
-            if (!conv.load(it, convert)) {
-                return false;
-            }
-            value.push_back(cast_op<Value &&>(std::move(conv)));
+        if (isinstance<sequence>(src)) {
+            return convert_elements(src, convert);
         }
-        return true;
+        if (!convert) {
+            return false;
+        }
+        // Designed to be behavior-equivalent to passing tuple(src) from Python:
+        // The conversion to a tuple will first exhaust the generator object, to ensure that
+        // the generator is not left in an unpredictable (to the caller) partially-consumed
+        // state.
+        assert(isinstance<iterable>(src));
+        return convert_elements(tuple(reinterpret_borrow<iterable>(src)), convert);
     }
 
 private:
@@ -189,6 +307,20 @@ private:
     }
     void reserve_maybe(const sequence &, void *) {}
 
+    bool convert_elements(handle seq, bool convert) {
+        auto s = reinterpret_borrow<sequence>(seq);
+        value.clear();
+        reserve_maybe(s, &value);
+        for (const auto &it : seq) {
+            value_conv conv;
+            if (!conv.load(it, convert)) {
+                return false;
+            }
+            value.push_back(cast_op<Value &&>(std::move(conv)));
+        }
+        return true;
+    }
+
 public:
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
@@ -220,43 +352,87 @@ struct type_caster<std::deque<Type, Alloc>> : list_caster<std::deque<Type, Alloc
 template <typename Type, typename Alloc>
 struct type_caster<std::list<Type, Alloc>> : list_caster<std::list<Type, Alloc>, Type> {};
 
+template <typename ArrayType, typename V, size_t... I>
+ArrayType vector_to_array_impl(V &&v, index_sequence<I...>) {
+    return {{std::move(v[I])...}};
+}
+
+// Based on https://en.cppreference.com/w/cpp/container/array/to_array
+template <typename ArrayType, size_t N, typename V>
+ArrayType vector_to_array(V &&v) {
+    return vector_to_array_impl<ArrayType, V>(std::forward<V>(v), make_index_sequence<N>{});
+}
+
 template <typename ArrayType, typename Value, bool Resizable, size_t Size = 0>
 struct array_caster {
     using value_conv = make_caster<Value>;
 
 private:
-    template <bool R = Resizable>
-    bool require_size(enable_if_t<R, size_t> size) {
-        if (value.size() != size) {
-            value.resize(size);
-        }
-        return true;
-    }
-    template <bool R = Resizable>
-    bool require_size(enable_if_t<!R, size_t> size) {
-        return size == Size;
-    }
+    std::unique_ptr<ArrayType> value;
 
-public:
-    bool load(handle src, bool convert) {
-        if (!isinstance<sequence>(src)) {
-            return false;
-        }
-        auto l = reinterpret_borrow<sequence>(src);
-        if (!require_size(l.size())) {
-            return false;
-        }
+    template <bool R = Resizable, enable_if_t<R, int> = 0>
+    bool convert_elements(handle seq, bool convert) {
+        auto l = reinterpret_borrow<sequence>(seq);
+        value.reset(new ArrayType{});
+        // Using `resize` to preserve the behavior exactly as it was before PR #5305
+        // For the `resize` to work, `Value` must be default constructible.
+        // For `std::valarray`, this is a requirement:
+        // https://en.cppreference.com/w/cpp/named_req/NumericType
+        value->resize(l.size());
         size_t ctr = 0;
         for (const auto &it : l) {
             value_conv conv;
             if (!conv.load(it, convert)) {
                 return false;
             }
-            value[ctr++] = cast_op<Value &&>(std::move(conv));
+            (*value)[ctr++] = cast_op<Value &&>(std::move(conv));
         }
         return true;
     }
 
+    template <bool R = Resizable, enable_if_t<!R, int> = 0>
+    bool convert_elements(handle seq, bool convert) {
+        auto l = reinterpret_borrow<sequence>(seq);
+        if (l.size() != Size) {
+            return false;
+        }
+        // The `temp` storage is needed to support `Value` types that are not
+        // default-constructible.
+        // Deliberate choice: no template specializations, for simplicity, and
+        // because the compile time overhead for the specializations is deemed
+        // more significant than the runtime overhead for the `temp` storage.
+        std::vector<Value> temp;
+        temp.reserve(l.size());
+        for (auto it : l) {
+            value_conv conv;
+            if (!conv.load(it, convert)) {
+                return false;
+            }
+            temp.emplace_back(cast_op<Value &&>(std::move(conv)));
+        }
+        value.reset(new ArrayType(vector_to_array<ArrayType, Size>(std::move(temp))));
+        return true;
+    }
+
+public:
+    bool load(handle src, bool convert) {
+        if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) {
+            return false;
+        }
+        if (isinstance<sequence>(src)) {
+            return convert_elements(src, convert);
+        }
+        if (!convert) {
+            return false;
+        }
+        // Designed to be behavior-equivalent to passing tuple(src) from Python:
+        // The conversion to a tuple will first exhaust the generator object, to ensure that
+        // the generator is not left in an unpredictable (to the caller) partially-consumed
+        // state.
+        assert(isinstance<iterable>(src));
+        return convert_elements(tuple(reinterpret_borrow<iterable>(src)), convert);
+    }
+
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         list l(src.size());
@@ -272,12 +448,36 @@ public:
         return l.release();
     }
 
-    PYBIND11_TYPE_CASTER(ArrayType,
-                         const_name<Resizable>(const_name(""), const_name("Annotated["))
-                             + const_name("list[") + value_conv::name + const_name("]")
-                             + const_name<Resizable>(const_name(""),
-                                                     const_name(", FixedSize(")
-                                                         + const_name<Size>() + const_name(")]")));
+    // Code copied from PYBIND11_TYPE_CASTER macro.
+    // Intentionally preserving the behavior exactly as it was before PR #5305
+    template <typename T_, enable_if_t<std::is_same<ArrayType, remove_cv_t<T_>>::value, int> = 0>
+    static handle cast(T_ *src, return_value_policy policy, handle parent) {
+        if (!src) {
+            return none().release();
+        }
+        if (policy == return_value_policy::take_ownership) {
+            auto h = cast(std::move(*src), policy, parent);
+            delete src; // WARNING: Assumes `src` was allocated with `new`.
+            return h;
+        }
+        return cast(*src, policy, parent);
+    }
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator ArrayType *() { return &(*value); }
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator ArrayType &() { return *value; }
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator ArrayType &&() && { return std::move(*value); }
+
+    template <typename T_>
+    using cast_op_type = movable_cast_op_type<T_>;
+
+    static constexpr auto name
+        = const_name<Resizable>(const_name(""), const_name("Annotated[")) + const_name("list[")
+          + value_conv::name + const_name("]")
+          + const_name<Resizable>(
+              const_name(""), const_name(", FixedSize(") + const_name<Size>() + const_name(")]"));
 };
 
 template <typename Type, size_t Size>

pybind11/include/pybind11/stl/filesystem.h

@@ -4,11 +4,11 @@
 
 #pragma once
 
-#include "../pybind11.h"
-#include "../detail/common.h"
-#include "../detail/descr.h"
-#include "../cast.h"
-#include "../pytypes.h"
+#include <pybind11/cast.h>
+#include <pybind11/detail/common.h>
+#include <pybind11/detail/descr.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/pytypes.h>
 
 #include <string>
 
@@ -33,6 +33,13 @@
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
 
+#ifdef PYPY_VERSION
+#    define PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(...) (__VA_ARGS__)
+#else
+#    define PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(...)                                   \
+        (reinterpret_cast<void *>(__VA_ARGS__))
+#endif
+
 #if defined(PYBIND11_HAS_FILESYSTEM) || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM)
 template <typename T>
 struct path_caster {
@@ -72,7 +79,8 @@ public:
         }
         PyObject *native = nullptr;
         if constexpr (std::is_same_v<typename T::value_type, char>) {
-            if (PyUnicode_FSConverter(buf, &native) != 0) {
+            if (PyUnicode_FSConverter(buf, PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(&native))
+                != 0) {
                 if (auto *c_str = PyBytes_AsString(native)) {
                     // AsString returns a pointer to the internal buffer, which
                     // must not be free'd.
@@ -80,7 +88,8 @@ public:
                 }
             }
         } else if constexpr (std::is_same_v<typename T::value_type, wchar_t>) {
-            if (PyUnicode_FSDecoder(buf, &native) != 0) {
+            if (PyUnicode_FSDecoder(buf, PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(&native))
+                != 0) {
                 if (auto *c_str = PyUnicode_AsWideCharString(native, nullptr)) {
                     // AsWideCharString returns a new string that must be free'd.
                     value = c_str; // Copies the string.
@@ -97,7 +106,7 @@ public:
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(T, const_name("os.PathLike"));
+    PYBIND11_TYPE_CASTER(T, io_name("Union[os.PathLike, str, bytes]", "pathlib.Path"));
 };
 
 #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM)

pybind11/include/pybind11/stl_bind.h

@@ -180,7 +180,7 @@ void vector_modifiers(
                         v.end());
                 try {
                     v.shrink_to_fit();
-                } catch (const std::exception &) {
+                } catch (const std::exception &) { // NOLINT(bugprone-empty-catch)
                     // Do nothing
                 }
                 throw;
@@ -487,7 +487,7 @@ PYBIND11_NAMESPACE_END(detail)
 //
 // std::vector
 //
-template <typename Vector, typename holder_type = std::unique_ptr<Vector>, typename... Args>
+template <typename Vector, typename holder_type = default_holder_type<Vector>, typename... Args>
 class_<Vector, holder_type> bind_vector(handle scope, std::string const &name, Args &&...args) {
     using Class_ = class_<Vector, holder_type>;
 
@@ -694,9 +694,43 @@ struct ItemsViewImpl : public detail::items_view {
     Map &map;
 };
 
+inline str format_message_key_error_key_object(handle py_key) {
+    str message = "pybind11::bind_map key";
+    if (!py_key) {
+        return message;
+    }
+    try {
+        message = str(py_key);
+    } catch (const std::exception &) {
+        try {
+            message = repr(py_key);
+        } catch (const std::exception &) {
+            return message;
+        }
+    }
+    const ssize_t cut_length = 100;
+    if (len(message) > 2 * cut_length + 3) {
+        return str(message[slice(0, cut_length, 1)]) + str("✄✄✄")
+               + str(message[slice(-cut_length, static_cast<ssize_t>(len(message)), 1)]);
+    }
+    return message;
+}
+
+template <typename KeyType>
+str format_message_key_error(const KeyType &key) {
+    object py_key;
+    try {
+        py_key = cast(key);
+    } catch (const std::exception &) {
+        do { // Trick to avoid "empty catch" warning/error.
+        } while (false);
+    }
+    return format_message_key_error_key_object(py_key);
+}
+
 PYBIND11_NAMESPACE_END(detail)
 
-template <typename Map, typename holder_type = std::unique_ptr<Map>, typename... Args>
+template <typename Map, typename holder_type = default_holder_type<Map>, typename... Args>
 class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&...args) {
     using KeyType = typename Map::key_type;
     using MappedType = typename Map::mapped_type;
@@ -785,7 +819,8 @@ class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&
         [](Map &m, const KeyType &k) -> MappedType & {
             auto it = m.find(k);
             if (it == m.end()) {
-                throw key_error();
+                set_error(PyExc_KeyError, detail::format_message_key_error(k));
+                throw error_already_set();
             }
             return it->second;
         },
@@ -808,7 +843,8 @@ class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&
     cl.def("__delitem__", [](Map &m, const KeyType &k) {
         auto it = m.find(k);
         if (it == m.end()) {
-            throw key_error();
+            set_error(PyExc_KeyError, detail::format_message_key_error(k));
+            throw error_already_set();
         }
         m.erase(it);
     });

pybind11/include/pybind11/trampoline_self_life_support.h

@@ -0,0 +1,60 @@
+// Copyright (c) 2021 The Pybind Development Team.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "detail/common.h"
+#include "detail/using_smart_holder.h"
+#include "detail/value_and_holder.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code.
+inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo);
+PYBIND11_NAMESPACE_END(detail)
+
+// The original core idea for this struct goes back to PyCLIF:
+// https://github.com/google/clif/blob/07f95d7e69dca2fcf7022978a55ef3acff506c19/clif/python/runtime.cc#L37
+// URL provided here mainly to give proper credit.
+struct trampoline_self_life_support {
+    detail::value_and_holder v_h;
+
+    trampoline_self_life_support() = default;
+
+    void activate_life_support(const detail::value_and_holder &v_h_) {
+        Py_INCREF((PyObject *) v_h_.inst);
+        v_h = v_h_;
+    }
+
+    void deactivate_life_support() {
+        Py_DECREF((PyObject *) v_h.inst);
+        v_h = detail::value_and_holder();
+    }
+
+    ~trampoline_self_life_support() {
+        if (v_h.inst != nullptr && v_h.vh != nullptr) {
+            void *value_void_ptr = v_h.value_ptr();
+            if (value_void_ptr != nullptr) {
+                PyGILState_STATE threadstate = PyGILState_Ensure();
+                v_h.value_ptr() = nullptr;
+                v_h.holder<smart_holder>().release_disowned();
+                detail::deregister_instance(v_h.inst, value_void_ptr, v_h.type);
+                Py_DECREF((PyObject *) v_h.inst); // Must be after deregister.
+                PyGILState_Release(threadstate);
+            }
+        }
+    }
+
+    // For the next two, the default implementations generate undefined behavior (ASAN failures
+    // manually verified). The reason is that v_h needs to be kept default-initialized.
+    trampoline_self_life_support(const trampoline_self_life_support &) {}
+    trampoline_self_life_support(trampoline_self_life_support &&) noexcept {}
+
+    // These should never be needed (please provide test cases if you think they are).
+    trampoline_self_life_support &operator=(const trampoline_self_life_support &) = delete;
+    trampoline_self_life_support &operator=(trampoline_self_life_support &&) = delete;
+};
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/include/pybind11/typing.h

@@ -14,6 +14,15 @@
 #include "cast.h"
 #include "pytypes.h"
 
+#include <algorithm>
+
+#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L
+#    define PYBIND11_TYPING_H_HAS_STRING_LITERAL
+#    include <numeric>
+#    include <ranges>
+#    include <string_view>
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(typing)
 
@@ -80,6 +89,18 @@ class Optional : public object {
     using object::object;
 };
 
+template <typename T>
+class Final : public object {
+    PYBIND11_OBJECT_DEFAULT(Final, object, PyObject_Type)
+    using object::object;
+};
+
+template <typename T>
+class ClassVar : public object {
+    PYBIND11_OBJECT_DEFAULT(ClassVar, object, PyObject_Type)
+    using object::object;
+};
+
 template <typename T>
 class TypeGuard : public bool_ {
     using bool_::bool_;
@@ -98,7 +119,7 @@ class Never : public none {
     using none::none;
 };
 
-#if defined(__cpp_nontype_template_parameter_class)
+#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL)
 template <size_t N>
 struct StringLiteral {
     constexpr StringLiteral(const char (&str)[N]) { std::copy_n(str, N, name); }
@@ -173,16 +194,19 @@ template <typename Return, typename... Args>
 struct handle_type_name<typing::Callable<Return(Args...)>> {
     using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>;
     static constexpr auto name
-        = const_name("Callable[[") + ::pybind11::detail::concat(make_caster<Args>::name...)
-          + const_name("], ") + make_caster<retval_type>::name + const_name("]");
+        = const_name("Callable[[")
+          + ::pybind11::detail::concat(::pybind11::detail::arg_descr(make_caster<Args>::name)...)
+          + const_name("], ") + ::pybind11::detail::return_descr(make_caster<retval_type>::name)
+          + const_name("]");
 };
 
 template <typename Return>
 struct handle_type_name<typing::Callable<Return(ellipsis)>> {
     // PEP 484 specifies this syntax for defining only return types of callables
     using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>;
-    static constexpr auto name
-        = const_name("Callable[..., ") + make_caster<retval_type>::name + const_name("]");
+    static constexpr auto name = const_name("Callable[..., ")
+                                 + ::pybind11::detail::return_descr(make_caster<retval_type>::name)
+                                 + const_name("]");
 };
 
 template <typename T>
@@ -202,6 +226,16 @@ struct handle_type_name<typing::Optional<T>> {
     static constexpr auto name = const_name("Optional[") + make_caster<T>::name + const_name("]");
 };
 
+template <typename T>
+struct handle_type_name<typing::Final<T>> {
+    static constexpr auto name = const_name("Final[") + make_caster<T>::name + const_name("]");
+};
+
+template <typename T>
+struct handle_type_name<typing::ClassVar<T>> {
+    static constexpr auto name = const_name("ClassVar[") + make_caster<T>::name + const_name("]");
+};
+
 template <typename T>
 struct handle_type_name<typing::TypeGuard<T>> {
     static constexpr auto name = const_name("TypeGuard[") + make_caster<T>::name + const_name("]");
@@ -222,16 +256,36 @@ struct handle_type_name<typing::Never> {
     static constexpr auto name = const_name("Never");
 };
 
-#if defined(__cpp_nontype_template_parameter_class)
+#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL)
+template <typing::StringLiteral StrLit>
+consteval auto sanitize_string_literal() {
+    constexpr std::string_view v(StrLit.name);
+    constexpr std::string_view special_chars("!@%{}-");
+    constexpr auto num_special_chars = std::accumulate(
+        special_chars.begin(), special_chars.end(), (size_t) 0, [&v](auto acc, const char &c) {
+            return std::move(acc) + std::ranges::count(v, c);
+        });
+    char result[v.size() + num_special_chars + 1];
+    size_t i = 0;
+    for (auto c : StrLit.name) {
+        if (special_chars.find(c) != std::string_view::npos) {
+            result[i++] = '!';
+        }
+        result[i++] = c;
+    }
+    return typing::StringLiteral(result);
+}
+
 template <typing::StringLiteral... Literals>
 struct handle_type_name<typing::Literal<Literals...>> {
-    static constexpr auto name = const_name("Literal[")
-                                 + pybind11::detail::concat(const_name(Literals.name)...)
-                                 + const_name("]");
+    static constexpr auto name
+        = const_name("Literal[")
+          + pybind11::detail::concat(const_name(sanitize_string_literal<Literals>().name)...)
+          + const_name("]");
 };
 template <typing::StringLiteral StrLit>
 struct handle_type_name<typing::TypeVar<StrLit>> {
-    static constexpr auto name = const_name(StrLit.name);
+    static constexpr auto name = const_name(sanitize_string_literal<StrLit>().name);
 };
 #endif
 

pybind11/include/pybind11/warnings.h

@@ -0,0 +1,75 @@
+/*
+    pybind11/warnings.h: Python warnings wrappers.
+
+    Copyright (c) 2024 Jan Iwaszkiewicz <jiwaszkiewicz6@gmail.com>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "pybind11.h"
+#include "detail/common.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+inline bool PyWarning_Check(PyObject *obj) {
+    int result = PyObject_IsSubclass(obj, PyExc_Warning);
+    if (result == 1) {
+        return true;
+    }
+    if (result == -1) {
+        raise_from(PyExc_SystemError,
+                   "pybind11::detail::PyWarning_Check(): PyObject_IsSubclass() call failed.");
+        throw error_already_set();
+    }
+    return false;
+}
+
+PYBIND11_NAMESPACE_END(detail)
+
+PYBIND11_NAMESPACE_BEGIN(warnings)
+
+inline object
+new_warning_type(handle scope, const char *name, handle base = PyExc_RuntimeWarning) {
+    if (!detail::PyWarning_Check(base.ptr())) {
+        pybind11_fail("pybind11::warnings::new_warning_type(): cannot create custom warning, base "
+                      "must be a subclass of "
+                      "PyExc_Warning!");
+    }
+    if (hasattr(scope, name)) {
+        pybind11_fail("pybind11::warnings::new_warning_type(): an attribute with name \""
+                      + std::string(name) + "\" exists already.");
+    }
+    std::string full_name = scope.attr("__name__").cast<std::string>() + std::string(".") + name;
+    handle h(PyErr_NewException(full_name.c_str(), base.ptr(), nullptr));
+    if (!h) {
+        raise_from(PyExc_SystemError,
+                   "pybind11::warnings::new_warning_type(): PyErr_NewException() call failed.");
+        throw error_already_set();
+    }
+    auto obj = reinterpret_steal<object>(h);
+    scope.attr(name) = obj;
+    return obj;
+}
+
+// Similar to Python `warnings.warn()`
+inline void
+warn(const char *message, handle category = PyExc_RuntimeWarning, int stack_level = 2) {
+    if (!detail::PyWarning_Check(category.ptr())) {
+        pybind11_fail(
+            "pybind11::warnings::warn(): cannot raise warning, category must be a subclass of "
+            "PyExc_Warning!");
+    }
+
+    if (PyErr_WarnEx(category.ptr(), message, stack_level) == -1) {
+        throw error_already_set();
+    }
+}
+
+PYBIND11_NAMESPACE_END(warnings)
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/pybind11/__init__.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 import sys
 
-if sys.version_info < (3, 7):  # noqa: UP036
-    msg = "pybind11 does not support Python < 3.7. v2.12 was the last release supporting Python 3.6."
+if sys.version_info < (3, 8):  # noqa: UP036
+    msg = "pybind11 does not support Python < 3.8. v2.13 was the last release supporting Python 3.7."
     raise ImportError(msg)
 
 

pybind11/pybind11/__main__.py

@@ -2,12 +2,35 @@
 from __future__ import annotations
 
 import argparse
+import re
 import sys
 import sysconfig
 
 from ._version import __version__
 from .commands import get_cmake_dir, get_include, get_pkgconfig_dir
 
+# This is the conditional used for os.path being posixpath
+if "posix" in sys.builtin_module_names:
+    from shlex import quote
+elif "nt" in sys.builtin_module_names:
+    # See https://github.com/mesonbuild/meson/blob/db22551ed9d2dd7889abea01cc1c7bba02bf1c75/mesonbuild/utils/universal.py#L1092-L1121
+    # and the original documents:
+    # https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments and
+    # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
+    UNSAFE = re.compile("[ \t\n\r]")
+
+    def quote(s: str) -> str:
+        if s and not UNSAFE.search(s):
+            return s
+
+        # Paths cannot contain a '"' on Windows, so we don't need to worry
+        # about nuanced counting here.
+        return f'"{s}\\"' if s.endswith("\\") else f'"{s}"'
+else:
+
+    def quote(s: str) -> str:
+        return s
+
 
 def print_includes() -> None:
     dirs = [
@@ -22,7 +45,7 @@ def print_includes() -> None:
         if d and d not in unique_dirs:
             unique_dirs.append(d)
 
-    print(" ".join("-I" + d for d in unique_dirs))
+    print(" ".join(quote(f"-I{d}") for d in unique_dirs))
 
 
 def main() -> None:
@@ -48,15 +71,22 @@ def main() -> None:
         action="store_true",
         help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.",
     )
+    parser.add_argument(
+        "--extension-suffix",
+        action="store_true",
+        help="Print the extension for a Python module",
+    )
     args = parser.parse_args()
     if not sys.argv[1:]:
         parser.print_help()
     if args.includes:
         print_includes()
     if args.cmakedir:
-        print(get_cmake_dir())
+        print(quote(get_cmake_dir()))
     if args.pkgconfigdir:
-        print(get_pkgconfig_dir())
+        print(quote(get_pkgconfig_dir()))
+    if args.extension_suffix:
+        print(sysconfig.get_config_var("EXT_SUFFIX"))
 
 
 if __name__ == "__main__":

pybind11/pybind11/_version.py

@@ -8,5 +8,5 @@ def _to_int(s: str) -> int | str:
         return s
 
 
-__version__ = "2.13.1"
+__version__ = "3.0.0.dev1"
 version_info = tuple(_to_int(s) for s in __version__.split("."))

pybind11/pybind11/setup_helpers.py

@@ -249,7 +249,7 @@ def has_flag(compiler: Any, flag: str) -> bool:
 cpp_flag_cache = None
 
 
-@lru_cache()
+@lru_cache
 def auto_cpp_level(compiler: Any) -> str | int:
     """
     Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows.

pybind11/pyproject.toml

@@ -30,7 +30,7 @@ ignore_missing_imports = true
 
 
 [tool.pylint]
-master.py-version = "3.7"
+master.py-version = "3.8"
 reports.output-format = "colorized"
 messages_control.disable = [
   "design",
@@ -45,7 +45,7 @@ messages_control.disable = [
 ]
 
 [tool.ruff]
-target-version = "py37"
+target-version = "py38"
 src = ["src"]
 
 [tool.ruff.lint]
@@ -71,7 +71,6 @@ ignore = [
   "PLR",    # Design related pylint
   "E501",   # Line too long (Black is enough)
   "PT011",  # Too broad with raises in pytest
-  "PT004",  # Fixture that doesn't return needs underscore (no, it is fine)
   "SIM118", # iter(x) is not always the same as iter(x.keys())
 ]
 unfixable = ["T20"]

pybind11/setup.cfg

@@ -14,7 +14,6 @@ classifiers =
     Topic :: Utilities
     Programming Language :: C++
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
@@ -39,5 +38,5 @@ project_urls =
     Chat = https://gitter.im/pybind/Lobby
 
 [options]
-python_requires = >=3.7
+python_requires = >=3.8
 zip_safe = False

pybind11/setup.py

@@ -144,6 +144,10 @@ with remove_output("pybind11/include", "pybind11/share"):
             stderr=sys.stderr,
         )
 
+    # pkgconf-pypi needs pybind11/share/pkgconfig to be importable
+    Path("pybind11/share/__init__.py").touch()
+    Path("pybind11/share/pkgconfig/__init__.py").touch()
+
     txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd)
     code = compile(txt, setup_py, "exec")
     exec(code, {"SDist": SDist})

pybind11/tests/CMakeLists.txt

@@ -5,16 +5,7 @@
 # All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
-cmake_minimum_required(VERSION 3.5)
-
-# The `cmake_minimum_required(VERSION 3.5...3.29)` syntax does not work with
-# some versions of VS that have a patched CMake 3.11. This forces us to emulate
-# the behavior using the following workaround:
-if(${CMAKE_VERSION} VERSION_LESS 3.29)
-  cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
-else()
-  cmake_policy(VERSION 3.29)
-endif()
+cmake_minimum_required(VERSION 3.15...3.30)
 
 # Filter out items; print an optional message if any items filtered. This ignores extensions.
 #
@@ -76,8 +67,8 @@ project(pybind11_tests CXX)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/../tools")
 
 option(PYBIND11_WERROR "Report all warnings as errors" OFF)
-option(DOWNLOAD_EIGEN "Download EIGEN (requires CMake 3.11+)" OFF)
-option(PYBIND11_CUDA_TESTS "Enable building CUDA tests (requires CMake 3.12+)" OFF)
+option(DOWNLOAD_EIGEN "Download EIGEN" OFF)
+option(PYBIND11_CUDA_TESTS "Enable building CUDA tests" OFF)
 set(PYBIND11_TEST_OVERRIDE
     ""
     CACHE STRING "Tests from ;-separated list of *.cpp files will be built instead of all tests")
@@ -88,7 +79,12 @@ set(PYBIND11_TEST_FILTER
 if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
   # We're being loaded directly, i.e. not via add_subdirectory, so make this
   # work as its own project and load the pybind11Config to get the tools we need
-  find_package(pybind11 REQUIRED CONFIG)
+
+  if(SKBUILD)
+    add_subdirectory(.. pybind11_src)
+  else()
+    find_package(pybind11 REQUIRED CONFIG)
+  endif()
 endif()
 
 if(NOT CMAKE_BUILD_TYPE AND NOT DEFINED CMAKE_CONFIGURATION_TYPES)
@@ -119,12 +115,32 @@ set(PYBIND11_TEST_FILES
     test_callbacks
     test_chrono
     test_class
+    test_class_release_gil_before_calling_cpp_dtor
+    test_class_sh_basic
+    test_class_sh_disowning
+    test_class_sh_disowning_mi
+    test_class_sh_factory_constructors
+    test_class_sh_inheritance
+    test_class_sh_mi_thunks
+    test_class_sh_property
+    test_class_sh_property_non_owning
+    test_class_sh_shared_ptr_copy_move
+    test_class_sh_trampoline_basic
+    test_class_sh_trampoline_self_life_support
+    test_class_sh_trampoline_shared_from_this
+    test_class_sh_trampoline_shared_ptr_cpp_arg
+    test_class_sh_trampoline_unique_ptr
+    test_class_sh_unique_ptr_custom_deleter
+    test_class_sh_unique_ptr_member
+    test_class_sh_virtual_py_cpp_mix
     test_const_name
     test_constants_and_functions
     test_copy_move
+    test_cpp_conduit
     test_custom_type_casters
     test_custom_type_setup
     test_docstring_options
+    test_docs_advanced_cast_custom
     test_eigen_matrix
     test_eigen_tensor
     test_enum
@@ -153,11 +169,13 @@ set(PYBIND11_TEST_FILES
     test_tagbased_polymorphic
     test_thread
     test_type_caster_pyobject_ptr
+    test_type_caster_std_function_specializations
     test_union
     test_unnamed_namespace_a
     test_unnamed_namespace_b
     test_vector_unique_ptr_member
-    test_virtual_functions)
+    test_virtual_functions
+    test_warnings)
 
 # Invoking cmake with something like:
 #     cmake -DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp" ..
@@ -220,6 +238,8 @@ tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_
 # And add additional targets for other tests.
 tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set")
 tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils")
+tests_extra_targets("test_cpp_conduit.py"
+                    "exo_planet_pybind11;exo_planet_c_api;home_planet_very_lonely_traveler")
 
 set(PYBIND11_EIGEN_REPO
     "https://gitlab.com/libeigen/eigen.git"
@@ -243,25 +263,21 @@ endif()
 if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
   # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake).
   # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also
-  # produces a fatal error if loaded from a pre-3.0 cmake.
   if(DOWNLOAD_EIGEN)
-    if(CMAKE_VERSION VERSION_LESS 3.11)
-      message(FATAL_ERROR "CMake 3.11+ required when using DOWNLOAD_EIGEN")
+    if(CMAKE_VERSION VERSION_LESS 3.18)
+      set(_opts)
+    else()
+      set(_opts SOURCE_SUBDIR no-cmake-build)
     endif()
-
     include(FetchContent)
     FetchContent_Declare(
       eigen
       GIT_REPOSITORY "${PYBIND11_EIGEN_REPO}"
-      GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}")
-
-    FetchContent_GetProperties(eigen)
-    if(NOT eigen_POPULATED)
-      message(
-        STATUS
-          "Downloading Eigen ${PYBIND11_EIGEN_VERSION_STRING} (${PYBIND11_EIGEN_VERSION_HASH}) from ${PYBIND11_EIGEN_REPO}"
-      )
-      FetchContent_Populate(eigen)
+      GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}"
+      ${_opts})
+    FetchContent_MakeAvailable(eigen)
+    if(NOT CMAKE_VERSION VERSION_LESS 3.18)
+      set(EIGEN3_INCLUDE_DIR "${eigen_SOURCE_DIR}")
     endif()
 
     set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR})
@@ -309,8 +325,7 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
     if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
       list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I})
     endif()
-    message(
-      STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download")
+    message(STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON to download")
   endif()
 endif()
 
@@ -390,6 +405,9 @@ function(pybind11_enable_warnings target_name)
               -Wdeprecated
               -Wundef
               -Wnon-virtual-dtor)
+    if(DEFINED CMAKE_CXX_STANDARD AND NOT CMAKE_CXX_STANDARD VERSION_LESS 20)
+      target_compile_options(${target_name} PRIVATE -Wpedantic)
+    endif()
   endif()
 
   if(PYBIND11_WERROR)
@@ -489,15 +507,22 @@ foreach(target ${test_targets})
       endforeach()
     endif()
   endif()
+  if(SKBUILD)
+    install(TARGETS ${target} LIBRARY DESTINATION .)
+  endif()
+
+  if("${target}" STREQUAL "exo_planet_c_api")
+    if(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang|NVHPC)")
+      target_compile_options(${target} PRIVATE -fno-exceptions)
+    endif()
+  endif()
 endforeach()
 
 # Provide nice organisation in IDEs
-if(NOT CMAKE_VERSION VERSION_LESS 3.8)
-  source_group(
-    TREE "${CMAKE_CURRENT_SOURCE_DIR}/../include"
-    PREFIX "Header Files"
-    FILES ${PYBIND11_HEADERS})
-endif()
+source_group(
+  TREE "${CMAKE_CURRENT_SOURCE_DIR}/../include"
+  PREFIX "Header Files"
+  FILES ${PYBIND11_HEADERS})
 
 # Make sure pytest is found or produce a warning
 pybind11_find_import(pytest VERSION 3.1)
@@ -581,6 +606,9 @@ add_custom_command(
     ${CMAKE_CURRENT_BINARY_DIR}/sosize-$<TARGET_FILE_NAME:pybind11_tests>.txt)
 
 if(NOT PYBIND11_CUDA_TESTS)
+  # Test pure C++ code (not depending on Python). Provides the `test_pure_cpp` target.
+  add_subdirectory(pure_cpp)
+
   # Test embedding the interpreter. Provides the `cpptest` target.
   add_subdirectory(test_embed)
 

pybind11/tests/conftest.py

@@ -28,8 +28,8 @@ except Exception:
 
 @pytest.fixture(scope="session", autouse=True)
 def use_multiprocessing_forkserver_on_linux():
-    if sys.platform != "linux":
-        # The default on Windows and macOS is "spawn": If it's not broken, don't fix it.
+    if sys.platform != "linux" or sys.implementation.name == "graalpy":
+        # The default on Windows, macOS and GraalPy is "spawn": If it's not broken, don't fix it.
         return
 
     # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592
@@ -136,7 +136,7 @@ class Capture:
         return Output(self.err)
 
 
-@pytest.fixture()
+@pytest.fixture
 def capture(capsys):
     """Extended `capsys` with context manager and custom equality operators"""
     return Capture(capsys)
@@ -172,7 +172,7 @@ def _sanitize_docstring(thing):
     return _sanitize_general(s)
 
 
-@pytest.fixture()
+@pytest.fixture
 def doc():
     """Sanitize docstrings and add custom failure explanation"""
     return SanitizedString(_sanitize_docstring)
@@ -184,7 +184,7 @@ def _sanitize_message(thing):
     return _hexadecimal.sub("0", s)
 
 
-@pytest.fixture()
+@pytest.fixture
 def msg():
     """Sanitize messages and add custom failure explanation"""
     return SanitizedString(_sanitize_message)
@@ -198,10 +198,11 @@ def pytest_assertrepr_compare(op, left, right):  # noqa: ARG001
 
 
 def gc_collect():
-    """Run the garbage collector twice (needed when running
+    """Run the garbage collector three times (needed when running
     reference counting tests with PyPy)"""
     gc.collect()
     gc.collect()
+    gc.collect()
 
 
 def pytest_configure():
@@ -211,9 +212,9 @@ def pytest_configure():
 
 def pytest_report_header(config):
     del config  # Unused.
-    assert (
-        pybind11_tests.compiler_info is not None
-    ), "Please update pybind11_tests.cpp if this assert fails."
+    assert pybind11_tests.compiler_info is not None, (
+        "Please update pybind11_tests.cpp if this assert fails."
+    )
     return (
         "C++ Info:"
         f" {pybind11_tests.compiler_info}"

pybind11/tests/constructor_stats.h

@@ -190,7 +190,7 @@ public:
                     t1 = &p.first;
                 }
             }
-        } catch (const std::out_of_range &) {
+        } catch (const std::out_of_range &) { // NOLINT(bugprone-empty-catch)
         }
         if (!t1) {
             throw std::runtime_error("Unknown class passed to ConstructorStats::get()");
@@ -312,8 +312,16 @@ void print_created(T *inst, Values &&...values) {
 }
 template <class T, typename... Values>
 void print_destroyed(T *inst, Values &&...values) { // Prints but doesn't store given values
+    /*
+     * On GraalPy, destructors can trigger anywhere and this can cause random
+     * failures in unrelated tests.
+     */
+#if !defined(GRAALVM_PYTHON)
     print_constr_details(inst, "destroyed", values...);
     track_destroyed(inst);
+#else
+    py::detail::silence_unused_warnings(inst, values...);
+#endif
 }
 template <class T, typename... Values>
 void print_values(T *inst, Values &&...values) {

pybind11/tests/custom_exceptions.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+
+class PythonMyException7(Exception):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(message)
+
+    def __str__(self):
+        return "[PythonMyException7]: " + self.message.a

pybind11/tests/env.py

@@ -12,6 +12,7 @@ WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin")
 
 CPYTHON = platform.python_implementation() == "CPython"
 PYPY = platform.python_implementation() == "PyPy"
+GRAALPY = sys.implementation.name == "graalpy"
 PY_GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
 
 

pybind11/tests/exo_planet_c_api.cpp

@@ -0,0 +1,76 @@
+// Copyright (c) 2024 The pybind Community.
+
+// In production situations it is totally fine to build with
+// C++ Exception Handling enabled. However, here we want to ensure that
+// C++ Exception Handling is not required.
+#if defined(_MSC_VER) || defined(__EMSCRIPTEN__)
+// Too much trouble making the required cmake changes (see PR #5375).
+#else
+#    ifdef __cpp_exceptions
+// https://isocpp.org/std/standing-documents/sd-6-sg10-feature-test-recommendations#__cpp_exceptions
+#    error This test is meant to be built with C++ Exception Handling disabled, but __cpp_exceptions is defined.
+#    endif
+#    ifdef __EXCEPTIONS
+// https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
+#    error This test is meant to be built with C++ Exception Handling disabled, but __EXCEPTIONS is defined.
+#    endif
+#endif
+
+// THIS MUST STAY AT THE TOP!
+#include <pybind11/conduit/pybind11_conduit_v1.h> // VERY light-weight dependency.
+
+#include "test_cpp_conduit_traveler_types.h"
+
+#include <Python.h>
+
+namespace {
+
+extern "C" PyObject *wrapGetLuggage(PyObject * /*self*/, PyObject *traveler) {
+    const auto *cpp_traveler = pybind11_conduit_v1::get_type_pointer_ephemeral<
+        pybind11_tests::test_cpp_conduit::Traveler>(traveler);
+    if (cpp_traveler == nullptr) {
+        return nullptr;
+    }
+    return PyUnicode_FromString(cpp_traveler->luggage.c_str());
+}
+
+extern "C" PyObject *wrapGetPoints(PyObject * /*self*/, PyObject *premium_traveler) {
+    const auto *cpp_premium_traveler = pybind11_conduit_v1::get_type_pointer_ephemeral<
+        pybind11_tests::test_cpp_conduit::PremiumTraveler>(premium_traveler);
+    if (cpp_premium_traveler == nullptr) {
+        return nullptr;
+    }
+    return PyLong_FromLong(static_cast<long>(cpp_premium_traveler->points));
+}
+
+PyMethodDef ThisMethodDef[] = {{"GetLuggage", wrapGetLuggage, METH_O, nullptr},
+                               {"GetPoints", wrapGetPoints, METH_O, nullptr},
+                               {nullptr, nullptr, 0, nullptr}};
+
+struct PyModuleDef ThisModuleDef = {
+    PyModuleDef_HEAD_INIT, // m_base
+    "exo_planet_c_api",    // m_name
+    nullptr,               // m_doc
+    -1,                    // m_size
+    ThisMethodDef,         // m_methods
+    nullptr,               // m_slots
+    nullptr,               // m_traverse
+    nullptr,               // m_clear
+    nullptr                // m_free
+};
+
+} // namespace
+
+#if defined(WIN32) || defined(_WIN32)
+#    define EXO_PLANET_C_API_EXPORT __declspec(dllexport)
+#else
+#    define EXO_PLANET_C_API_EXPORT __attribute__((visibility("default")))
+#endif
+
+extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() {
+    PyObject *m = PyModule_Create(&ThisModuleDef);
+    if (m == nullptr) {
+        return nullptr;
+    }
+    return m;
+}

pybind11/tests/exo_planet_pybind11.cpp

@@ -0,0 +1,19 @@
+// Copyright (c) 2024 The pybind Community.
+
+#if defined(PYBIND11_INTERNALS_VERSION)
+#    undef PYBIND11_INTERNALS_VERSION
+#endif
+#define PYBIND11_INTERNALS_VERSION 900000001
+
+#include "test_cpp_conduit_traveler_bindings.h"
+
+namespace pybind11_tests {
+namespace test_cpp_conduit {
+
+PYBIND11_MODULE(exo_planet_pybind11, m) {
+    wrap_traveler(m);
+    m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); });
+}
+
+} // namespace test_cpp_conduit
+} // namespace pybind11_tests

pybind11/tests/extra_python_package/test_files.py

@@ -9,7 +9,6 @@ import tarfile
 import zipfile
 
 # These tests must be run explicitly
-# They require CMake 3.15+ (--install)
 
 DIR = os.path.abspath(os.path.dirname(__file__))
 MAIN_DIR = os.path.dirname(os.path.dirname(DIR))
@@ -46,18 +45,33 @@ main_headers = {
     "include/pybind11/pytypes.h",
     "include/pybind11/stl.h",
     "include/pybind11/stl_bind.h",
+    "include/pybind11/trampoline_self_life_support.h",
     "include/pybind11/type_caster_pyobject_ptr.h",
     "include/pybind11/typing.h",
+    "include/pybind11/warnings.h",
+}
+
+conduit_headers = {
+    "include/pybind11/conduit/README.txt",
+    "include/pybind11/conduit/pybind11_conduit_v1.h",
+    "include/pybind11/conduit/pybind11_platform_abi_id.h",
+    "include/pybind11/conduit/wrap_include_python_h.h",
 }
 
 detail_headers = {
     "include/pybind11/detail/class.h",
     "include/pybind11/detail/common.h",
+    "include/pybind11/detail/cpp_conduit.h",
     "include/pybind11/detail/descr.h",
+    "include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h",
     "include/pybind11/detail/init.h",
     "include/pybind11/detail/internals.h",
+    "include/pybind11/detail/struct_smart_holder.h",
     "include/pybind11/detail/type_caster_base.h",
     "include/pybind11/detail/typeid.h",
+    "include/pybind11/detail/using_smart_holder.h",
+    "include/pybind11/detail/value_and_holder.h",
+    "include/pybind11/detail/exception_translation.h",
 }
 
 eigen_headers = {
@@ -92,9 +106,11 @@ py_files = {
     "commands.py",
     "py.typed",
     "setup_helpers.py",
+    "share/__init__.py",
+    "share/pkgconfig/__init__.py",
 }
 
-headers = main_headers | detail_headers | eigen_headers | stl_headers
+headers = main_headers | conduit_headers | detail_headers | eigen_headers | stl_headers
 src_files = headers | cmake_files | pkgconfig_files
 all_files = src_files | py_files
 
@@ -103,6 +119,7 @@ sdist_files = {
     "pybind11",
     "pybind11/include",
     "pybind11/include/pybind11",
+    "pybind11/include/pybind11/conduit",
     "pybind11/include/pybind11/detail",
     "pybind11/include/pybind11/eigen",
     "pybind11/include/pybind11/stl",

pybind11/tests/home_planet_very_lonely_traveler.cpp

@@ -0,0 +1,13 @@
+// Copyright (c) 2024 The pybind Community.
+
+#include "test_cpp_conduit_traveler_bindings.h"
+
+namespace pybind11_tests {
+namespace test_cpp_conduit {
+
+PYBIND11_MODULE(home_planet_very_lonely_traveler, m) {
+    m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); });
+}
+
+} // namespace test_cpp_conduit
+} // namespace pybind11_tests

pybind11/tests/local_bindings.h

@@ -56,13 +56,13 @@ private:
     std::string message = "";
 };
 
-PYBIND11_MAKE_OPAQUE(LocalVec);
-PYBIND11_MAKE_OPAQUE(LocalVec2);
-PYBIND11_MAKE_OPAQUE(LocalMap);
-PYBIND11_MAKE_OPAQUE(NonLocalVec);
-// PYBIND11_MAKE_OPAQUE(NonLocalVec2); // same type as LocalVec2
-PYBIND11_MAKE_OPAQUE(NonLocalMap);
-PYBIND11_MAKE_OPAQUE(NonLocalMap2);
+PYBIND11_MAKE_OPAQUE(LocalVec)
+PYBIND11_MAKE_OPAQUE(LocalVec2)
+PYBIND11_MAKE_OPAQUE(LocalMap)
+PYBIND11_MAKE_OPAQUE(NonLocalVec)
+// PYBIND11_MAKE_OPAQUE(NonLocalVec2) // same type as LocalVec2
+PYBIND11_MAKE_OPAQUE(NonLocalMap)
+PYBIND11_MAKE_OPAQUE(NonLocalMap2)
 
 // Simple bindings (used with the above):
 template <typename T, int Adjust = 0, typename... Args>
@@ -70,7 +70,7 @@ py::class_<T> bind_local(Args &&...args) {
     return py::class_<T>(std::forward<Args>(args)...).def(py::init<int>()).def("get", [](T &i) {
         return i.i + Adjust;
     });
-};
+}
 
 // Simulate a foreign library base class (to match the example in the docs):
 namespace pets {

pybind11/tests/pure_cpp/CMakeLists.txt

@@ -0,0 +1,20 @@
+find_package(Catch 2.13.2)
+
+if(CATCH_FOUND)
+  message(STATUS "Building pure C++ tests (not depending on Python) using Catch v${CATCH_VERSION}")
+else()
+  message(STATUS "Catch not detected. Interpreter tests will be skipped. Install Catch headers"
+                 " manually or use `cmake -DDOWNLOAD_CATCH=ON` to fetch them automatically.")
+  return()
+endif()
+
+add_executable(smart_holder_poc_test smart_holder_poc_test.cpp)
+pybind11_enable_warnings(smart_holder_poc_test)
+target_link_libraries(smart_holder_poc_test PRIVATE pybind11::headers Catch2::Catch2)
+
+add_custom_target(
+  test_pure_cpp
+  COMMAND "$<TARGET_FILE:smart_holder_poc_test>"
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+
+add_dependencies(check test_pure_cpp)

pybind11/tests/pure_cpp/smart_holder_poc.h

@@ -0,0 +1,51 @@
+// Copyright (c) 2020-2024 The Pybind Development Team.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "pybind11/detail/struct_smart_holder.h"
+
+namespace pybindit {
+namespace memory {
+namespace smart_holder_poc { // Proof-of-Concept implementations.
+
+template <typename T>
+T &as_lvalue_ref(const smart_holder &hld) {
+    static const char *context = "as_lvalue_ref";
+    hld.ensure_is_populated(context);
+    hld.ensure_has_pointee(context);
+    return *hld.as_raw_ptr_unowned<T>();
+}
+
+template <typename T>
+T &&as_rvalue_ref(const smart_holder &hld) {
+    static const char *context = "as_rvalue_ref";
+    hld.ensure_is_populated(context);
+    hld.ensure_has_pointee(context);
+    return std::move(*hld.as_raw_ptr_unowned<T>());
+}
+
+template <typename T>
+T *as_raw_ptr_release_ownership(smart_holder &hld,
+                                const char *context = "as_raw_ptr_release_ownership") {
+    hld.ensure_can_release_ownership(context);
+    T *raw_ptr = hld.as_raw_ptr_unowned<T>();
+    hld.release_ownership();
+    return raw_ptr;
+}
+
+template <typename T, typename D = std::default_delete<T>>
+std::unique_ptr<T, D> as_unique_ptr(smart_holder &hld) {
+    static const char *context = "as_unique_ptr";
+    hld.ensure_compatible_rtti_uqp_del<T, D>(context);
+    hld.ensure_use_count_1(context);
+    T *raw_ptr = hld.as_raw_ptr_unowned<T>();
+    hld.release_ownership();
+    // KNOWN DEFECT (see PR #4850): Does not copy the deleter.
+    return std::unique_ptr<T, D>(raw_ptr);
+}
+
+} // namespace smart_holder_poc
+} // namespace memory
+} // namespace pybindit

pybind11/tests/pure_cpp/smart_holder_poc_test.cpp

@@ -0,0 +1,415 @@
+#include "smart_holder_poc.h"
+
+#include <functional>
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+// Catch uses _ internally, which breaks gettext style defines
+#ifdef _
+#    undef _
+#endif
+
+#define CATCH_CONFIG_MAIN
+#include "catch.hpp"
+
+using pybindit::memory::smart_holder;
+namespace poc = pybindit::memory::smart_holder_poc;
+
+namespace helpers {
+
+struct movable_int {
+    int valu;
+    explicit movable_int(int v) : valu{v} {}
+    movable_int(movable_int &&other) noexcept : valu(other.valu) { other.valu = 91; }
+};
+
+template <typename T>
+struct functor_builtin_delete {
+    void operator()(T *ptr) { delete ptr; }
+#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8)                                   \
+    || (defined(__clang_major__) && __clang_major__ == 3 && __clang_minor__ == 6)
+    // Workaround for these errors:
+    // gcc 4.8.5: too many initializers for 'helpers::functor_builtin_delete<int>'
+    // clang 3.6: excess elements in struct initializer
+    functor_builtin_delete() = default;
+    functor_builtin_delete(const functor_builtin_delete &) {}
+    functor_builtin_delete(functor_builtin_delete &&) {}
+#endif
+};
+
+template <typename T>
+struct functor_other_delete : functor_builtin_delete<T> {};
+
+struct indestructible_int {
+    int valu;
+    explicit indestructible_int(int v) : valu{v} {}
+
+private:
+    ~indestructible_int() = default;
+};
+
+struct base {
+    virtual int get() { return 10; }
+    virtual ~base() = default;
+};
+
+struct derived : public base {
+    int get() override { return 100; }
+};
+
+} // namespace helpers
+
+TEST_CASE("from_raw_ptr_unowned+as_raw_ptr_unowned", "[S]") {
+    static int value = 19;
+    auto hld = smart_holder::from_raw_ptr_unowned(&value);
+    REQUIRE(*hld.as_raw_ptr_unowned<int>() == 19);
+}
+
+TEST_CASE("from_raw_ptr_unowned+as_lvalue_ref", "[S]") {
+    static int value = 19;
+    auto hld = smart_holder::from_raw_ptr_unowned(&value);
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+}
+
+TEST_CASE("from_raw_ptr_unowned+as_rvalue_ref", "[S]") {
+    helpers::movable_int orig(19);
+    {
+        auto hld = smart_holder::from_raw_ptr_unowned(&orig);
+        helpers::movable_int othr(poc::as_rvalue_ref<helpers::movable_int>(hld));
+        REQUIRE(othr.valu == 19);
+        REQUIRE(orig.valu == 91);
+    }
+}
+
+TEST_CASE("from_raw_ptr_unowned+as_raw_ptr_release_ownership", "[E]") {
+    static int value = 19;
+    auto hld = smart_holder::from_raw_ptr_unowned(&value);
+    REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership<int>(hld),
+                        "Cannot disown non-owning holder (as_raw_ptr_release_ownership).");
+}
+
+TEST_CASE("from_raw_ptr_unowned+as_unique_ptr", "[E]") {
+    static int value = 19;
+    auto hld = smart_holder::from_raw_ptr_unowned(&value);
+    REQUIRE_THROWS_WITH(poc::as_unique_ptr<int>(hld),
+                        "Cannot disown non-owning holder (as_unique_ptr).");
+}
+
+TEST_CASE("from_raw_ptr_unowned+as_unique_ptr_with_deleter", "[E]") {
+    static int value = 19;
+    auto hld = smart_holder::from_raw_ptr_unowned(&value);
+    REQUIRE_THROWS_WITH((poc::as_unique_ptr<int, helpers::functor_builtin_delete<int>>(hld)),
+                        "Missing unique_ptr deleter (as_unique_ptr).");
+}
+
+TEST_CASE("from_raw_ptr_unowned+as_shared_ptr", "[S]") {
+    static int value = 19;
+    auto hld = smart_holder::from_raw_ptr_unowned(&value);
+    REQUIRE(*hld.as_shared_ptr<int>() == 19);
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_lvalue_ref", "[S]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    REQUIRE(hld.has_pointee());
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_raw_ptr_release_ownership1", "[S]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    auto new_owner = std::unique_ptr<int>(poc::as_raw_ptr_release_ownership<int>(hld));
+    REQUIRE(!hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_raw_ptr_release_ownership2", "[E]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    auto shd_ptr = hld.as_shared_ptr<int>();
+    REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership<int>(hld),
+                        "Cannot disown use_count != 1 (as_raw_ptr_release_ownership).");
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr1", "[S]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    std::unique_ptr<int> new_owner = poc::as_unique_ptr<int>(hld);
+    REQUIRE(!hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr2", "[E]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    auto shd_ptr = hld.as_shared_ptr<int>();
+    REQUIRE_THROWS_WITH(poc::as_unique_ptr<int>(hld),
+                        "Cannot disown use_count != 1 (as_unique_ptr).");
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr_with_deleter", "[E]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    REQUIRE_THROWS_WITH((poc::as_unique_ptr<int, helpers::functor_builtin_delete<int>>(hld)),
+                        "Missing unique_ptr deleter (as_unique_ptr).");
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_shared_ptr", "[S]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    std::shared_ptr<int> new_owner = hld.as_shared_ptr<int>();
+    REQUIRE(hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+disown+reclaim_disowned", "[S]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    std::unique_ptr<int> new_owner(hld.as_raw_ptr_unowned<int>());
+    hld.disown();
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+    REQUIRE(*new_owner == 19);
+    hld.reclaim_disowned(); // Manually veriified: without this, clang++ -fsanitize=address reports
+                            // "detected memory leaks".
+    // NOLINTNEXTLINE(bugprone-unused-return-value)
+    (void) new_owner.release(); // Manually verified: without this, clang++ -fsanitize=address
+                                // reports "attempting double-free".
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+    REQUIRE(new_owner.get() == nullptr);
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+disown+release_disowned", "[S]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    std::unique_ptr<int> new_owner(hld.as_raw_ptr_unowned<int>());
+    hld.disown();
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+    REQUIRE(*new_owner == 19);
+    hld.release_disowned();
+    REQUIRE(!hld.has_pointee());
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+disown+ensure_is_not_disowned", "[E]") {
+    const char *context = "test_case";
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    hld.ensure_is_not_disowned(context); // Does not throw.
+    std::unique_ptr<int> new_owner(hld.as_raw_ptr_unowned<int>());
+    hld.disown();
+    REQUIRE_THROWS_WITH(hld.ensure_is_not_disowned(context),
+                        "Holder was disowned already (test_case).");
+}
+
+TEST_CASE("from_unique_ptr+as_lvalue_ref", "[S]") {
+    std::unique_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+}
+
+TEST_CASE("from_unique_ptr+as_raw_ptr_release_ownership1", "[S]") {
+    std::unique_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    auto new_owner = std::unique_ptr<int>(poc::as_raw_ptr_release_ownership<int>(hld));
+    REQUIRE(!hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_unique_ptr+as_raw_ptr_release_ownership2", "[E]") {
+    std::unique_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    auto shd_ptr = hld.as_shared_ptr<int>();
+    REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership<int>(hld),
+                        "Cannot disown use_count != 1 (as_raw_ptr_release_ownership).");
+}
+
+TEST_CASE("from_unique_ptr+as_unique_ptr1", "[S]") {
+    std::unique_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    std::unique_ptr<int> new_owner = poc::as_unique_ptr<int>(hld);
+    REQUIRE(!hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_unique_ptr+as_unique_ptr2", "[E]") {
+    std::unique_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    auto shd_ptr = hld.as_shared_ptr<int>();
+    REQUIRE_THROWS_WITH(poc::as_unique_ptr<int>(hld),
+                        "Cannot disown use_count != 1 (as_unique_ptr).");
+}
+
+TEST_CASE("from_unique_ptr+as_unique_ptr_with_deleter", "[E]") {
+    std::unique_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE_THROWS_WITH((poc::as_unique_ptr<int, helpers::functor_builtin_delete<int>>(hld)),
+                        "Incompatible unique_ptr deleter (as_unique_ptr).");
+}
+
+TEST_CASE("from_unique_ptr+as_shared_ptr", "[S]") {
+    std::unique_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    std::shared_ptr<int> new_owner = hld.as_shared_ptr<int>();
+    REQUIRE(hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_unique_ptr_derived+as_unique_ptr_base", "[S]") {
+    std::unique_ptr<helpers::derived> orig_owner(new helpers::derived());
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    std::unique_ptr<helpers::base> new_owner = poc::as_unique_ptr<helpers::base>(hld);
+    REQUIRE(!hld.has_pointee());
+    REQUIRE(new_owner->get() == 100);
+}
+
+TEST_CASE("from_unique_ptr_derived+as_unique_ptr_base2", "[E]") {
+    std::unique_ptr<helpers::derived, helpers::functor_other_delete<helpers::derived>> orig_owner(
+        new helpers::derived());
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE_THROWS_WITH(
+        (poc::as_unique_ptr<helpers::base, helpers::functor_builtin_delete<helpers::base>>(hld)),
+        "Incompatible unique_ptr deleter (as_unique_ptr).");
+}
+
+TEST_CASE("from_unique_ptr_with_deleter+as_lvalue_ref", "[S]") {
+    std::unique_ptr<int, helpers::functor_builtin_delete<int>> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+}
+
+TEST_CASE("from_unique_ptr_with_std_function_deleter+as_lvalue_ref", "[S]") {
+    std::unique_ptr<int, std::function<void(const int *)>> orig_owner(
+        new int(19), [](const int *raw_ptr) { delete raw_ptr; });
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+}
+
+TEST_CASE("from_unique_ptr_with_deleter+as_raw_ptr_release_ownership", "[E]") {
+    std::unique_ptr<int, helpers::functor_builtin_delete<int>> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership<int>(hld),
+                        "Cannot disown custom deleter (as_raw_ptr_release_ownership).");
+}
+
+TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr", "[E]") {
+    std::unique_ptr<int, helpers::functor_builtin_delete<int>> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE_THROWS_WITH(poc::as_unique_ptr<int>(hld),
+                        "Incompatible unique_ptr deleter (as_unique_ptr).");
+}
+
+TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr_with_deleter1", "[S]") {
+    std::unique_ptr<int, helpers::functor_builtin_delete<int>> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    std::unique_ptr<int, helpers::functor_builtin_delete<int>> new_owner
+        = poc::as_unique_ptr<int, helpers::functor_builtin_delete<int>>(hld);
+    REQUIRE(!hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr_with_deleter2", "[E]") {
+    std::unique_ptr<int, helpers::functor_builtin_delete<int>> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    REQUIRE_THROWS_WITH((poc::as_unique_ptr<int, helpers::functor_other_delete<int>>(hld)),
+                        "Incompatible unique_ptr deleter (as_unique_ptr).");
+}
+
+TEST_CASE("from_unique_ptr_with_deleter+as_shared_ptr", "[S]") {
+    std::unique_ptr<int, helpers::functor_builtin_delete<int>> orig_owner(new int(19));
+    auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+    REQUIRE(orig_owner.get() == nullptr);
+    std::shared_ptr<int> new_owner = hld.as_shared_ptr<int>();
+    REQUIRE(hld.has_pointee());
+    REQUIRE(*new_owner == 19);
+}
+
+TEST_CASE("from_shared_ptr+as_lvalue_ref", "[S]") {
+    std::shared_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_shared_ptr(orig_owner);
+    REQUIRE(poc::as_lvalue_ref<int>(hld) == 19);
+}
+
+TEST_CASE("from_shared_ptr+as_raw_ptr_release_ownership", "[E]") {
+    std::shared_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_shared_ptr(orig_owner);
+    REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership<int>(hld),
+                        "Cannot disown external shared_ptr (as_raw_ptr_release_ownership).");
+}
+
+TEST_CASE("from_shared_ptr+as_unique_ptr", "[E]") {
+    std::shared_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_shared_ptr(orig_owner);
+    REQUIRE_THROWS_WITH(poc::as_unique_ptr<int>(hld),
+                        "Cannot disown external shared_ptr (as_unique_ptr).");
+}
+
+TEST_CASE("from_shared_ptr+as_unique_ptr_with_deleter", "[E]") {
+    std::shared_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_shared_ptr(orig_owner);
+    REQUIRE_THROWS_WITH((poc::as_unique_ptr<int, helpers::functor_builtin_delete<int>>(hld)),
+                        "Missing unique_ptr deleter (as_unique_ptr).");
+}
+
+TEST_CASE("from_shared_ptr+as_shared_ptr", "[S]") {
+    std::shared_ptr<int> orig_owner(new int(19));
+    auto hld = smart_holder::from_shared_ptr(orig_owner);
+    REQUIRE(*hld.as_shared_ptr<int>() == 19);
+}
+
+TEST_CASE("error_unpopulated_holder", "[E]") {
+    smart_holder hld;
+    REQUIRE_THROWS_WITH(poc::as_lvalue_ref<int>(hld), "Unpopulated holder (as_lvalue_ref).");
+}
+
+TEST_CASE("error_disowned_holder", "[E]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    poc::as_unique_ptr<int>(hld);
+    REQUIRE_THROWS_WITH(poc::as_lvalue_ref<int>(hld), "Disowned holder (as_lvalue_ref).");
+}
+
+TEST_CASE("error_cannot_disown_nullptr", "[E]") {
+    auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+    poc::as_unique_ptr<int>(hld);
+    REQUIRE_THROWS_WITH(poc::as_unique_ptr<int>(hld), "Cannot disown nullptr (as_unique_ptr).");
+}
+
+TEST_CASE("indestructible_int-from_raw_ptr_unowned+as_raw_ptr_unowned", "[S]") {
+    using zombie = helpers::indestructible_int;
+    // Using placement new instead of plain new, to not trigger leak sanitizer errors.
+    static std::aligned_storage<sizeof(zombie), alignof(zombie)>::type memory_block[1];
+    auto *value = new (memory_block) zombie(19);
+    auto hld = smart_holder::from_raw_ptr_unowned(value);
+    REQUIRE(hld.as_raw_ptr_unowned<zombie>()->valu == 19);
+}
+
+TEST_CASE("indestructible_int-from_raw_ptr_take_ownership", "[E]") {
+    helpers::indestructible_int *value = nullptr;
+    REQUIRE_THROWS_WITH(smart_holder::from_raw_ptr_take_ownership(value),
+                        "Pointee is not destructible (from_raw_ptr_take_ownership).");
+}
+
+TEST_CASE("from_raw_ptr_take_ownership+as_shared_ptr-outliving_smart_holder", "[S]") {
+    // Exercises guarded_builtin_delete flag_ptr validity past destruction of smart_holder.
+    std::shared_ptr<int> longer_living;
+    {
+        auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19));
+        longer_living = hld.as_shared_ptr<int>();
+    }
+    REQUIRE(*longer_living == 19);
+}
+
+TEST_CASE("from_unique_ptr_with_deleter+as_shared_ptr-outliving_smart_holder", "[S]") {
+    // Exercises guarded_custom_deleter flag_ptr validity past destruction of smart_holder.
+    std::shared_ptr<int> longer_living;
+    {
+        std::unique_ptr<int, helpers::functor_builtin_delete<int>> orig_owner(new int(19));
+        auto hld = smart_holder::from_unique_ptr(std::move(orig_owner));
+        longer_living = hld.as_shared_ptr<int>();
+    }
+    REQUIRE(*longer_living == 19);
+}

pybind11/tests/pybind11_tests.cpp

@@ -128,4 +128,9 @@ PYBIND11_MODULE(pybind11_tests, m, py::mod_gil_not_used()) {
     for (const auto &initializer : initializers()) {
         initializer(m);
     }
+
+    py::class_<TestContext>(m, "TestContext")
+        .def(py::init<>(&TestContext::createNewContextForInit))
+        .def("__enter__", &TestContext::contextEnter)
+        .def("__exit__", &TestContext::contextExit);
 }

pybind11/tests/pybind11_tests.h

@@ -96,3 +96,24 @@ void ignoreOldStyleInitWarnings(F &&body) {
     )",
              py::dict(py::arg("body") = py::cpp_function(body)));
 }
+
+// See PR #5419 for background.
+class TestContext {
+public:
+    TestContext() = delete;
+    TestContext(const TestContext &) = delete;
+    TestContext(TestContext &&) = delete;
+    static TestContext *createNewContextForInit() { return new TestContext("new-context"); }
+
+    pybind11::object contextEnter() {
+        py::object contextObj = py::cast(*this);
+        return contextObj;
+    }
+    void contextExit(const pybind11::object & /*excType*/,
+                     const pybind11::object & /*excVal*/,
+                     const pybind11::object & /*excTb*/) {}
+
+private:
+    explicit TestContext(const std::string &context) : context(context) {}
+    std::string context;
+};

pybind11/tests/pyproject.toml

@@ -0,0 +1,17 @@
+# Warning: this is currently used for pyodide, and is not a general out-of-tree
+# builder for the tests (yet). Specifically, wheels can't be built from SDists.
+
+[build-system]
+requires = ["scikit-build-core"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "pybind11_tests"
+version = "0.0.1"
+dependencies = ["pytest", "pytest-timeout", "numpy", "scipy"]
+
+[tool.scikit-build.cmake.define]
+PYBIND11_FINDPYTHON = true
+
+[tool.cibuildwheel]
+test-command = "pytest -o timeout=0 -p no:cacheprovider {project}/tests/test_*.py"

pybind11/tests/requirements.txt

@@ -1,13 +1,13 @@
 --only-binary=:all:
-build~=1.0; python_version>="3.7"
-numpy~=1.20.0; python_version=="3.7" and platform_python_implementation=="PyPy"
+build~=1.0; python_version>="3.8"
 numpy~=1.23.0; python_version=="3.8" and platform_python_implementation=="PyPy"
 numpy~=1.25.0; python_version=="3.9" and platform_python_implementation=='PyPy'
-numpy~=1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10"
-numpy~=1.22.2; platform_python_implementation!="PyPy" and python_version=="3.10"
-numpy~=1.26.0; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13"
+numpy~=1.26.0; platform_python_implementation=="GraalVM" and sys_platform=="linux"
+numpy~=1.21.5; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.8" and python_version<"3.10"
+numpy~=1.22.2; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10"
+numpy~=1.26.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13"
 pytest~=7.0
 pytest-timeout
-scipy~=1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10"
-scipy~=1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" and sys_platform!='win32'
-scipy~=1.11.1; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32'
+scipy~=1.5.4; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version<"3.10"
+scipy~=1.8.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10" and sys_platform!='win32'
+scipy~=1.11.1; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32'

pybind11/tests/test_async.py

@@ -1,12 +1,17 @@
 from __future__ import annotations
 
+import sys
+
 import pytest
 
 asyncio = pytest.importorskip("asyncio")
 m = pytest.importorskip("pybind11_tests.async_module")
 
+if sys.platform.startswith("emscripten"):
+    pytest.skip("Can't run a new event_loop in pyodide", allow_module_level=True)
 
-@pytest.fixture()
+
+@pytest.fixture
 def event_loop():
     loop = asyncio.new_event_loop()
     yield loop

pybind11/tests/test_buffers.cpp

@@ -167,6 +167,137 @@ TEST_SUBMODULE(buffers, m) {
                  sizeof(float)});
         });
 
+    // A matrix that uses Fortran storage order.
+    class FortranMatrix : public Matrix {
+    public:
+        FortranMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(cols, rows) {
+            print_created(this,
+                          std::to_string(rows) + "x" + std::to_string(cols) + " Fortran matrix");
+        }
+
+        float operator()(py::ssize_t i, py::ssize_t j) const { return Matrix::operator()(j, i); }
+
+        float &operator()(py::ssize_t i, py::ssize_t j) { return Matrix::operator()(j, i); }
+
+        using Matrix::data;
+
+        py::ssize_t rows() const { return Matrix::cols(); }
+        py::ssize_t cols() const { return Matrix::rows(); }
+    };
+    py::class_<FortranMatrix, Matrix>(m, "FortranMatrix", py::buffer_protocol())
+        .def(py::init<py::ssize_t, py::ssize_t>())
+
+        .def("rows", &FortranMatrix::rows)
+        .def("cols", &FortranMatrix::cols)
+
+        /// Bare bones interface
+        .def("__getitem__",
+             [](const FortranMatrix &m, std::pair<py::ssize_t, py::ssize_t> i) {
+                 if (i.first >= m.rows() || i.second >= m.cols()) {
+                     throw py::index_error();
+                 }
+                 return m(i.first, i.second);
+             })
+        .def("__setitem__",
+             [](FortranMatrix &m, std::pair<py::ssize_t, py::ssize_t> i, float v) {
+                 if (i.first >= m.rows() || i.second >= m.cols()) {
+                     throw py::index_error();
+                 }
+                 m(i.first, i.second) = v;
+             })
+        /// Provide buffer access
+        .def_buffer([](FortranMatrix &m) -> py::buffer_info {
+            return py::buffer_info(m.data(),             /* Pointer to buffer */
+                                   {m.rows(), m.cols()}, /* Buffer dimensions */
+                                   /* Strides (in bytes) for each index */
+                                   {sizeof(float), sizeof(float) * size_t(m.rows())});
+        });
+
+    // A matrix that uses a discontiguous underlying memory block.
+    class DiscontiguousMatrix : public Matrix {
+    public:
+        DiscontiguousMatrix(py::ssize_t rows,
+                            py::ssize_t cols,
+                            py::ssize_t row_factor,
+                            py::ssize_t col_factor)
+            : Matrix(rows * row_factor, cols * col_factor), m_row_factor(row_factor),
+              m_col_factor(col_factor) {
+            print_created(this,
+                          std::to_string(rows) + "(*" + std::to_string(row_factor) + ")x"
+                              + std::to_string(cols) + "(*" + std::to_string(col_factor)
+                              + ") matrix");
+        }
+
+        ~DiscontiguousMatrix() {
+            print_destroyed(this,
+                            std::to_string(rows() / m_row_factor) + "(*"
+                                + std::to_string(m_row_factor) + ")x"
+                                + std::to_string(cols() / m_col_factor) + "(*"
+                                + std::to_string(m_col_factor) + ") matrix");
+        }
+
+        float operator()(py::ssize_t i, py::ssize_t j) const {
+            return Matrix::operator()(i * m_row_factor, j * m_col_factor);
+        }
+
+        float &operator()(py::ssize_t i, py::ssize_t j) {
+            return Matrix::operator()(i * m_row_factor, j * m_col_factor);
+        }
+
+        using Matrix::data;
+
+        py::ssize_t rows() const { return Matrix::rows() / m_row_factor; }
+        py::ssize_t cols() const { return Matrix::cols() / m_col_factor; }
+        py::ssize_t row_factor() const { return m_row_factor; }
+        py::ssize_t col_factor() const { return m_col_factor; }
+
+    private:
+        py::ssize_t m_row_factor;
+        py::ssize_t m_col_factor;
+    };
+    py::class_<DiscontiguousMatrix, Matrix>(m, "DiscontiguousMatrix", py::buffer_protocol())
+        .def(py::init<py::ssize_t, py::ssize_t, py::ssize_t, py::ssize_t>())
+
+        .def("rows", &DiscontiguousMatrix::rows)
+        .def("cols", &DiscontiguousMatrix::cols)
+
+        /// Bare bones interface
+        .def("__getitem__",
+             [](const DiscontiguousMatrix &m, std::pair<py::ssize_t, py::ssize_t> i) {
+                 if (i.first >= m.rows() || i.second >= m.cols()) {
+                     throw py::index_error();
+                 }
+                 return m(i.first, i.second);
+             })
+        .def("__setitem__",
+             [](DiscontiguousMatrix &m, std::pair<py::ssize_t, py::ssize_t> i, float v) {
+                 if (i.first >= m.rows() || i.second >= m.cols()) {
+                     throw py::index_error();
+                 }
+                 m(i.first, i.second) = v;
+             })
+        /// Provide buffer access
+        .def_buffer([](DiscontiguousMatrix &m) -> py::buffer_info {
+            return py::buffer_info(m.data(),             /* Pointer to buffer */
+                                   {m.rows(), m.cols()}, /* Buffer dimensions */
+                                   /* Strides (in bytes) for each index */
+                                   {size_t(m.col_factor()) * sizeof(float) * size_t(m.cols())
+                                        * size_t(m.row_factor()),
+                                    size_t(m.col_factor()) * sizeof(float)});
+        });
+
+    class BrokenMatrix : public Matrix {
+    public:
+        BrokenMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(rows, cols) {}
+        void throw_runtime_error() { throw std::runtime_error("See PR #5324 for context."); }
+    };
+    py::class_<BrokenMatrix>(m, "BrokenMatrix", py::buffer_protocol())
+        .def(py::init<py::ssize_t, py::ssize_t>())
+        .def_buffer([](BrokenMatrix &m) {
+            m.throw_runtime_error();
+            return py::buffer_info();
+        });
+
     // test_inherited_protocol
     class SquareMatrix : public Matrix {
     public:
@@ -256,4 +387,56 @@ TEST_SUBMODULE(buffers, m) {
         });
 
     m.def("get_buffer_info", [](const py::buffer &buffer) { return buffer.request(); });
+
+    // Expose Py_buffer for testing.
+    m.attr("PyBUF_FORMAT") = PyBUF_FORMAT;
+    m.attr("PyBUF_SIMPLE") = PyBUF_SIMPLE;
+    m.attr("PyBUF_ND") = PyBUF_ND;
+    m.attr("PyBUF_STRIDES") = PyBUF_STRIDES;
+    m.attr("PyBUF_INDIRECT") = PyBUF_INDIRECT;
+    m.attr("PyBUF_C_CONTIGUOUS") = PyBUF_C_CONTIGUOUS;
+    m.attr("PyBUF_F_CONTIGUOUS") = PyBUF_F_CONTIGUOUS;
+    m.attr("PyBUF_ANY_CONTIGUOUS") = PyBUF_ANY_CONTIGUOUS;
+
+    m.def("get_py_buffer", [](const py::object &object, int flags) {
+        Py_buffer buffer;
+        memset(&buffer, 0, sizeof(Py_buffer));
+        if (PyObject_GetBuffer(object.ptr(), &buffer, flags) == -1) {
+            throw py::error_already_set();
+        }
+
+        auto SimpleNamespace = py::module_::import("types").attr("SimpleNamespace");
+        py::object result = SimpleNamespace("len"_a = buffer.len,
+                                            "readonly"_a = buffer.readonly,
+                                            "itemsize"_a = buffer.itemsize,
+                                            "format"_a = buffer.format,
+                                            "ndim"_a = buffer.ndim,
+                                            "shape"_a = py::none(),
+                                            "strides"_a = py::none(),
+                                            "suboffsets"_a = py::none());
+        if (buffer.shape != nullptr) {
+            py::list l;
+            for (auto i = 0; i < buffer.ndim; i++) {
+                l.append(buffer.shape[i]);
+            }
+            py::setattr(result, "shape", l);
+        }
+        if (buffer.strides != nullptr) {
+            py::list l;
+            for (auto i = 0; i < buffer.ndim; i++) {
+                l.append(buffer.strides[i]);
+            }
+            py::setattr(result, "strides", l);
+        }
+        if (buffer.suboffsets != nullptr) {
+            py::list l;
+            for (auto i = 0; i < buffer.ndim; i++) {
+                l.append(buffer.suboffsets[i]);
+            }
+            py::setattr(result, "suboffsets", l);
+        }
+
+        PyBuffer_Release(&buffer);
+        return result;
+    });
 }

pybind11/tests/test_buffers.py

@@ -82,6 +82,8 @@ def test_from_python():
         for j in range(m4.cols()):
             assert m3[i, j] == m4[i, j]
 
+    if env.GRAALPY:
+        pytest.skip("ConstructorStats is incompatible with GraalPy.")
     cstats = ConstructorStats.get(m.Matrix)
     assert cstats.alive() == 1
     del m3, m4
@@ -118,6 +120,8 @@ def test_to_python():
     mat2[2, 3] = 5
     assert mat2[2, 3] == 5
 
+    if env.GRAALPY:
+        pytest.skip("ConstructorStats is incompatible with GraalPy.")
     cstats = ConstructorStats.get(m.Matrix)
     assert cstats.alive() == 1
     del mat
@@ -228,3 +232,170 @@ def test_buffer_docstring():
         m.get_buffer_info.__doc__.strip()
         == "get_buffer_info(arg0: Buffer) -> pybind11_tests.buffers.buffer_info"
     )
+
+
+def test_buffer_exception():
+    with pytest.raises(BufferError, match="Error getting buffer") as excinfo:
+        memoryview(m.BrokenMatrix(1, 1))
+    assert isinstance(excinfo.value.__cause__, RuntimeError)
+    assert "for context" in str(excinfo.value.__cause__)
+
+
+@pytest.mark.parametrize("type", ["pybind11", "numpy"])
+def test_c_contiguous_to_pybuffer(type):
+    if type == "pybind11":
+        mat = m.Matrix(5, 4)
+    elif type == "numpy":
+        mat = np.empty((5, 4), dtype=np.float32)
+    else:
+        raise ValueError(f"Unknown parametrization {type}")
+
+    info = m.get_py_buffer(mat, m.PyBUF_SIMPLE)
+    assert info.format is None
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 0  # See discussion on PR #5407.
+    assert info.shape is None
+    assert info.strides is None
+    assert info.suboffsets is None
+    assert not info.readonly
+    info = m.get_py_buffer(mat, m.PyBUF_SIMPLE | m.PyBUF_FORMAT)
+    assert info.format == "f"
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 0  # See discussion on PR #5407.
+    assert info.shape is None
+    assert info.strides is None
+    assert info.suboffsets is None
+    assert not info.readonly
+    info = m.get_py_buffer(mat, m.PyBUF_ND)
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 2
+    assert info.shape == [5, 4]
+    assert info.strides is None
+    assert info.suboffsets is None
+    assert not info.readonly
+    info = m.get_py_buffer(mat, m.PyBUF_STRIDES)
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 2
+    assert info.shape == [5, 4]
+    assert info.strides == [4 * info.itemsize, info.itemsize]
+    assert info.suboffsets is None
+    assert not info.readonly
+    info = m.get_py_buffer(mat, m.PyBUF_INDIRECT)
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 2
+    assert info.shape == [5, 4]
+    assert info.strides == [4 * info.itemsize, info.itemsize]
+    assert info.suboffsets is None  # Should be filled in here, but we don't use it.
+    assert not info.readonly
+
+
+@pytest.mark.parametrize("type", ["pybind11", "numpy"])
+def test_fortran_contiguous_to_pybuffer(type):
+    if type == "pybind11":
+        mat = m.FortranMatrix(5, 4)
+    elif type == "numpy":
+        mat = np.empty((5, 4), dtype=np.float32, order="F")
+    else:
+        raise ValueError(f"Unknown parametrization {type}")
+
+    # A Fortran-shaped buffer can only be accessed at PyBUF_STRIDES level or higher.
+    info = m.get_py_buffer(mat, m.PyBUF_STRIDES)
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 2
+    assert info.shape == [5, 4]
+    assert info.strides == [info.itemsize, 5 * info.itemsize]
+    assert info.suboffsets is None
+    assert not info.readonly
+    info = m.get_py_buffer(mat, m.PyBUF_INDIRECT)
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 2
+    assert info.shape == [5, 4]
+    assert info.strides == [info.itemsize, 5 * info.itemsize]
+    assert info.suboffsets is None  # Should be filled in here, but we don't use it.
+    assert not info.readonly
+
+
+@pytest.mark.parametrize("type", ["pybind11", "numpy"])
+def test_discontiguous_to_pybuffer(type):
+    if type == "pybind11":
+        mat = m.DiscontiguousMatrix(5, 4, 2, 3)
+    elif type == "numpy":
+        mat = np.empty((5 * 2, 4 * 3), dtype=np.float32)[::2, ::3]
+    else:
+        raise ValueError(f"Unknown parametrization {type}")
+
+    info = m.get_py_buffer(mat, m.PyBUF_STRIDES)
+    assert info.itemsize == ctypes.sizeof(ctypes.c_float)
+    assert info.len == 5 * 4 * info.itemsize
+    assert info.ndim == 2
+    assert info.shape == [5, 4]
+    assert info.strides == [2 * 4 * 3 * info.itemsize, 3 * info.itemsize]
+    assert info.suboffsets is None
+    assert not info.readonly
+
+
+@pytest.mark.parametrize("type", ["pybind11", "numpy"])
+def test_to_pybuffer_contiguity(type):
+    def check_strides(mat):
+        # The full block is memset to 0, so fill it with non-zero in real spots.
+        expected = np.arange(1, 5 * 4 + 1).reshape((5, 4))
+        for i in range(5):
+            for j in range(4):
+                mat[i, j] = expected[i, j]
+        # If all strides are correct, the exposed buffer should match the input.
+        np.testing.assert_array_equal(np.array(mat), expected)
+
+    if type == "pybind11":
+        cmat = m.Matrix(5, 4)  # C contiguous.
+        fmat = m.FortranMatrix(5, 4)  # Fortran contiguous.
+        dmat = m.DiscontiguousMatrix(5, 4, 2, 3)  # Not contiguous.
+        expected_exception = BufferError
+    elif type == "numpy":
+        cmat = np.empty((5, 4), dtype=np.float32)  # C contiguous.
+        fmat = np.empty((5, 4), dtype=np.float32, order="F")  # Fortran contiguous.
+        dmat = np.empty((5 * 2, 4 * 3), dtype=np.float32)[::2, ::3]  # Not contiguous.
+        # NumPy incorrectly raises ValueError; when the minimum NumPy requirement is
+        # above the version that fixes https://github.com/numpy/numpy/issues/3634 then
+        # BufferError can be used everywhere.
+        expected_exception = (BufferError, ValueError)
+    else:
+        raise ValueError(f"Unknown parametrization {type}")
+
+    check_strides(cmat)
+    # Should work in C-contiguous mode, but not Fortran order.
+    m.get_py_buffer(cmat, m.PyBUF_C_CONTIGUOUS)
+    m.get_py_buffer(cmat, m.PyBUF_ANY_CONTIGUOUS)
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(cmat, m.PyBUF_F_CONTIGUOUS)
+
+    check_strides(fmat)
+    # These flags imply C-contiguity, so won't work.
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(fmat, m.PyBUF_SIMPLE)
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(fmat, m.PyBUF_ND)
+    # Should work in Fortran-contiguous mode, but not C order.
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(fmat, m.PyBUF_C_CONTIGUOUS)
+    m.get_py_buffer(fmat, m.PyBUF_ANY_CONTIGUOUS)
+    m.get_py_buffer(fmat, m.PyBUF_F_CONTIGUOUS)
+
+    check_strides(dmat)
+    # Should never work.
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(dmat, m.PyBUF_SIMPLE)
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(dmat, m.PyBUF_ND)
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(dmat, m.PyBUF_C_CONTIGUOUS)
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(dmat, m.PyBUF_ANY_CONTIGUOUS)
+    with pytest.raises(expected_exception):
+        m.get_py_buffer(dmat, m.PyBUF_F_CONTIGUOUS)