Merge pull request #2029 from yambati03/develop

Add `cibuildwheel` workflow

.github/workflows/build-cibw.yml

@@ -0,0 +1,175 @@
+# This workflow builds the Python wheels using cibuildwheel and uploads them to TestPyPI.
+# It can be triggered on push to the develop branch or manually via Github Actions.
+
+name: Build Wheels (cibuildwheel)
+
+on:
+  push:
+    branches:
+      - develop
+  workflow_dispatch:
+
+jobs:
+  # Get the system time and store it in an output. This is used to tag the wheels.
+  # This needs to be done in a separate job so that each matrix job in build_wheels can
+  # access the same timestamp.
+  get_system_time:
+    name: Get System Time
+    runs-on: ubuntu-latest
+    outputs:
+      timestamp: ${{ steps.get_time.outputs.timestamp }}
+    steps:
+      - name: Get system time
+        id: get_time
+        run: echo "timestamp=$(date +'%Y%m%d%H%M')" >> "$GITHUB_OUTPUT"
+
+  build_wheels:
+    name: Build Wheels
+    needs: get_system_time
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Linux x86_64
+          - os: ubuntu-latest
+            python_version: "3.10"
+            cibw_python_version: 310
+            platform_id: manylinux_x86_64
+            manylinux_image: manylinux2014
+          - os: ubuntu-latest
+            python_version: "3.11"
+            cibw_python_version: 311
+            platform_id: manylinux_x86_64
+            manylinux_image: manylinux2014
+          - os: ubuntu-latest
+            python_version: "3.12"
+            cibw_python_version: 312
+            platform_id: manylinux_x86_64
+            manylinux_image: manylinux2014
+          - os: ubuntu-latest
+            python_version: "3.13"
+            cibw_python_version: 313
+            platform_id: manylinux_x86_64
+            manylinux_image: manylinux2014
+
+          # Linux aarch64
+          - os: ubuntu-24.04-arm
+            python_version: "3.10"
+            cibw_python_version: 310
+            platform_id: manylinux_aarch64
+            manylinux_image: manylinux2014
+          - os: ubuntu-24.04-arm
+            python_version: "3.11"
+            cibw_python_version: 311
+            platform_id: manylinux_aarch64
+            manylinux_image: manylinux2014
+          - os: ubuntu-24.04-arm
+            python_version: "3.12"
+            cibw_python_version: 312
+            platform_id: manylinux_aarch64
+            manylinux_image: manylinux2014
+          - os: ubuntu-24.04-arm
+            python_version: "3.13"
+            cibw_python_version: 313
+            platform_id: manylinux_aarch64
+            manylinux_image: manylinux2014
+
+          # MacOS x86_64
+          # - os: macos-13
+          #   python_version: "3.10"
+          #   cibw_python_version: 310
+          #   platform_id: macosx_x86_64
+          # - os: macos-13
+          #   python_version: "3.11"
+          #   cibw_python_version: 311
+          #   platform_id: macosx_x86_64
+          # - os: macos-13
+          #   python_version: "3.12"
+          #   cibw_python_version: 312
+          #   platform_id: macosx_x86_64
+          # - os: macos-13
+          #   python_version: "3.13"
+          #   cibw_python_version: 313
+          #   platform_id: macosx_x86_64
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python_version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+
+      # Set the DEVELOP flag and the TIMESTAMP environment variables. This is used in the
+      # top-level CMakeLists.txt to generate the GTSAM_VERSION_STRING.
+      - name: Set Develop Flag
+        run: |
+          echo "DEVELOP=1" >> $GITHUB_ENV
+          echo "TIMESTAMP=${{ needs.get_system_time.outputs.timestamp }}" >> $GITHUB_ENV
+
+      - name: Install Dependencies
+        run: |
+          python3 -m pip install -r python/dev_requirements.txt
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build
+          elif [ "$RUNNER_OS" == "macOS" ]; then
+            brew install wget icu4c boost ninja python-setuptools
+          else
+            echo "$RUNNER_OS not supported"
+            exit 1
+          fi
+
+      # We first build the Python wrapper module on the host machine. This is done because cibuildwheel
+      # expects a setup.py file to be present in the project directory.
+      #
+      # The Python wrapper module is then rebuilt within the cibuildwheel container before building
+      # the wheels to ensure platform compatibility.
+      - name: Run CMake
+        run: |
+          cmake . -B build -DGTSAM_BUILD_PYTHON=1 -DGTSAM_PYTHON_VERSION=${{ matrix.python_version }}
+
+      - name: Build and test wheels
+        env:
+          # Generate the platform identifier. See https://cibuildwheel.pypa.io/en/stable/options/#build-skip.
+          CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }}
+          CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
+          CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_image }}
+          CIBW_ARCHS: all
+          CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP TIMESTAMP
+
+          # Use build instead of pip wheel to build the wheels. This is recommended by PyPA.
+          # See https://cibuildwheel.pypa.io/en/stable/options/#build-frontend.
+          CIBW_BUILD_FRONTEND: "build"
+          CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project}
+
+          CIBW_BUILD_VERBOSITY: 1
+
+        run: bash build_tools/wheels/build_wheels.sh
+
+      - name: Store artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }}
+          path: wheelhouse/*.whl
+
+  upload_all:
+    name: Upload All
+    needs: build_wheels
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verbose: true
+          packages-dir: dist/
+          # repository-url: https://test.pypi.org/legacy/

.gitignore

@@ -1,4 +1,4 @@
-/build*
+/build
 /debug*
 .idea
 *.pyc

CMakeLists.txt

@@ -10,10 +10,23 @@ set (GTSAM_VERSION_PATCH 0)
 set (GTSAM_PRERELEASE_VERSION "a0")
 math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}")
 
-if ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "")
+# Set the version string for the library. 
+#
+# If the environment variable DEVELOP is set, then the version string will be
+# "MAJOR.MINORprerelease.devTIMESTAMP". TIMESTAMP is another environment variable that should be set to the current
+# datetime. See build-cibw.yaml for example usage.
+#
+# If the prerelease version is empty, then the version string will be "MAJOR.MINOR.PATCH". Otherwise, the version
+# string will be "MAJOR.MINORprerelease".
+if (DEFINED ENV{DEVELOP})
+    set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.dev$ENV{TIMESTAMP}")
+    set (SETUP_NAME "gtsam-develop")
+elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "")
     set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}")
+    set (SETUP_NAME "gtsam")
 else()
     set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}")
+    set (SETUP_NAME "gtsam")
 endif()
 
 project(GTSAM

build_tools/wheels/build_wheels.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# This script calls cibuildwheel to build the wheels for the project. It is used in the build-cibw.yml workflow in .github/workflows.
+# Note that the build/python directory contains the wrapper module built for the specified Python version.
+
+set -e
+set -x
+
+python -m pip install cibuildwheel
+python -m cibuildwheel build/python --output-dir wheelhouse

build_tools/wheels/cibw_before_all.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# This script is invoked prior to building the wheels with cibuildwheel. It is used in the build-cibw.yml workflow in .github/workflows.
+# It installs the necessary dependencies and builds the wrapper module for the specified Python version.
+
+set -e
+set -x
+
+PYTHON_VERSION="$1"
+PROJECT_DIR="$2"
+ARCH=$(uname -m)
+
+export PYTHON="python${PYTHON_VERSION}"
+
+if [ "$(uname)" == "Linux" ]; then
+    # manylinux2014 is based on CentOS 7, so use yum to install dependencies
+    yum install -y wget
+
+    # Install Boost from source
+    wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz --quiet
+    tar -xzf boost_1_87_0.tar.gz
+    cd boost_1_87_0
+    ./bootstrap.sh --prefix=/opt/boost
+    ./b2 install --prefix=/opt/boost --with=all 
+    cd ..
+elif [ "$(uname)" == "Darwin" ]; then
+    brew install wget cmake boost
+fi
+
+$(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt
+
+# Remove build/cache files that were generated on host
+rm -rf $PROJECT_DIR/build
+rm -rf CMakeCache.txt CMakeFiles
+
+# Build the Python wrapper module
+cmake $PROJECT_DIR \
+    -B build \
+    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+    -DGTSAM_BUILD_TESTS=OFF \
+    -DGTSAM_BUILD_UNSTABLE=${GTSAM_BUILD_UNSTABLE:-ON} \
+    -DGTSAM_USE_QUATERNIONS=OFF \
+    -DGTSAM_WITH_TBB=${GTSAM_WITH_TBB:-OFF} \
+    -DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF \
+    -DGTSAM_BUILD_WITH_MARCH_NATIVE=OFF \
+    -DGTSAM_BUILD_PYTHON=ON \
+    -DGTSAM_UNSTABLE_BUILD_PYTHON=${GTSAM_BUILD_UNSTABLE:-ON} \
+    -DGTSAM_PYTHON_VERSION=$PYTHON_VERSION \
+    -DPYTHON_EXECUTABLE:FILEPATH=$(which $PYTHON) \
+    -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \
+    -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install
+
+cd $PROJECT_DIR/build/python
+
+# Install the Python wrapper module and generate Python stubs
+if [ "$(uname)" == "Linux" ]; then
+    make -j $(nproc) install
+    make -j $(nproc) python-stubs
+elif [ "$(uname)" == "Darwin" ]; then
+    make -j $(sysctl -n hw.logicalcpu) install
+    make -j $(sysctl -n hw.logicalcpu) python-stubs
+fi
+

python/CMakeLists.txt

@@ -284,9 +284,9 @@ endif()
 add_custom_target(
         python-stubs
         COMMAND
-        ${CMAKE_COMMAND} -E env
+          ${CMAKE_COMMAND} -E env
-        "PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}"
+          "PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}"
-        pybind11-stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam
+        ${PYTHON_EXECUTABLE} -m pybind11_stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam
         DEPENDS ${GTSAM_PYTHON_DEPENDENCIES} ${GTSAM_PYTHON_TEST_FILES} ${GTSAM_PYTHON_TARGET}
         WORKING_DIRECTORY "${GTSAM_PYTHON_BUILD_DIRECTORY}/"
 )

python/setup.py.in

@@ -1,6 +1,6 @@
 """Setup file to install the GTSAM package."""
 
-from setuptools import setup, find_namespace_packages
+from setuptools import setup, find_namespace_packages, Distribution
 
 packages = find_namespace_packages(
     where=".",
@@ -20,8 +20,14 @@ package_data = {
 # Cleaner to read in the contents rather than copy them over.
 readme_contents = open("${GTSAM_SOURCE_DIR}/README.md").read()
 
+# The cibuildwheel tool won't recognize a wheel as platform-dependent unless the ext_modules option is defined in setup.py. This is used to define C/C++ source files that need to be built for the wheel.
+# However, we pre-build our C++ files. Thus, we force cibuildwheel to think that there are ext_modules defined by overwriting the has_ext_modules() function.
+class BinaryDistribution(Distribution):
+    def has_ext_modules(foo):
+        return True
+
 setup(
-    name='gtsam',
+    name='${SETUP_NAME}',
     description='Georgia Tech Smoothing And Mapping library',
     url='https://gtsam.org/',
     version='${GTSAM_VERSION_STRING}',  # https://www.python.org/dev/peps/pep-0440/
@@ -46,6 +52,7 @@ setup(
     packages=packages,
     include_package_data=True,
     package_data=package_data,
+    distclass=BinaryDistribution,
     test_suite="gtsam.tests",
     install_requires=open("${GTSAM_SOURCE_DIR}/python/requirements.txt").readlines(),
     zip_safe=False,