diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml new file mode 100644 index 000000000..987c1645c --- /dev/null +++ b/.github/workflows/build-cibw.yml @@ -0,0 +1,175 @@ +# This workflow builds the Python wheels using cibuildwheel and uploads them to TestPyPI. +# It can be triggered on push to the develop branch or manually via Github Actions. + +name: Build Wheels (cibuildwheel) + +on: + push: + branches: + - develop + workflow_dispatch: + +jobs: + # Get the system time and store it in an output. This is used to tag the wheels. + # This needs to be done in a separate job so that each matrix job in build_wheels can + # access the same timestamp. + get_system_time: + name: Get System Time + runs-on: ubuntu-latest + outputs: + timestamp: ${{ steps.get_time.outputs.timestamp }} + steps: + - name: Get system time + id: get_time + run: echo "timestamp=$(date +'%Y%m%d%H%M')" >> "$GITHUB_OUTPUT" + + build_wheels: + name: Build Wheels + needs: get_system_time + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Linux x86_64 + - os: ubuntu-latest + python_version: "3.10" + cibw_python_version: 310 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python_version: "3.11" + cibw_python_version: 311 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python_version: "3.12" + cibw_python_version: 312 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python_version: "3.13" + cibw_python_version: 313 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + + # Linux aarch64 + - os: ubuntu-24.04-arm + python_version: "3.10" + cibw_python_version: 310 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python_version: "3.11" + cibw_python_version: 311 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python_version: "3.12" + cibw_python_version: 312 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python_version: "3.13" + cibw_python_version: 313 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + + # MacOS x86_64 + # - os: macos-13 + # python_version: "3.10" + # cibw_python_version: 310 + # platform_id: macosx_x86_64 + # - os: macos-13 + # python_version: "3.11" + # cibw_python_version: 311 + # platform_id: macosx_x86_64 + # - os: macos-13 + # python_version: "3.12" + # cibw_python_version: 312 + # platform_id: macosx_x86_64 + # - os: macos-13 + # python_version: "3.13" + # cibw_python_version: 313 + # platform_id: macosx_x86_64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + # Set the DEVELOP flag and the TIMESTAMP environment variables. This is used in the + # top-level CMakeLists.txt to generate the GTSAM_VERSION_STRING. + - name: Set Develop Flag + run: | + echo "DEVELOP=1" >> $GITHUB_ENV + echo "TIMESTAMP=${{ needs.get_system_time.outputs.timestamp }}" >> $GITHUB_ENV + + - name: Install Dependencies + run: | + python3 -m pip install -r python/dev_requirements.txt + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build + elif [ "$RUNNER_OS" == "macOS" ]; then + brew install wget icu4c boost ninja python-setuptools + else + echo "$RUNNER_OS not supported" + exit 1 + fi + + # We first build the Python wrapper module on the host machine. This is done because cibuildwheel + # expects a setup.py file to be present in the project directory. + # + # The Python wrapper module is then rebuilt within the cibuildwheel container before building + # the wheels to ensure platform compatibility. + - name: Run CMake + run: | + cmake . -B build -DGTSAM_BUILD_PYTHON=1 -DGTSAM_PYTHON_VERSION=${{ matrix.python_version }} + + - name: Build and test wheels + env: + # Generate the platform identifier. See https://cibuildwheel.pypa.io/en/stable/options/#build-skip. + CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_ARCHS: all + CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP TIMESTAMP + + # Use build instead of pip wheel to build the wheels. This is recommended by PyPA. + # See https://cibuildwheel.pypa.io/en/stable/options/#build-frontend. + CIBW_BUILD_FRONTEND: "build" + CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project} + + CIBW_BUILD_VERBOSITY: 1 + + run: bash build_tools/wheels/build_wheels.sh + + - name: Store artifacts + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} + path: wheelhouse/*.whl + + upload_all: + name: Upload All + needs: build_wheels + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true + packages-dir: dist/ + # repository-url: https://test.pypi.org/legacy/ diff --git a/.gitignore b/.gitignore index 7496190b5..bca6a479f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/build* +/build /debug* .idea *.pyc diff --git a/CMakeLists.txt b/CMakeLists.txt index 262f38121..cabde7653 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,10 +10,23 @@ set (GTSAM_VERSION_PATCH 0) set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") -if ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") +# Set the version string for the library. +# +# If the environment variable DEVELOP is set, then the version string will be +# "MAJOR.MINORprerelease.devTIMESTAMP". TIMESTAMP is another environment variable that should be set to the current +# datetime. See build-cibw.yaml for example usage. +# +# If the prerelease version is empty, then the version string will be "MAJOR.MINOR.PATCH". Otherwise, the version +# string will be "MAJOR.MINORprerelease". +if (DEFINED ENV{DEVELOP}) + set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.dev$ENV{TIMESTAMP}") + set (SETUP_NAME "gtsam-develop") +elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}") + set (SETUP_NAME "gtsam") else() set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}") + set (SETUP_NAME "gtsam") endif() project(GTSAM diff --git a/build_tools/wheels/build_wheels.sh b/build_tools/wheels/build_wheels.sh new file mode 100644 index 000000000..29247953d --- /dev/null +++ b/build_tools/wheels/build_wheels.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# This script calls cibuildwheel to build the wheels for the project. It is used in the build-cibw.yml workflow in .github/workflows. +# Note that the build/python directory contains the wrapper module built for the specified Python version. + +set -e +set -x + +python -m pip install cibuildwheel +python -m cibuildwheel build/python --output-dir wheelhouse diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh new file mode 100644 index 000000000..2398877a8 --- /dev/null +++ b/build_tools/wheels/cibw_before_all.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# This script is invoked prior to building the wheels with cibuildwheel. It is used in the build-cibw.yml workflow in .github/workflows. +# It installs the necessary dependencies and builds the wrapper module for the specified Python version. + +set -e +set -x + +PYTHON_VERSION="$1" +PROJECT_DIR="$2" +ARCH=$(uname -m) + +export PYTHON="python${PYTHON_VERSION}" + +if [ "$(uname)" == "Linux" ]; then + # manylinux2014 is based on CentOS 7, so use yum to install dependencies + yum install -y wget + + # Install Boost from source + wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz --quiet + tar -xzf boost_1_87_0.tar.gz + cd boost_1_87_0 + ./bootstrap.sh --prefix=/opt/boost + ./b2 install --prefix=/opt/boost --with=all + cd .. +elif [ "$(uname)" == "Darwin" ]; then + brew install wget cmake boost +fi + +$(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt + +# Remove build/cache files that were generated on host +rm -rf $PROJECT_DIR/build +rm -rf CMakeCache.txt CMakeFiles + +# Build the Python wrapper module +cmake $PROJECT_DIR \ + -B build \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DGTSAM_BUILD_TESTS=OFF \ + -DGTSAM_BUILD_UNSTABLE=${GTSAM_BUILD_UNSTABLE:-ON} \ + -DGTSAM_USE_QUATERNIONS=OFF \ + -DGTSAM_WITH_TBB=${GTSAM_WITH_TBB:-OFF} \ + -DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF \ + -DGTSAM_BUILD_WITH_MARCH_NATIVE=OFF \ + -DGTSAM_BUILD_PYTHON=ON \ + -DGTSAM_UNSTABLE_BUILD_PYTHON=${GTSAM_BUILD_UNSTABLE:-ON} \ + -DGTSAM_PYTHON_VERSION=$PYTHON_VERSION \ + -DPYTHON_EXECUTABLE:FILEPATH=$(which $PYTHON) \ + -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \ + -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install + +cd $PROJECT_DIR/build/python + +# Install the Python wrapper module and generate Python stubs +if [ "$(uname)" == "Linux" ]; then + make -j $(nproc) install + make -j $(nproc) python-stubs +elif [ "$(uname)" == "Darwin" ]; then + make -j $(sysctl -n hw.logicalcpu) install + make -j $(sysctl -n hw.logicalcpu) python-stubs +fi + diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f0fc3f796..3237a9fa9 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -284,9 +284,9 @@ endif() add_custom_target( python-stubs COMMAND - ${CMAKE_COMMAND} -E env - "PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}" - pybind11-stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam + ${CMAKE_COMMAND} -E env + "PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}" + ${PYTHON_EXECUTABLE} -m pybind11_stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam DEPENDS ${GTSAM_PYTHON_DEPENDENCIES} ${GTSAM_PYTHON_TEST_FILES} ${GTSAM_PYTHON_TARGET} WORKING_DIRECTORY "${GTSAM_PYTHON_BUILD_DIRECTORY}/" ) diff --git a/python/setup.py.in b/python/setup.py.in index b9d7392c7..cd9177ab8 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,6 +1,6 @@ """Setup file to install the GTSAM package.""" -from setuptools import setup, find_namespace_packages +from setuptools import setup, find_namespace_packages, Distribution packages = find_namespace_packages( where=".", @@ -20,8 +20,14 @@ package_data = { # Cleaner to read in the contents rather than copy them over. readme_contents = open("${GTSAM_SOURCE_DIR}/README.md").read() +# The cibuildwheel tool won't recognize a wheel as platform-dependent unless the ext_modules option is defined in setup.py. This is used to define C/C++ source files that need to be built for the wheel. +# However, we pre-build our C++ files. Thus, we force cibuildwheel to think that there are ext_modules defined by overwriting the has_ext_modules() function. +class BinaryDistribution(Distribution): + def has_ext_modules(foo): + return True + setup( - name='gtsam', + name='${SETUP_NAME}', description='Georgia Tech Smoothing And Mapping library', url='https://gtsam.org/', version='${GTSAM_VERSION_STRING}', # https://www.python.org/dev/peps/pep-0440/ @@ -46,6 +52,7 @@ setup( packages=packages, include_package_data=True, package_data=package_data, + distclass=BinaryDistribution, test_suite="gtsam.tests", install_requires=open("${GTSAM_SOURCE_DIR}/python/requirements.txt").readlines(), zip_safe=False,