From ae433fbe033b5bb5fde7e5c811afe796e35d8e05 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 19 Feb 2025 22:51:52 -0500 Subject: [PATCH 01/31] Create initial workflow for cibuildwheel --- .github/workflows/build-cibw.yml | 53 +++++++++++++++++++++++++++ .gitignore | 2 +- build_tools/wheels/build_wheels.sh | 7 ++++ build_tools/wheels/cibw_before_all.sh | 28 ++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-cibw.yml create mode 100644 build_tools/wheels/build_wheels.sh create mode 100644 build_tools/wheels/cibw_before_all.sh diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml new file mode 100644 index 000000000..ef9603719 --- /dev/null +++ b/.github/workflows/build-cibw.yml @@ -0,0 +1,53 @@ +name: Build Wheels (cibuildwheel) + +on: [workflow_dispatch] + +jobs: + build_wheels: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python_version: "3.10" + cibw_python_version: 310 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + - name: Install Dependencies + run: | + sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build + python3 -m pip install -r python/dev_requirements.txt + + - name: Run CMake + run: | + cmake .. -DGTSAM_BUILD_PYTHON=1 -DGTSAM_PYTHON_VERSION=${{ matrix.python_version }} + + - name: Build and test wheels + env: + CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_ARCHS: all + + CIBW_BUILD_FRONTEND: "build" + CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project} + + CIBW_BUILD_VERBOSITY: 1 + + run: bash {project}/build_tools/wheels/build_wheels.sh + + - name: Store artifacts + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} + path: wheelhouse/*.whl diff --git a/.gitignore b/.gitignore index 7496190b5..bca6a479f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/build* +/build /debug* .idea *.pyc diff --git a/build_tools/wheels/build_wheels.sh b/build_tools/wheels/build_wheels.sh new file mode 100644 index 000000000..b8f0750a3 --- /dev/null +++ b/build_tools/wheels/build_wheels.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e +set -x + +python -m pip install cibuildwheel +python -m cibuildwheel build/python --output-dir wheelhouse \ No newline at end of file diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh new file mode 100644 index 000000000..0607a67ac --- /dev/null +++ b/build_tools/wheels/cibw_before_all.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e +set -x + +PYTHON_VERSION="$1" +PROJECT_DIR="$2" + +export PYTHON="python${PYTHON_VERSION}" + +rm -rf /build/python + +export CMAKE_GENERATOR=Ninja +cmake $PROJECT_DIR \ + -B build \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DGTSAM_BUILD_TESTS=OFF \ + -DGTSAM_BUILD_UNSTABLE=${GTSAM_BUILD_UNSTABLE:-ON} \ + -DGTSAM_USE_QUATERNIONS=OFF \ + -DGTSAM_WITH_TBB=${GTSAM_WITH_TBB:-OFF} \ + -DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF \ + -DGTSAM_BUILD_WITH_MARCH_NATIVE=OFF \ + -DGTSAM_BUILD_PYTHON=ON \ + -DGTSAM_UNSTABLE_BUILD_PYTHON=${GTSAM_BUILD_UNSTABLE:-ON} \ + -DGTSAM_PYTHON_VERSION=$PYTHON_VERSION \ + -DPYTHON_EXECUTABLE:FILEPATH=$(which $PYTHON) \ + -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \ + -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install \ No newline at end of file From 16a516c3f95726d3f01d222378c2846014bb2e64 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 19 Feb 2025 22:55:11 -0500 Subject: [PATCH 02/31] Update CMake command to point to right path --- .github/workflows/build-cibw.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index ef9603719..4e39d9d3f 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -31,7 +31,7 @@ jobs: - name: Run CMake run: | - cmake .. -DGTSAM_BUILD_PYTHON=1 -DGTSAM_PYTHON_VERSION=${{ matrix.python_version }} + cmake . -B build -DGTSAM_BUILD_PYTHON=1 -DGTSAM_PYTHON_VERSION=${{ matrix.python_version }} - name: Build and test wheels env: From 500808396a0eb734f9164733f25e369bb1196c58 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 19 Feb 2025 23:01:00 -0500 Subject: [PATCH 03/31] Update another bad path --- .github/workflows/build-cibw.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 4e39d9d3f..c6b6d609d 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -44,7 +44,7 @@ jobs: CIBW_BUILD_VERBOSITY: 1 - run: bash {project}/build_tools/wheels/build_wheels.sh + run: bash build_tools/wheels/build_wheels.sh - name: Store artifacts uses: actions/upload-artifact@v4 From 00b70e661a48337293b98507c924f30f00dda006 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 19 Feb 2025 23:08:26 -0500 Subject: [PATCH 04/31] Fix ANOTHER bad path --- build_tools/wheels/cibw_before_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 0607a67ac..72f87d626 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -8,7 +8,7 @@ PROJECT_DIR="$2" export PYTHON="python${PYTHON_VERSION}" -rm -rf /build/python +rm -rf $PROJECT_DIR/build export CMAKE_GENERATOR=Ninja cmake $PROJECT_DIR \ From 4a57824ea5dcf56595d9e95f6e3995ac575c69a6 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 19 Feb 2025 23:16:20 -0500 Subject: [PATCH 05/31] Install dependencies in manylinux container as well --- build_tools/wheels/cibw_before_all.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 72f87d626..d37db829f 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -8,6 +8,9 @@ PROJECT_DIR="$2" export PYTHON="python${PYTHON_VERSION}" +sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build +python -m pip install -r $PROJECT_DIR/python/dev_requirements.txt + rm -rf $PROJECT_DIR/build export CMAKE_GENERATOR=Ninja From 8cbb4fc154e5a0a4f3c707b1f22e675ff7855ce0 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 19 Feb 2025 23:34:14 -0500 Subject: [PATCH 06/31] Trying something --- build_tools/wheels/cibw_before_all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index d37db829f..392693624 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -8,8 +8,8 @@ PROJECT_DIR="$2" export PYTHON="python${PYTHON_VERSION}" -sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build -python -m pip install -r $PROJECT_DIR/python/dev_requirements.txt +apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build +pip install -r $PROJECT_DIR/python/dev_requirements.txt rm -rf $PROJECT_DIR/build From 5466a96de6c0d484cbaade79fe1d91c04b94bf9f Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 00:56:31 -0500 Subject: [PATCH 07/31] Install boost from source, delete CMakeCache before building --- build_tools/wheels/cibw_before_all.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 392693624..f922b46e6 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -8,10 +8,18 @@ PROJECT_DIR="$2" export PYTHON="python${PYTHON_VERSION}" -apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build +yum install -y wget ninja-build pip install -r $PROJECT_DIR/python/dev_requirements.txt +wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz +tar -xzf boost_1_87_0.tar.gz +cd boost_1_87_0 +./bootstrap.sh --prefix=/opt/boost +./b2 install --prefix=/opt/boost --with=all + +cd .. rm -rf $PROJECT_DIR/build +rm -rf CMakeCache.txt CMakeFiles export CMAKE_GENERATOR=Ninja cmake $PROJECT_DIR \ From 9c2ed3d7f0bb8f177a940c36dc691a890d11543c Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 10:08:24 -0500 Subject: [PATCH 08/31] Make shared libraries --- build_tools/wheels/cibw_before_all.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index f922b46e6..14c2665df 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -36,4 +36,7 @@ cmake $PROJECT_DIR \ -DGTSAM_PYTHON_VERSION=$PYTHON_VERSION \ -DPYTHON_EXECUTABLE:FILEPATH=$(which $PYTHON) \ -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \ - -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install \ No newline at end of file + -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install + +cd $PROJECT_DIR/build/python +make install From dc667c41220fca307227b8ffc1e890219262ebaa Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 10:35:33 -0500 Subject: [PATCH 09/31] Update make command --- build_tools/wheels/cibw_before_all.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 14c2665df..fb9da5734 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -11,12 +11,14 @@ export PYTHON="python${PYTHON_VERSION}" yum install -y wget ninja-build pip install -r $PROJECT_DIR/python/dev_requirements.txt +# Install Boost wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz tar -xzf boost_1_87_0.tar.gz cd boost_1_87_0 ./bootstrap.sh --prefix=/opt/boost ./b2 install --prefix=/opt/boost --with=all +# Remove build/cache files that were generated on host cd .. rm -rf $PROJECT_DIR/build rm -rf CMakeCache.txt CMakeFiles @@ -38,5 +40,4 @@ cmake $PROJECT_DIR \ -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \ -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install -cd $PROJECT_DIR/build/python -make install +make install -C $PROJECT_DIR/build From c4d7e622a99012441576b7eb400a7864ad0f55b4 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 11:02:23 -0500 Subject: [PATCH 10/31] Use CMake as generator --- build_tools/wheels/cibw_before_all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index fb9da5734..871ece45f 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -23,7 +23,6 @@ cd .. rm -rf $PROJECT_DIR/build rm -rf CMakeCache.txt CMakeFiles -export CMAKE_GENERATOR=Ninja cmake $PROJECT_DIR \ -B build \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ @@ -40,4 +39,5 @@ cmake $PROJECT_DIR \ -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \ -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install -make install -C $PROJECT_DIR/build +cd $PROJECT_DIR/build +make -j$(nproc) install From d0e25bc41b5cfa734d1d3ef6843fb5f0a564718a Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 16:05:06 -0500 Subject: [PATCH 11/31] Modify setup.py.in to force root to be platform-dependent --- build_tools/wheels/cibw_before_all.sh | 6 +++--- python/setup.py.in | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 871ece45f..789d65c5a 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -9,10 +9,10 @@ PROJECT_DIR="$2" export PYTHON="python${PYTHON_VERSION}" yum install -y wget ninja-build -pip install -r $PROJECT_DIR/python/dev_requirements.txt +$(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt # Install Boost -wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz +wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz --quiet tar -xzf boost_1_87_0.tar.gz cd boost_1_87_0 ./bootstrap.sh --prefix=/opt/boost @@ -39,5 +39,5 @@ cmake $PROJECT_DIR \ -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \ -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install -cd $PROJECT_DIR/build +cd $PROJECT_DIR/build/python make -j$(nproc) install diff --git a/python/setup.py.in b/python/setup.py.in index b9d7392c7..4be235ab0 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,6 +1,6 @@ """Setup file to install the GTSAM package.""" -from setuptools import setup, find_namespace_packages +from setuptools import setup, find_namespace_packages, Distribution packages = find_namespace_packages( where=".", @@ -20,6 +20,10 @@ package_data = { # Cleaner to read in the contents rather than copy them over. readme_contents = open("${GTSAM_SOURCE_DIR}/README.md").read() +class BinaryDistribution(Distribution): + def has_ext_modules(foo): + return True + setup( name='gtsam', description='Georgia Tech Smoothing And Mapping library', @@ -46,6 +50,7 @@ setup( packages=packages, include_package_data=True, package_data=package_data, + distclass=BinaryDistribution, test_suite="gtsam.tests", install_requires=open("${GTSAM_SOURCE_DIR}/python/requirements.txt").readlines(), zip_safe=False, From c17a3f80f18d5bf9adbe0927701dba8aa04474fa Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 16:48:06 -0500 Subject: [PATCH 12/31] Publish to TestPyPI after wheels are built --- .github/workflows/build-cibw.yml | 19 +++++++++++++++++++ build_tools/wheels/cibw_before_all.sh | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index c6b6d609d..b42e60731 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -51,3 +51,22 @@ jobs: with: name: cibw-wheels-cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} path: wheelhouse/*.whl + + upload_all: + name: Upload All + needs: build_wheels + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + repository-url: https://test.pypi.org/legacy/ diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 789d65c5a..5b0e71940 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -40,4 +40,4 @@ cmake $PROJECT_DIR \ -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install cd $PROJECT_DIR/build/python -make -j$(nproc) install +make -j $(nproc) install From 5da3691609731b539e6248ebe090dbbb29eb158f Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 17:28:42 -0500 Subject: [PATCH 13/31] Set package name to gtsam-nightly if nightly build --- .github/workflows/build-cibw.yml | 3 +++ CMakeLists.txt | 8 +++++++- python/setup.py.in | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index b42e60731..516cd0f12 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -24,6 +24,9 @@ jobs: with: python-version: ${{ matrix.python_version }} + - name: Set Nightly Flag + run: echo "NIGHTLY=1" >> $GITHUB_ENV + - name: Install Dependencies run: | sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build diff --git a/CMakeLists.txt b/CMakeLists.txt index 262f38121..e2bc9da5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,10 +10,16 @@ set (GTSAM_VERSION_PATCH 0) set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") -if ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") +if (DEFINED ENV{NIGHTLY}) + string(TIMESTAMP NOW "%Y.%m.%d.%H.%M") + set (GTSAM_VERSION_STRING "${NOW}") + set (SETUP_NAME "gtsam-nightly") +elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}") + set (SETUP_NAME "gtsam") else() set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}") + set (SETUP_NAME "gtsam") endif() project(GTSAM diff --git a/python/setup.py.in b/python/setup.py.in index 4be235ab0..96c21a55e 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -25,7 +25,7 @@ class BinaryDistribution(Distribution): return True setup( - name='gtsam', + name='${SETUP_NAME}', description='Georgia Tech Smoothing And Mapping library', url='https://gtsam.org/', version='${GTSAM_VERSION_STRING}', # https://www.python.org/dev/peps/pep-0440/ From abe4cd77273d805fb1a76610a327ffd8c4da0826 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Thu, 20 Feb 2025 18:06:30 -0500 Subject: [PATCH 14/31] Pass NIGHTLY env variable to cibw container --- .github/workflows/build-cibw.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 516cd0f12..e2b8919cb 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -41,6 +41,7 @@ jobs: CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_ARCHS: all + CIBW_ENVIRONMENT_PASS_LINUX: NIGHTLY CIBW_BUILD_FRONTEND: "build" CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project} From 408aaeafa44c9b90c08edb73d9d68d3d10873d51 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Fri, 21 Feb 2025 00:22:49 -0500 Subject: [PATCH 15/31] Address some comments on PR --- .github/workflows/build-cibw.yml | 8 +++++--- CMakeLists.txt | 7 +++---- build_tools/wheels/build_wheels.sh | 2 +- build_tools/wheels/cibw_before_all.sh | 3 +++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index e2b8919cb..018d22a60 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -24,8 +24,10 @@ jobs: with: python-version: ${{ matrix.python_version }} - - name: Set Nightly Flag - run: echo "NIGHTLY=1" >> $GITHUB_ENV + - name: Set Develop Flag + run: | + echo "DEVELOP=1" >> $GITHUB_ENV + echo "COMMIT_HASH=$(git rev-parse --short=7 "$GITHUB_SHA")" >> $GITHUB_ENV - name: Install Dependencies run: | @@ -41,7 +43,7 @@ jobs: CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_ARCHS: all - CIBW_ENVIRONMENT_PASS_LINUX: NIGHTLY + CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP COMMIT_HASH CIBW_BUILD_FRONTEND: "build" CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project} diff --git a/CMakeLists.txt b/CMakeLists.txt index e2bc9da5e..f1070bfc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,10 +10,9 @@ set (GTSAM_VERSION_PATCH 0) set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") -if (DEFINED ENV{NIGHTLY}) - string(TIMESTAMP NOW "%Y.%m.%d.%H.%M") - set (GTSAM_VERSION_STRING "${NOW}") - set (SETUP_NAME "gtsam-nightly") +if (DEFINED ENV{DEVELOP}) + set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.$ENV{COMMIT_HASH}") + set (SETUP_NAME "gtsam-develop") elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}") set (SETUP_NAME "gtsam") diff --git a/build_tools/wheels/build_wheels.sh b/build_tools/wheels/build_wheels.sh index b8f0750a3..a8cbb93de 100644 --- a/build_tools/wheels/build_wheels.sh +++ b/build_tools/wheels/build_wheels.sh @@ -4,4 +4,4 @@ set -e set -x python -m pip install cibuildwheel -python -m cibuildwheel build/python --output-dir wheelhouse \ No newline at end of file +python -m cibuildwheel build/python --output-dir wheelhouse diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 5b0e71940..2659ff1b6 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This script is invoked prior to building the wheels with cibuildwheel. It is used in the build-cibw.yml workflow in .github/workflows. +# It installs the necessary dependencies and builds the wrapper module for the specified Python version. + set -e set -x From 15e1926ccfc72108882f8be7d33ca9dd51bca6a4 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Fri, 21 Feb 2025 01:04:23 -0500 Subject: [PATCH 16/31] Make version string PEP440 compliant --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f1070bfc6..a8724632b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") if (DEFINED ENV{DEVELOP}) - set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.$ENV{COMMIT_HASH}") + set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}+$ENV{COMMIT_HASH}") set (SETUP_NAME "gtsam-develop") elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}") From a3e42a823b2056e7dde6dc17bf3958c4a3ac391d Mon Sep 17 00:00:00 2001 From: Yashas Ambati <32080306+yambati03@users.noreply.github.com> Date: Fri, 21 Feb 2025 10:13:48 -0500 Subject: [PATCH 17/31] Make upload to PyPI verbose --- .github/workflows/build-cibw.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 018d22a60..90cb52254 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -74,5 +74,6 @@ jobs: - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: + verbose: true packages-dir: dist/ repository-url: https://test.pypi.org/legacy/ From be9ea30a39932aa0e0cefb165c5c03ac0293feac Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Tue, 25 Feb 2025 17:24:57 -0500 Subject: [PATCH 18/31] New version string for gtsam-develop --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a8724632b..91b51fc40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,9 @@ set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") if (DEFINED ENV{DEVELOP}) - set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}+$ENV{COMMIT_HASH}") + string(TIMESTAMP NOW "%Y%m%d%H%M") + # set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}+$ENV{COMMIT_HASH}") + set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.dev${NOW}") set (SETUP_NAME "gtsam-develop") elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}") From 590fae60ba267bcf39865442fbfd26a93fcc03a2 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Tue, 25 Feb 2025 17:31:08 -0500 Subject: [PATCH 19/31] Run workflow on push to develop --- .github/workflows/build-cibw.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 90cb52254..0e7bf6918 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -1,6 +1,10 @@ name: Build Wheels (cibuildwheel) -on: [workflow_dispatch] +on: + push: + branches: + - develop + workflow_dispatch: jobs: build_wheels: From 5243f1fe8202a0ab2a72cc3d5df99a13ae898bcb Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Tue, 25 Feb 2025 17:38:16 -0500 Subject: [PATCH 20/31] Add target for Linux ARM --- .github/workflows/build-cibw.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 0e7bf6918..86140a4f1 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -19,6 +19,12 @@ jobs: platform_id: manylinux_x86_64 manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python_version: "3.10" + cibw_python_version: 310 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + steps: - name: Checkout uses: actions/checkout@v4 @@ -46,6 +52,7 @@ jobs: env: CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_image }} CIBW_ARCHS: all CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP COMMIT_HASH From 190093f4048e5dec2e4b87ea6dbaa42c5aa37df2 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Tue, 25 Feb 2025 22:49:41 -0500 Subject: [PATCH 21/31] Add more python versions for Linux x86, do not install ninja for arm64 archs --- .github/workflows/build-cibw.yml | 17 +++++++++++++++++ build_tools/wheels/cibw_before_all.sh | 8 +++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 86140a4f1..46516e824 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -13,12 +13,29 @@ jobs: fail-fast: false matrix: include: + # Linux x86_64 - os: ubuntu-latest python_version: "3.10" cibw_python_version: 310 platform_id: manylinux_x86_64 manylinux_image: manylinux2014 + - os: ubuntu-latest + python_version: "3.11" + cibw_python_version: 311 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python_version: "3.12" + cibw_python_version: 312 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python_version: "3.13" + cibw_python_version: 313 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + # Linux aarch64 - os: ubuntu-24.04-arm python_version: "3.10" cibw_python_version: 310 diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 2659ff1b6..f91c13dce 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -11,7 +11,13 @@ PROJECT_DIR="$2" export PYTHON="python${PYTHON_VERSION}" -yum install -y wget ninja-build +ARCH=$(uname -m) +if [[ $ARCH == x86_64* ]]; then + yum install -y wget ninja-build +elif [[ $ARCH == arm* ]] || [[ $ARCH = aarch64 ]]; then + yum install -y wget +fi + $(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt # Install Boost From ba9a70bf1653e61a79887b6af38e9c4ea39cc3f9 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 00:04:08 -0500 Subject: [PATCH 22/31] Add MacOS targets, modify build script for Darwin, generate type annotations --- .github/workflows/build-cibw.yml | 18 ++++++++++++++++++ build_tools/wheels/cibw_before_all.sh | 23 ++++++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 46516e824..1c9e47f28 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -42,6 +42,24 @@ jobs: platform_id: manylinux_aarch64 manylinux_image: manylinux2014 + # MacOS x86_64 + - os: macos-13 + python_version: "3.10" + cibw_python_version: 310 + platform_id: macosx_x86_64 + # - os: macos-13 + # python_version: "3.11" + # cibw_python_version: 311 + # platform_id: macosx_x86_64 + # - os: macos-13 + # python_version: "3.12" + # cibw_python_version: 312 + # platform_id: macosx_x86_64 + # - os: macos-13 + # python_version: "3.13" + # cibw_python_version: 313 + # platform_id: macosx_x86_64 + steps: - name: Checkout uses: actions/checkout@v4 diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index f91c13dce..f5c6c7f0c 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -8,25 +8,25 @@ set -x PYTHON_VERSION="$1" PROJECT_DIR="$2" +ARCH=$(uname -m) export PYTHON="python${PYTHON_VERSION}" -ARCH=$(uname -m) -if [[ $ARCH == x86_64* ]]; then - yum install -y wget ninja-build -elif [[ $ARCH == arm* ]] || [[ $ARCH = aarch64 ]]; then +if [ "$(uname)" == "Linux" ]; then yum install -y wget + + # Install Boost from source + wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz --quiet + tar -xzf boost_1_87_0.tar.gz + cd boost_1_87_0 + ./bootstrap.sh --prefix=/opt/boost + ./b2 install --prefix=/opt/boost --with=all +elif [ "$(uname)" == "Darwin" ]; then + brew install wget cmake boost fi $(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt -# Install Boost -wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz --quiet -tar -xzf boost_1_87_0.tar.gz -cd boost_1_87_0 -./bootstrap.sh --prefix=/opt/boost -./b2 install --prefix=/opt/boost --with=all - # Remove build/cache files that were generated on host cd .. rm -rf $PROJECT_DIR/build @@ -50,3 +50,4 @@ cmake $PROJECT_DIR \ cd $PROJECT_DIR/build/python make -j $(nproc) install +make -j $(nproc) python-stubs From 3e00f98f023b8508a5e787fdfda9d53f0dab5b05 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 00:20:13 -0500 Subject: [PATCH 23/31] Fix install rules for macOS support --- .github/workflows/build-cibw.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 1c9e47f28..450b2fc61 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -76,8 +76,15 @@ jobs: - name: Install Dependencies run: | - sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build python3 -m pip install -r python/dev_requirements.txt + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build + elif [ "$RUNNER_OS" == "macOS" ]; then + brew install wget icu4c boost ninja python-setuptools + else + echo "$RUNNER_OS not supported" + exit 1 + fi - name: Run CMake run: | From 170b59a1e0155921da309981a6b23f78732194ef Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 09:49:15 -0500 Subject: [PATCH 24/31] Potential fix for python-stubs target file not found --- python/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f0fc3f796..3237a9fa9 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -284,9 +284,9 @@ endif() add_custom_target( python-stubs COMMAND - ${CMAKE_COMMAND} -E env - "PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}" - pybind11-stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam + ${CMAKE_COMMAND} -E env + "PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}" + ${PYTHON_EXECUTABLE} -m pybind11_stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam DEPENDS ${GTSAM_PYTHON_DEPENDENCIES} ${GTSAM_PYTHON_TEST_FILES} ${GTSAM_PYTHON_TARGET} WORKING_DIRECTORY "${GTSAM_PYTHON_BUILD_DIRECTORY}/" ) From 26ea15c40eeb65dfbf7d68418e188c05614dd402 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 09:59:55 -0500 Subject: [PATCH 25/31] Minor fix --- build_tools/wheels/cibw_before_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index f5c6c7f0c..1c1d13581 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -21,6 +21,7 @@ if [ "$(uname)" == "Linux" ]; then cd boost_1_87_0 ./bootstrap.sh --prefix=/opt/boost ./b2 install --prefix=/opt/boost --with=all + cd .. elif [ "$(uname)" == "Darwin" ]; then brew install wget cmake boost fi @@ -28,7 +29,6 @@ fi $(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt # Remove build/cache files that were generated on host -cd .. rm -rf $PROJECT_DIR/build rm -rf CMakeCache.txt CMakeFiles From 1cb51b81f40d7c67b14b2f603d8f6685cce31aad Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 10:38:00 -0500 Subject: [PATCH 26/31] Use sysctl to get number of CPUs if on MacOS --- build_tools/wheels/cibw_before_all.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 1c1d13581..4f0f33f16 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -49,5 +49,12 @@ cmake $PROJECT_DIR \ -DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install cd $PROJECT_DIR/build/python -make -j $(nproc) install -make -j $(nproc) python-stubs + +if [ "$(uname)" == "Linux" ]; then + make -j $(nproc) install + make -j $(nproc) python-stubs +elif [ "$(uname)" == "Darwin" ]; then + make -j $(sysctl -n hw.logicalcpu) install + make -j $(sysctl -n hw.logicalcpu) python-stubs +fi + From 741602d63a8aebd408b6fbbef6bd304ffad87ce9 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 13:46:15 -0500 Subject: [PATCH 27/31] Comment out MacOS targets for now --- .github/workflows/build-cibw.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 450b2fc61..273cc353c 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -43,10 +43,10 @@ jobs: manylinux_image: manylinux2014 # MacOS x86_64 - - os: macos-13 - python_version: "3.10" - cibw_python_version: 310 - platform_id: macosx_x86_64 + # - os: macos-13 + # python_version: "3.10" + # cibw_python_version: 310 + # platform_id: macosx_x86_64 # - os: macos-13 # python_version: "3.11" # cibw_python_version: 311 From b3b2a679a90da155130b2d38e932447e98db7db4 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 15:44:15 -0500 Subject: [PATCH 28/31] Use same system time for version string across jobs --- .github/workflows/build-cibw.yml | 14 ++++++++++++-- CMakeLists.txt | 4 +--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 273cc353c..19b7e65ce 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -7,7 +7,17 @@ on: workflow_dispatch: jobs: + get_system_time: + runs-on: ubuntu-latest + outputs: + timestamp: ${{ steps.get_time.outputs.timestamp }} + steps: + - name: Get system time + id: get_time + run: echo "timestamp=$(date +'%Y%m%d%H%M')" >> "$GITHUB_OUTPUT" + build_wheels: + needs: get_system_time runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -72,7 +82,7 @@ jobs: - name: Set Develop Flag run: | echo "DEVELOP=1" >> $GITHUB_ENV - echo "COMMIT_HASH=$(git rev-parse --short=7 "$GITHUB_SHA")" >> $GITHUB_ENV + echo "TIMESTAMP=${{ needs.get_system_time.outputs.timestamp }}" >> $GITHUB_ENV - name: Install Dependencies run: | @@ -96,7 +106,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_image }} CIBW_ARCHS: all - CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP COMMIT_HASH + CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP TIMESTAMP CIBW_BUILD_FRONTEND: "build" CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project} diff --git a/CMakeLists.txt b/CMakeLists.txt index 91b51fc40..826785bec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,9 +11,7 @@ set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") if (DEFINED ENV{DEVELOP}) - string(TIMESTAMP NOW "%Y%m%d%H%M") - # set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}+$ENV{COMMIT_HASH}") - set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.dev${NOW}") + set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.dev$ENV{TIMESTAMP}") set (SETUP_NAME "gtsam-develop") elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}") From ebbde3c3f709ab645d6ca41e6e9273d108b6529f Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Wed, 26 Feb 2025 19:23:30 -0500 Subject: [PATCH 29/31] Add all Python version for Linux aarch64, add names to stages --- .github/workflows/build-cibw.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index 19b7e65ce..dac64d19d 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -8,6 +8,7 @@ on: jobs: get_system_time: + name: Get System Time runs-on: ubuntu-latest outputs: timestamp: ${{ steps.get_time.outputs.timestamp }} @@ -17,6 +18,7 @@ jobs: run: echo "timestamp=$(date +'%Y%m%d%H%M')" >> "$GITHUB_OUTPUT" build_wheels: + name: Build Wheels needs: get_system_time runs-on: ${{ matrix.os }} strategy: @@ -51,6 +53,21 @@ jobs: cibw_python_version: 310 platform_id: manylinux_aarch64 manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python_version: "3.11" + cibw_python_version: 311 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python_version: "3.12" + cibw_python_version: 312 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python_version: "3.13" + cibw_python_version: 313 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 # MacOS x86_64 # - os: macos-13 From 9f25382cbad7922b6cd8ffdc97e5d792708837da Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Fri, 28 Feb 2025 11:42:43 -0500 Subject: [PATCH 30/31] Add comments --- .github/workflows/build-cibw.yml | 16 ++++++++++++++++ CMakeLists.txt | 8 ++++++++ build_tools/wheels/build_wheels.sh | 3 +++ build_tools/wheels/cibw_before_all.sh | 3 +++ python/setup.py.in | 2 ++ 5 files changed, 32 insertions(+) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index dac64d19d..bae27470f 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -1,3 +1,6 @@ +# This workflow builds the Python wheels using cibuildwheel and uploads them to TestPyPI. +# It can be triggered on push to the develop branch or manually via Github Actions. + name: Build Wheels (cibuildwheel) on: @@ -7,6 +10,9 @@ on: workflow_dispatch: jobs: + # Get the system time and store it in an output. This is used to tag the wheels. + # This needs to be done in a separate job so that each matrix job in build_wheels can + # access the same timestamp. get_system_time: name: Get System Time runs-on: ubuntu-latest @@ -96,6 +102,8 @@ jobs: with: python-version: ${{ matrix.python_version }} + # Set the DEVELOP flag and the TIMESTAMP environment variables. This is used in the + # top-level CMakeLists.txt to generate the GTSAM_VERSION_STRING. - name: Set Develop Flag run: | echo "DEVELOP=1" >> $GITHUB_ENV @@ -113,18 +121,26 @@ jobs: exit 1 fi + # We first build the Python wrapper module on the host machine. This is done because cibuildwheel + # expects a setup.py file to be present in the project directory. + # + # The Python wrapper module is then rebuilt within the cibuildwheel container before building + # the wheels to ensure platform compatibility. - name: Run CMake run: | cmake . -B build -DGTSAM_BUILD_PYTHON=1 -DGTSAM_PYTHON_VERSION=${{ matrix.python_version }} - name: Build and test wheels env: + # Generate the platform identifier. See https://cibuildwheel.pypa.io/en/stable/options/#build-skip. CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }} CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_image }} CIBW_ARCHS: all CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP TIMESTAMP + # Use build instead of pip wheel to build the wheels. This is recommended by PyPA. + # See https://cibuildwheel.pypa.io/en/stable/options/#build-frontend. CIBW_BUILD_FRONTEND: "build" CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project} diff --git a/CMakeLists.txt b/CMakeLists.txt index 826785bec..cabde7653 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,14 @@ set (GTSAM_VERSION_PATCH 0) set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") +# Set the version string for the library. +# +# If the environment variable DEVELOP is set, then the version string will be +# "MAJOR.MINORprerelease.devTIMESTAMP". TIMESTAMP is another environment variable that should be set to the current +# datetime. See build-cibw.yaml for example usage. +# +# If the prerelease version is empty, then the version string will be "MAJOR.MINOR.PATCH". Otherwise, the version +# string will be "MAJOR.MINORprerelease". if (DEFINED ENV{DEVELOP}) set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.dev$ENV{TIMESTAMP}") set (SETUP_NAME "gtsam-develop") diff --git a/build_tools/wheels/build_wheels.sh b/build_tools/wheels/build_wheels.sh index a8cbb93de..29247953d 100644 --- a/build_tools/wheels/build_wheels.sh +++ b/build_tools/wheels/build_wheels.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This script calls cibuildwheel to build the wheels for the project. It is used in the build-cibw.yml workflow in .github/workflows. +# Note that the build/python directory contains the wrapper module built for the specified Python version. + set -e set -x diff --git a/build_tools/wheels/cibw_before_all.sh b/build_tools/wheels/cibw_before_all.sh index 4f0f33f16..2398877a8 100644 --- a/build_tools/wheels/cibw_before_all.sh +++ b/build_tools/wheels/cibw_before_all.sh @@ -13,6 +13,7 @@ ARCH=$(uname -m) export PYTHON="python${PYTHON_VERSION}" if [ "$(uname)" == "Linux" ]; then + # manylinux2014 is based on CentOS 7, so use yum to install dependencies yum install -y wget # Install Boost from source @@ -32,6 +33,7 @@ $(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt rm -rf $PROJECT_DIR/build rm -rf CMakeCache.txt CMakeFiles +# Build the Python wrapper module cmake $PROJECT_DIR \ -B build \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ @@ -50,6 +52,7 @@ cmake $PROJECT_DIR \ cd $PROJECT_DIR/build/python +# Install the Python wrapper module and generate Python stubs if [ "$(uname)" == "Linux" ]; then make -j $(nproc) install make -j $(nproc) python-stubs diff --git a/python/setup.py.in b/python/setup.py.in index 96c21a55e..cd9177ab8 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -20,6 +20,8 @@ package_data = { # Cleaner to read in the contents rather than copy them over. readme_contents = open("${GTSAM_SOURCE_DIR}/README.md").read() +# The cibuildwheel tool won't recognize a wheel as platform-dependent unless the ext_modules option is defined in setup.py. This is used to define C/C++ source files that need to be built for the wheel. +# However, we pre-build our C++ files. Thus, we force cibuildwheel to think that there are ext_modules defined by overwriting the has_ext_modules() function. class BinaryDistribution(Distribution): def has_ext_modules(foo): return True From 895fc8829f951c48e17949edd3b8c3b06d251488 Mon Sep 17 00:00:00 2001 From: Yashas Ambati Date: Fri, 28 Feb 2025 14:59:54 -0500 Subject: [PATCH 31/31] Switch to PyPI --- .github/workflows/build-cibw.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cibw.yml b/.github/workflows/build-cibw.yml index bae27470f..987c1645c 100644 --- a/.github/workflows/build-cibw.yml +++ b/.github/workflows/build-cibw.yml @@ -167,9 +167,9 @@ jobs: path: dist/ merge-multiple: true - - name: Publish to TestPyPI + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: verbose: true packages-dir: dist/ - repository-url: https://test.pypi.org/legacy/ + # repository-url: https://test.pypi.org/legacy/