From 68c6bf4a381141b8101a0235c917e9c99aa5a609 Mon Sep 17 00:00:00 2001 From: Richard Roberts Date: Fri, 16 Aug 2013 20:26:07 +0000 Subject: [PATCH] Added CMake code and preprocessor macros so that everything works without TBB and falls back on single-threading --- CMakeLists.txt | 54 ++++++++++++++++++----------- gtsam/CMakeLists.txt | 6 ++-- gtsam/base/ConcurrentMap.h | 41 ++++++++++++++++++---- gtsam/base/FastDefaultAllocator.h | 30 ++++++++++++++-- gtsam/base/FastVector.h | 11 +++--- gtsam/base/timing.cpp | 32 +++++++++++++---- gtsam/base/timing.h | 22 +++++++----- gtsam/base/treeTraversal-inst.h | 24 ++++++++----- gtsam/config.h.in | 5 ++- gtsam/inference/JunctionTree-inst.h | 4 --- gtsam/linear/HessianFactor.h | 4 --- gtsam_unstable/CMakeLists.txt | 2 +- 12 files changed, 161 insertions(+), 74 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d804d27b..df11d9af5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,7 @@ endif() option(GTSAM_POSE3_EXPMAP "Enable/Disable using Pose3::EXPMAP as the default mode. If disabled, Pose3::FIRST_ORDER will be used." OFF) option(GTSAM_ROT3_EXPMAP "Ignore if GTSAM_USE_QUATERNIONS is OFF (Rot3::EXPMAP by default). Otherwise, enable Rot3::EXPMAP, or if disabled, use Rot3::CAYLEY." OFF) option(GTSAM_ENABLE_CONSISTENCY_CHECKS "Enable/Disable expensive consistency checks" OFF) +option(GTSAM_WITH_TBB "Use Intel Threaded Building Blocks (TBB) if available" ON) # Options relating to MATLAB wrapper # TODO: Check for matlab mex binary before handling building of binaries @@ -135,6 +136,27 @@ else() endif() +############################################################################### +# Find TBB +find_package(TBB) + +# Set up variables if we're using TBB +if(TBB_FOUND AND GTSAM_WITH_TBB) + set(GTSAM_USE_TBB 1) # This will go into config.h + set(GTSAM_TBB_LIBRARIES "") + if(TBB_DEBUG_LIBRARIES) + foreach(lib ${TBB_LIBRARIES}) + list(APPEND GTSAM_TBB_LIBRARIES optimized "${lib}") + endforeach() + foreach(lib ${TBB_DEBUG_LIBRARIES}) + list(APPEND GTSAM_TBB_LIBRARIES debug "${lib}") + endforeach() + else() + set(GTSAM_TBB_LIBRARIES ${TBB_LIBRARIES}) + endif() +endif() + + ############################################################################### # Option for using system Eigen or GTSAM-bundled Eigen option(GTSAM_USE_SYSTEM_EIGEN "Find and use system-installed Eigen. If 'off', use the one bundled with GTSAM" OFF) @@ -163,26 +185,6 @@ configure_file(gtsam/3rdparty/gtsam_eigen_includes.h.in gtsam/3rdparty/gtsam_eig install(FILES ${CMAKE_BINARY_DIR}/gtsam/3rdparty/gtsam_eigen_includes.h DESTINATION include/gtsam/3rdparty) -############################################################################### -# Find TBB -set(ENV{TBB_ARCH_PLATFORM} "intel64/vc11") -set(ENV{TBB_BIN_DIR} "C:/Program Files/Intel/TBB/bin") -set(ENV{TBB_TARGET_ARCH} intel64) -set(ENV{TBB_TARGET_VS} vc11) -find_package(TBB REQUIRED) -set(TBB_LIBS "") -if(TBB_DEBUG_LIBRARIES) - foreach(lib ${TBB_LIBRARIES}) - list(APPEND TBB_LIBS optimized "${lib}") - endforeach() - foreach(lib ${TBB_DEBUG_LIBRARIES}) - list(APPEND TBB_LIBS debug "${lib}") - endforeach() -else() - set(TBB_LIBS ${TBB_LIBRARIES}) -endif() - - ############################################################################### # Global compile options @@ -317,6 +319,14 @@ string(TOUPPER "${CMAKE_BUILD_TYPE}" cmake_build_type_toupper) message(STATUS " Build type : ${CMAKE_BUILD_TYPE}") message(STATUS " C compilation flags : ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${cmake_build_type_toupper}}") message(STATUS " C++ compilation flags : ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${cmake_build_type_toupper}}") +if(GTSAM_USE_TBB) + message(STATUS " Use Intel TBB : Yes") +elseif(TBB_FOUND) + message(STATUS " Use Intel TBB : TBB found but GTSAM_WITH_TBB is disabled") +else() + message(STATUS " Use Intel TBB : TBB not found") +endif() + message(STATUS "Packaging flags ") message(STATUS " CPack Source Generator : ${CPACK_SOURCE_GENERATOR}") @@ -333,5 +343,9 @@ print_config_flag(${GTSAM_BUILD_WRAP} "Build Wrap print_config_flag(${GTSAM_INSTALL_WRAP} "Install wrap utility ") message(STATUS "===============================================================") +if(GTSAM_WITH_TBB AND NOT TBB_FOUND) + message(WARNING "GTSAM_USE_TBB is set to 'On' but TBB was not found. This is ok, but GTSAM parallelization will be disabled. Set GTSAM_USE_TBB to 'Off' to avoid this warning.") +endif() + # Include CPack *after* all flags include(CPack) diff --git a/gtsam/CMakeLists.txt b/gtsam/CMakeLists.txt index 703a186a4..56146317e 100644 --- a/gtsam/CMakeLists.txt +++ b/gtsam/CMakeLists.txt @@ -104,7 +104,7 @@ message(STATUS "Install prefix: ${CMAKE_INSTALL_PREFIX}") if (GTSAM_BUILD_STATIC_LIBRARY) message(STATUS "Building GTSAM - static") add_library(gtsam-static STATIC ${gtsam_srcs}) - target_link_libraries(gtsam-static ${GTSAM_BOOST_LIBRARIES} ${TBB_LIBS}) + target_link_libraries(gtsam-static ${GTSAM_BOOST_LIBRARIES} ${GTSAM_TBB_LIBRARIES}) set_target_properties(gtsam-static PROPERTIES OUTPUT_NAME gtsam CLEAN_DIRECT_OUTPUT 1 @@ -123,7 +123,7 @@ endif () if (GTSAM_BUILD_SHARED_LIBRARY) message(STATUS "Building GTSAM - shared") add_library(gtsam-shared SHARED ${gtsam_srcs}) - target_link_libraries(gtsam-shared ${GTSAM_BOOST_LIBRARIES} ${TBB_LIBS}) + target_link_libraries(gtsam-shared ${GTSAM_BOOST_LIBRARIES} ${GTSAM_TBB_LIBRARIES}) set_target_properties(gtsam-shared PROPERTIES OUTPUT_NAME gtsam CLEAN_DIRECT_OUTPUT 1 @@ -170,5 +170,5 @@ if (GTSAM_INSTALL_MATLAB_TOOLBOX) # Macro to handle details of setting up targets # FIXME: issue with dependency between wrap_gtsam and wrap_gtsam_build, only shows up on CMake 2.8.3 - wrap_library(gtsam "${mexFlags}" "../" "${TBB_LIBS}") + wrap_library(gtsam "${mexFlags}" "../" "${GTSAM_TBB_LIBRARIES}") endif () diff --git a/gtsam/base/ConcurrentMap.h b/gtsam/base/ConcurrentMap.h index aa0521b08..2263ae6d1 100644 --- a/gtsam/base/ConcurrentMap.h +++ b/gtsam/base/ConcurrentMap.h @@ -18,13 +18,31 @@ #pragma once -#include -#undef min -#undef max -#undef ERROR +#include + +// Change class depending on whether we are using TBB +#ifdef GTSAM_USE_TBB + +// Include TBB header +# include +# undef min // TBB seems to include Windows.h which defines these macros that cause problems +# undef max +# undef ERROR + +// Use TBB concurrent_unordered_map for ConcurrentMap +# define CONCURRENT_MAP_BASE tbb::concurrent_unordered_map + +#else + +// If we're not using TBB, use a FastMap for ConcurrentMap +# include +# define CONCURRENT_MAP_BASE gtsam::FastMap + +#endif #include #include +#include #include @@ -39,11 +57,11 @@ namespace gtsam { * @addtogroup base */ template -class ConcurrentMap : public tbb::concurrent_unordered_map { +class ConcurrentMap : public CONCURRENT_MAP_BASE { public: - typedef tbb::concurrent_unordered_map Base; + typedef CONCURRENT_MAP_BASE Base; /** Default constructor */ ConcurrentMap() {} @@ -52,7 +70,7 @@ public: template ConcurrentMap(INPUTITERATOR first, INPUTITERATOR last) : Base(first, last) {} - /** Copy constructor from another FastMap */ + /** Copy constructor from another ConcurrentMap */ ConcurrentMap(const ConcurrentMap& x) : Base(x) {} /** Copy constructor from the base map class */ @@ -61,6 +79,15 @@ public: /** Handy 'exists' function */ bool exists(const KEY& e) const { return this->count(e); } +#ifndef GTSAM_USE_TBB + // If we're not using TBB and this is actually a FastMap, we need to add these functions and hide + // the original erase functions. + iterator unsafe_erase(const_iterator position) { return Base::erase(position); } + size_type unsafe_erase(const key_type& k) { return Base::erase(k); } + iterator unsafe_erase(const_iterator first, const_iterator last) { return Base::erase(first, last); } + void erase() { BOOST_STATIC_ASSERT_MSG(0, "For ConcurrentMap, use unsafe_erase instead of erase."); } +#endif + private: /** Serialization function */ friend class boost::serialization::access; diff --git a/gtsam/base/FastDefaultAllocator.h b/gtsam/base/FastDefaultAllocator.h index 4a9d9d8f2..156a87f55 100644 --- a/gtsam/base/FastDefaultAllocator.h +++ b/gtsam/base/FastDefaultAllocator.h @@ -18,16 +18,22 @@ #pragma once +#include + #if !defined GTSAM_ALLOCATOR_BOOSTPOOL && !defined GTSAM_ALLOCATOR_TBB && !defined GTSAM_ALLOCATOR_STL -// Use TBB allocator by default -# define GTSAM_ALLOCATOR_TBB +# ifdef GTSAM_USE_TBB +// Use TBB allocator by default if we have TBB, otherwise boost pool +# define GTSAM_ALLOCATOR_TBB +# else +# define GTSAM_ALLOCATOR_BOOSTPOOL +# endif #endif #if defined GTSAM_ALLOCATOR_BOOSTPOOL # include #elif defined GTSAM_ALLOCATOR_TBB # include -# undef min +# undef min // TBB seems to include Windows.h which defines these macros that cause problems # undef max # undef ERROR #elif defined GTSAM_ALLOCATOR_STL @@ -39,6 +45,7 @@ namespace gtsam namespace internal { + /// Default allocator for list, map, and set types template struct FastDefaultAllocator { @@ -57,6 +64,23 @@ namespace gtsam static const bool isBoost = false; static const bool isTBB = false; static const bool isSTL = true; +#endif + }; + + /// Default allocator for vector types (we never use boost pool for vectors) + template + struct FastDefaultVectorAllocator + { +#if defined GTSAM_ALLOCATOR_TBB + typedef tbb::tbb_allocator type; + static const bool isBoost = false; + static const bool isTBB = true; + static const bool isSTL = false; +#else + typedef std::allocator type; + static const bool isBoost = false; + static const bool isTBB = false; + static const bool isSTL = true; #endif }; } diff --git a/gtsam/base/FastVector.h b/gtsam/base/FastVector.h index 309f4d2f0..23d0e05af 100644 --- a/gtsam/base/FastVector.h +++ b/gtsam/base/FastVector.h @@ -35,11 +35,11 @@ namespace gtsam { * @addtogroup base */ template -class FastVector: public std::vector::type> { +class FastVector: public std::vector::type> { public: - typedef std::vector::type> Base; + typedef std::vector Base; /** Default constructor */ FastVector() {} @@ -60,9 +60,6 @@ public: Base::assign(first, last); } - /** Copy constructor from another FastVector */ - FastVector(const FastVector& x) : Base(x) {} - /** Copy constructor from a FastList */ FastVector(const FastList& x) { if(x.size() > 0) @@ -79,7 +76,9 @@ public: FastVector(const Base& x) : Base(x) {} /** Copy constructor from a standard STL container */ - FastVector(const std::vector& x, typename boost::disable_if_c::isSTL>::type* = 0) { + template + FastVector(const std::vector& x) + { // This if statement works around a bug in boost pool allocator and/or // STL vector where if the size is zero, the pool allocator will allocate // huge amounts of memory. diff --git a/gtsam/base/timing.cpp b/gtsam/base/timing.cpp index 8f6f9f346..05b52fb81 100644 --- a/gtsam/base/timing.cpp +++ b/gtsam/base/timing.cpp @@ -149,7 +149,8 @@ const boost::shared_ptr& TimingOutline::child(size_t child, const } /* ************************************************************************* */ -void TimingOutline::ticInternal() { +void TimingOutline::ticInternal() +{ #ifdef GTSAM_USING_NEW_BOOST_TIMERS assert(timer_.is_stopped()); timer_.start(); @@ -158,24 +159,41 @@ void TimingOutline::ticInternal() { timer_.restart(); *timerActive_ = true; #endif + +#ifdef GTSAM_USE_TBB tbbTimer_ = tbb::tick_count::now(); +#endif } /* ************************************************************************* */ -void TimingOutline::tocInternal() { +void TimingOutline::tocInternal() +{ #ifdef GTSAM_USING_NEW_BOOST_TIMERS + assert(!timer_.is_stopped()); timer_.stop(); - add((timer_.elapsed().user + timer_.elapsed().system) / 1000, - //timer_.elapsed().wall / 1000 - size_t((tbb::tick_count::now() - tbbTimer_).seconds() * 1e6) - ); + size_t cpuTime = (timer_.elapsed().user + timer_.elapsed().system) / 1000; +# ifndef GTSAM_USE_TBB + size_t wallTime = timer_.elapsed().wall / 1000; +# endif + #else + assert(timerActive_); double elapsed = timer_.elapsed(); - add(size_t(elapsed * 1000000.0), 0); + size_t cpuTime = size_t(elapsed * 1000000.0); *timerActive_ = false; +# ifndef GTSAM_USE_TBB + size_t wallTime = cpuTime; +# endif + #endif + +#ifdef GTSAM_USE_TBB + size_t wallTime = size_t((tbb::tick_count::now() - tbbTimer_).seconds() * 1e6); +#endif + + add(cpuTime, wallTime); } /* ************************************************************************* */ diff --git a/gtsam/base/timing.h b/gtsam/base/timing.h index 68dba5f28..48eec7de7 100644 --- a/gtsam/base/timing.h +++ b/gtsam/base/timing.h @@ -26,21 +26,23 @@ // Automatically use the new Boost timers if version is recent enough. #if BOOST_VERSION >= 104800 -#ifndef GTSAM_DISABLE_NEW_TIMERS -#define GTSAM_USING_NEW_BOOST_TIMERS -#endif +# ifndef GTSAM_DISABLE_NEW_TIMERS +# define GTSAM_USING_NEW_BOOST_TIMERS +# endif #endif #ifdef GTSAM_USING_NEW_BOOST_TIMERS -#include +# include #else -#include +# include #endif -#include -#undef min -#undef max -#undef ERROR +#ifdef GTSAM_USE_TBB +# include +# undef min +# undef max +# undef ERROR +#endif namespace gtsam { @@ -71,7 +73,9 @@ namespace gtsam { boost::timer timer_; gtsam::ValueWithDefault timerActive_; #endif +#ifdef GTSAM_USE_TBB tbb::tick_count tbbTimer_; +#endif void add(size_t usecs, size_t usecsWall); public: TimingOutline(const std::string& label, size_t myId); diff --git a/gtsam/base/treeTraversal-inst.h b/gtsam/base/treeTraversal-inst.h index e3a4eaeb4..2d3c135ad 100644 --- a/gtsam/base/treeTraversal-inst.h +++ b/gtsam/base/treeTraversal-inst.h @@ -28,9 +28,12 @@ #include #include -#include -#undef max // TBB seems to include windows.h and we don't want these macros -#undef min +#ifdef GTSAM_USE_TBB +# include +# undef max // TBB seems to include windows.h and we don't want these macros +# undef min +# undef ERROR +#endif namespace gtsam { @@ -56,6 +59,9 @@ namespace gtsam { void operator()(const boost::shared_ptr& node, const DATA& data) {} }; + +#ifdef GTSAM_USE_TBB + // Internal node used in parallel traversal stack template struct ParallelTraversalNode { @@ -172,12 +178,8 @@ namespace gtsam { return *new(tbb::task::allocate_root()) RootTask(roots, rootData, visitorPre, visitorPost, problemSizeThreshold); } - /* ************************************************************************* */ - //template - //struct ParallelDFSData { - // DATA myData; - // FastList >& - //}; +#endif + } /** Traverse a forest depth-first with pre-order and post-order visits. @@ -275,12 +277,16 @@ namespace gtsam { void DepthFirstForestParallel(FOREST& forest, DATA& rootData, VISITOR_PRE& visitorPre, VISITOR_POST& visitorPost, int problemSizeThreshold = 10) { +#ifdef GTSAM_USE_TBB // Typedefs typedef typename FOREST::Node Node; typedef boost::shared_ptr sharedNode; tbb::task::spawn_root_and_wait(CreateRootTask( forest.roots(), rootData, visitorPre, visitorPost, problemSizeThreshold)); +#else + DepthFirstForest(forest, rootData, visitorPre, visitorPost); +#endif } diff --git a/gtsam/config.h.in b/gtsam/config.h.in index 120e06565..fadb460b9 100644 --- a/gtsam/config.h.in +++ b/gtsam/config.h.in @@ -30,4 +30,7 @@ // Whether GTSAM is compiled to use Rot3::EXPMAP as the default coordinates mode for Rot3's retract and localCoordinates (otherwise, Pose3::CAYLEY will be used) #ifndef GTSAM_USE_QUATERNIONS #cmakedefine GTSAM_ROT3_EXPMAP -#endif \ No newline at end of file +#endif + +// Whether we are using TBB (if TBB was found and GTSAM_WITH_TBB is enabled in CMake) +#cmakedefine GTSAM_USE_TBB diff --git a/gtsam/inference/JunctionTree-inst.h b/gtsam/inference/JunctionTree-inst.h index 298bf6389..5f50763fb 100644 --- a/gtsam/inference/JunctionTree-inst.h +++ b/gtsam/inference/JunctionTree-inst.h @@ -293,10 +293,6 @@ namespace gtsam { boost::shared_ptr result = boost::make_shared(); EliminationData rootsContainer(0, roots_.size()); EliminationPostOrderVisitor visitorPost(function, result->nodes_); -#ifdef ENABLE_TIMING - // In timing mode, use a single thread only, timing outline is not thread-safe. - tbb::task_scheduler_init init(1); -#endif treeTraversal::DepthFirstForestParallel(*this, rootsContainer, eliminationPreOrderVisitor, visitorPost, 10); diff --git a/gtsam/linear/HessianFactor.h b/gtsam/linear/HessianFactor.h index 0cc1b2fdc..2f5b5d9da 100644 --- a/gtsam/linear/HessianFactor.h +++ b/gtsam/linear/HessianFactor.h @@ -24,10 +24,6 @@ #include #include -#include -#undef max -#undef min -#undef ERROR namespace gtsam { diff --git a/gtsam_unstable/CMakeLists.txt b/gtsam_unstable/CMakeLists.txt index 7a9acc787..be5171049 100644 --- a/gtsam_unstable/CMakeLists.txt +++ b/gtsam_unstable/CMakeLists.txt @@ -122,7 +122,7 @@ if (GTSAM_INSTALL_MATLAB_TOOLBOX) endif() # Macro to handle details of setting up targets - wrap_library(gtsam_unstable "${mexFlags}" "./" "gtsam;${TBB_LIBS}") + wrap_library(gtsam_unstable "${mexFlags}" "./" "gtsam;${GTSAM_TBB_LIBRARIES}") endif(GTSAM_INSTALL_MATLAB_TOOLBOX)