Added support for MKL in Eigen and fixed several errors and warnings within Eigen when using MKL

release/4.3a0
Richard Roberts 2013-11-18 19:23:23 +00:00
parent 13a43f41a7
commit 0608e9a05b
19 changed files with 127 additions and 26 deletions

View File

@ -59,6 +59,8 @@ option(GTSAM_POSE3_EXPMAP "Enable/Disable using Pose3::EXPMAP as the defau
option(GTSAM_ROT3_EXPMAP "Ignore if GTSAM_USE_QUATERNIONS is OFF (Rot3::EXPMAP by default). Otherwise, enable Rot3::EXPMAP, or if disabled, use Rot3::CAYLEY." OFF)
option(GTSAM_ENABLE_CONSISTENCY_CHECKS "Enable/Disable expensive consistency checks" OFF)
option(GTSAM_WITH_TBB "Use Intel Threaded Building Blocks (TBB) if available" ON)
option(GTSAM_WITH_EIGEN_MKL "Eigen will use Intel MKL if available" ON)
option(GTSAM_WITH_EIGEN_MKL_OPENMP "Eigen, when using Intel MKL, will also use OpenMP for multithreading if available" ON)
option(GTSAM_THROW_CHEIRALITY_EXCEPTION "Throw exception when a triangulated point is behind a camera" ON)
# Options relating to MATLAB wrapper
@ -157,6 +159,7 @@ if(TBB_FOUND AND GTSAM_WITH_TBB)
else()
set(GTSAM_TBB_LIBRARIES ${TBB_LIBRARIES})
endif()
list(APPEND GTSAM_ADDITIONAL_LIBRARIES ${GTSAM_TBB_LIBRARIES})
else()
set(GTSAM_USE_TBB 0) # This will go into config.h
endif()
@ -167,6 +170,28 @@ endif()
find_package(GooglePerfTools)
###############################################################################
# Find MKL
find_package(MKL)
if(MKL_FOUND AND GTSAM_WITH_EIGEN_MKL)
set(GTSAM_USE_EIGEN_MKL 1) # This will go into config.h
set(EIGEN_USE_MKL_ALL 1) # This will go into config.h - it makes Eigen use MKL
include_directories(${MKL_INCLUDE_DIR})
list(APPEND GTSAM_ADDITIONAL_LIBRARIES ${MKL_LIBRARIES})
endif()
###############################################################################
# Find OpenMP
find_package(OpenMP)
if(OPENMP_FOUND AND GTSAM_USE_EIGEN_MKL AND GTSAM_WITH_EIGEN_MKL_OPENMP)
set(GTSAM_USE_EIGEN_MKL_OPENMP 1) # This will go into config.h
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
endif()
###############################################################################
# Option for using system Eigen or GTSAM-bundled Eigen
option(GTSAM_USE_SYSTEM_EIGEN "Find and use system-installed Eigen. If 'off', use the one bundled with GTSAM" OFF)
@ -205,7 +230,7 @@ if(GTSAM_USE_TBB)
set(preferred_allocator TBB)
else()
list(APPEND possible_allocators BoostPool STL)
set(preferred_allocator BoostPool)
set(preferred_allocator STL)
endif()
if(GOOGLE_PERFTOOLS_FOUND)
list(APPEND possible_allocators tcmalloc)
@ -368,6 +393,20 @@ elseif(TBB_FOUND)
else()
message(STATUS " Use Intel TBB : TBB not found")
endif()
if(GTSAM_USE_EIGEN_MKL)
message(STATUS " Eigen will use MKL : Yes")
elseif(MKL_FOUND)
message(STATUS " Eigen will use MKL : MKL found but GTSAM_WITH_EIGEN_MKL is disabled")
else()
message(STATUS " Eigen will use MKL : MKL not found")
endif()
if(GTSAM_USE_EIGEN_MKL_OPENMP)
message(STATUS " Eigen will use MKL and OpenMP : Yes")
elseif(OPENMP_FOUND)
message(STATUS " Eigen will use MKL and OpenMP : OpenMP found but Eigen using MKL is disabled")
else()
message(STATUS " Eigen will use MKL and OpenMP : OpenMP not found")
endif()
message(STATUS " Default allocator : ${GTSAM_DEFAULT_ALLOCATOR}")
@ -386,8 +425,15 @@ print_config_flag(${GTSAM_BUILD_WRAP} "Build Wrap
print_config_flag(${GTSAM_INSTALL_WRAP} "Install wrap utility ")
message(STATUS "===============================================================")
# Print warnings at the end
if(GTSAM_WITH_TBB AND NOT TBB_FOUND)
message(WARNING "GTSAM_USE_TBB is set to 'On' but TBB was not found. This is ok, but GTSAM parallelization will be disabled. Set GTSAM_WITH_TBB to 'Off' to avoid this warning.")
message(WARNING "GTSAM_WITH_TBB is set to 'On' but TBB was not found. This is ok, but note that GTSAM parallelization will be disabled. Set GTSAM_WITH_TBB to 'Off' to avoid this warning.")
endif()
if(GTSAM_WITH_EIGEN_MKL AND NOT MKL_FOUND)
message(WARNING "GTSAM_WITH_EIGEN_MKL is set to 'On' but MKL was not found. This is ok, but note that MKL yields better performance. Set GTSAM_WITH_EIGEN_MKL to 'Off' to avoid this warning.")
endif()
if(GTSAM_WITH_EIGEN_MKL_OPENMP AND NOT OPENMP_FOUND AND MKL_FOUND)
message(WARNING "GTSAM_WITH_EIGEN_MKL_OPENMP is set to 'On' but your compiler does not support OpenMP. This is ok, but performance may be improved with OpenMP. Set GTSAM_WITH_EIGEN_MKL_OPENMP to 'Off' to avoid this warning.")
endif()
# Include CPack *after* all flags

View File

@ -33,7 +33,7 @@
#ifndef EIGEN_LLT_MKL_H
#define EIGEN_LLT_MKL_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
#include <iostream>
namespace Eigen {
@ -60,7 +60,7 @@ template<> struct mkl_llt<EIGTYPE> \
lda = m.outerStride(); \
\
info = LAPACKE_##MKLPREFIX##potrf( matrix_order, uplo, size, (MKLTYPE*)a, lda ); \
info = (info==0) ? Success : NumericalIssue; \
info = (info==0) ? -1 : 1; \
return info; \
} \
}; \

View File

@ -33,7 +33,7 @@
#ifndef EIGEN_COMPLEX_SCHUR_MKL_H
#define EIGEN_COMPLEX_SCHUR_MKL_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
namespace Eigen {

View File

@ -33,7 +33,7 @@
#ifndef EIGEN_REAL_SCHUR_MKL_H
#define EIGEN_REAL_SCHUR_MKL_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
namespace Eigen {

View File

@ -33,7 +33,7 @@
#ifndef EIGEN_SAEIGENSOLVER_MKL_H
#define EIGEN_SAEIGENSOLVER_MKL_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
namespace Eigen {

View File

@ -33,7 +33,7 @@
#ifndef EIGEN_PARTIALLU_LAPACK_H
#define EIGEN_PARTIALLU_LAPACK_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
namespace Eigen {

View File

@ -34,7 +34,7 @@
#ifndef EIGEN_COLPIVOTINGHOUSEHOLDERQR_MKL_H
#define EIGEN_COLPIVOTINGHOUSEHOLDERQR_MKL_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
namespace Eigen {
@ -63,12 +63,12 @@ ColPivHouseholderQR<Matrix<EIGTYPE, Dynamic, Dynamic, EIGCOLROW, Dynamic, Dynami
\
m_nonzero_pivots = 0; \
m_maxpivot = RealScalar(0);\
m_colsPermutation.resize(cols); \
m_colsPermutation.resize((int)cols); \
m_colsPermutation.indices().setZero(); \
\
lapack_int lda = m_qr.outerStride(), i; \
lapack_int lda = (lapack_int) m_qr.outerStride(), i; \
lapack_int matrix_order = MKLCOLROW; \
LAPACKE_##MKLPREFIX##geqp3( matrix_order, rows, cols, (MKLTYPE*)m_qr.data(), lda, (lapack_int*)m_colsPermutation.indices().data(), (MKLTYPE*)m_hCoeffs.data()); \
LAPACKE_##MKLPREFIX##geqp3( matrix_order, (lapack_int)rows, (lapack_int)cols, (MKLTYPE*)m_qr.data(), lda, (lapack_int*)m_colsPermutation.indices().data(), (MKLTYPE*)m_hCoeffs.data()); \
m_isInitialized = true; \
m_maxpivot=m_qr.diagonal().cwiseAbs().maxCoeff(); \
m_hCoeffs.adjointInPlace(); \

View File

@ -251,6 +251,7 @@ void householder_qr_inplace_unblocked(MatrixQR& mat, HCoeffs& hCoeffs, typename
}
/** \internal */
#ifndef EIGEN_USE_LAPACKE
template<typename MatrixQR, typename HCoeffs>
void householder_qr_inplace_blocked(MatrixQR& mat, HCoeffs& hCoeffs,
typename MatrixQR::Index maxBlockSize=32,
@ -301,6 +302,7 @@ void householder_qr_inplace_blocked(MatrixQR& mat, HCoeffs& hCoeffs,
}
}
}
#endif
template<typename _MatrixType, typename Rhs>
struct solve_retval<HouseholderQR<_MatrixType>, Rhs>

View File

@ -34,7 +34,7 @@
#ifndef EIGEN_QR_MKL_H
#define EIGEN_QR_MKL_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
namespace Eigen {
@ -48,9 +48,9 @@ void householder_qr_inplace_blocked(MatrixQR& mat, HCoeffs& hCoeffs, \
typename MatrixQR::Index maxBlockSize=32, \
EIGTYPE* tempData = 0) \
{ \
lapack_int m = mat.rows(); \
lapack_int n = mat.cols(); \
lapack_int lda = mat.outerStride(); \
lapack_int m = (lapack_int) mat.rows(); \
lapack_int n = (lapack_int) mat.cols(); \
lapack_int lda = (lapack_int) mat.outerStride(); \
lapack_int matrix_order = (MatrixQR::IsRowMajor) ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \
LAPACKE_##MKLPREFIX##geqrf( matrix_order, m, n, (MKLTYPE*)mat.data(), lda, (MKLTYPE*)hCoeffs.data()); \
hCoeffs.adjointInPlace(); \

View File

@ -33,7 +33,7 @@
#ifndef EIGEN_JACOBISVD_MKL_H
#define EIGEN_JACOBISVD_MKL_H
#include "Eigen/src/Core/util/MKL_support.h"
#include "../Core/util/MKL_support.h"
namespace Eigen {

View File

@ -17,6 +17,7 @@
#pragma once
#cmakedefine EIGEN_USE_MKL_ALL // This is also defined in config.h
#include <@GTSAM_EIGEN_INCLUDE_PREFIX@Eigen/Dense>
#include <@GTSAM_EIGEN_INCLUDE_PREFIX@Eigen/QR>
#include <@GTSAM_EIGEN_INCLUDE_PREFIX@Eigen/LU>

View File

@ -95,7 +95,7 @@ message(STATUS "Install prefix: ${CMAKE_INSTALL_PREFIX}")
if (GTSAM_BUILD_STATIC_LIBRARY)
message(STATUS "Building GTSAM - static")
add_library(gtsam-static STATIC ${gtsam_srcs})
target_link_libraries(gtsam-static ${GTSAM_BOOST_LIBRARIES} ${GTSAM_TBB_LIBRARIES} ${GTSAM_ADDITIONAL_LIBRARIES})
target_link_libraries(gtsam-static ${GTSAM_BOOST_LIBRARIES} ${GTSAM_ADDITIONAL_LIBRARIES})
set_target_properties(gtsam-static PROPERTIES
OUTPUT_NAME gtsam
CLEAN_DIRECT_OUTPUT 1
@ -114,7 +114,7 @@ endif ()
if (GTSAM_BUILD_SHARED_LIBRARY)
message(STATUS "Building GTSAM - shared")
add_library(gtsam-shared SHARED ${gtsam_srcs})
target_link_libraries(gtsam-shared ${GTSAM_BOOST_LIBRARIES} ${GTSAM_TBB_LIBRARIES} ${GTSAM_ADDITIONAL_LIBRARIES})
target_link_libraries(gtsam-shared ${GTSAM_BOOST_LIBRARIES} ${GTSAM_ADDITIONAL_LIBRARIES})
set_target_properties(gtsam-shared PROPERTIES
OUTPUT_NAME gtsam
CLEAN_DIRECT_OUTPUT 1
@ -163,5 +163,5 @@ if (GTSAM_INSTALL_MATLAB_TOOLBOX)
# Macro to handle details of setting up targets
# FIXME: issue with dependency between wrap_gtsam and wrap_gtsam_build, only shows up on CMake 2.8.3
wrap_library(gtsam "${mexFlags}" "../" "${GTSAM_TBB_LIBRARIES}")
wrap_library(gtsam "${mexFlags}" "../" "${GTSAM_ADDITIONAL_LIBRARIES}")
endif ()

View File

@ -129,8 +129,17 @@ bool choleskyPartial(Matrix& ABC, size_t nFrontal) {
// Compute Cholesky factorization of A, overwrites A.
gttic(lld);
Eigen::LLT<Matrix, Eigen::Upper> llt = ABC.block(0,0,nFrontal,nFrontal).selfadjointView<Eigen::Upper>().llt();
ABC.block(0,0,nFrontal,nFrontal).triangularView<Eigen::Upper>() = llt.matrixU();
Eigen::ComputationInfo lltResult;
if(nFrontal > 0)
{
Eigen::LLT<Matrix, Eigen::Upper> llt = ABC.block(0, 0, nFrontal, nFrontal).selfadjointView<Eigen::Upper>().llt();
ABC.block(0, 0, nFrontal, nFrontal).triangularView<Eigen::Upper>() = llt.matrixU();
lltResult = llt.info();
}
else
{
lltResult = Eigen::Success;
}
gttoc(lld);
if(debug) cout << "R:\n" << Eigen::MatrixXd(ABC.topLeftCorner(nFrontal,nFrontal).triangularView<Eigen::Upper>()) << endl;
@ -155,7 +164,7 @@ bool choleskyPartial(Matrix& ABC, size_t nFrontal) {
// Check last diagonal element - Eigen does not check it
bool ok;
if(llt.info() == Eigen::Success) {
if(lltResult == Eigen::Success) {
if(nFrontal >= 2) {
int exp2, exp1;
(void)frexp(ABC(nFrontal-2, nFrontal-2), &exp2);

View File

@ -30,10 +30,15 @@
#include <boost/optional.hpp>
#ifdef GTSAM_USE_TBB
#include <tbb/task_scheduler_init.h>
#include <tbb/tbb_exception.h>
#include <tbb/scalable_allocator.h>
#endif
#ifdef GTSAM_USE_EIGEN_MKL_OPENMP
#include <omp.h>
#endif
namespace gtsam {
/// Integer nonlinear key type
@ -227,6 +232,32 @@ namespace gtsam {
InvalidArgumentThreadsafe(const std::string& description) : ThreadsafeException<InvalidArgumentThreadsafe>(description) {}
};
/* ************************************************************************* */
/// An object whose scope defines a block where TBB and OpenMP parallelism are mixed. In such a
/// block, we use default threads for TBB, and p/2 threads for OpenMP. If GTSAM is not compiled to
/// use both TBB and OpenMP, this has no effect.
class TbbOpenMPMixedScope
{
int previousOpenMPThreads;
public:
#if defined GTSAM_USE_TBB && defined GTSAM_USE_EIGEN_MKL_OPENMP
TbbOpenMPMixedScope() :
previousOpenMPThreads(omp_get_num_threads())
{
omp_set_num_threads(omp_get_num_procs() / 2);
}
~TbbOpenMPMixedScope()
{
omp_set_num_threads(previousOpenMPThreads);
}
#else
TbbOpenMPMixedScope() : previousOpenMPThreads(-1) {}
~TbbOpenMPMixedScope() {}
#endif
};
}
/* ************************************************************************* */

View File

@ -42,6 +42,13 @@
// Whether we are using TBB (if TBB was found and GTSAM_WITH_TBB is enabled in CMake)
#cmakedefine GTSAM_USE_TBB
// Whether Eigen will use MKL (if MKL was found and GTSAM_WITH_EIGEN_MKL is enabled in CMake)
#cmakedefine GTSAM_USE_EIGEN_MKL
#cmakedefine EIGEN_USE_MKL_ALL // This is also defined in gtsam_eigen_includes.h
// Whether Eigen with MKL will use OpenMP (if OpenMP was found, Eigen uses MKL, and GTSAM_WITH_EIGEN_MKL_OPENMP is enabled in CMake)
#cmakedefine GTSAM_USE_EIGEN_MKL_OPENMP
// The default allocator to use
#cmakedefine GTSAM_ALLOCATOR_BOOSTPOOL
#cmakedefine GTSAM_ALLOCATOR_TBB

View File

@ -162,8 +162,11 @@ namespace gtsam
boost::shared_ptr<BayesTreeType> result = boost::make_shared<BayesTreeType>();
EliminationData<This> rootsContainer(0, roots_.size());
EliminationPostOrderVisitor<This> visitorPost(function, result->nodes_);
treeTraversal::DepthFirstForestParallel(*this, rootsContainer,
eliminationPreOrderVisitor<This>, visitorPost, 10);
{
TbbOpenMPMixedScope threadLimiter; // Limits OpenMP threads since we're mixing TBB and OpenMP
treeTraversal::DepthFirstForestParallel(*this, rootsContainer,
eliminationPreOrderVisitor<This>, visitorPost, 10);
}
// Create BayesTree from roots stored in the dummy BayesTree node.
result->roots_.insert(result->roots_.end(), rootsContainer.bayesTreeNode->children.begin(), rootsContainer.bayesTreeNode->children.end());

View File

@ -134,6 +134,7 @@ namespace gtsam
OptimizeData rootData;
OptimizeClique<typename BAYESTREE::Clique> preVisitor;
treeTraversal::no_op postVisitor;
TbbOpenMPMixedScope threadLimiter; // Limits OpenMP threads since we're mixing TBB and OpenMP
treeTraversal::DepthFirstForestParallel(bayesTree, rootData, preVisitor, postVisitor);
return preVisitor.collectedResult;
}

View File

@ -286,6 +286,7 @@ GaussianFactorGraph::shared_ptr NonlinearFactorGraph::linearize(const Values& li
#ifdef GTSAM_USE_TBB
linearFG->resize(this->size());
TbbOpenMPMixedScope threadLimiter; // Limits OpenMP threads since we're mixing TBB and OpenMP
tbb::parallel_for(tbb::blocked_range<size_t>(0, this->size()),
_LinearizeOneFactor(*this, linearizationPoint, *linearFG));

View File

@ -114,7 +114,7 @@ if (GTSAM_INSTALL_MATLAB_TOOLBOX)
endif()
# Macro to handle details of setting up targets
wrap_library(gtsam_unstable "${mexFlags}" "./" "gtsam;${GTSAM_TBB_LIBRARIES}")
wrap_library(gtsam_unstable "${mexFlags}" "./" "gtsam")
endif(GTSAM_INSTALL_MATLAB_TOOLBOX)