diff --git a/cmake/HandlePerfTools.cmake b/cmake/HandlePerfTools.cmake
index 6c455d952..4b5e5c8b3 100644
--- a/cmake/HandlePerfTools.cmake
+++ b/cmake/HandlePerfTools.cmake
@@ -1,5 +1,4 @@
 
 ###############################################################################
 # Find Google perftools
-find_package(GooglePerfTools)
-target_link_libraries(${PROJECT_NAME} ${GPERFTOOLS_TCMALLOC} ${GPERFTOOLS_PROFILER})
\ No newline at end of file
+find_package(GooglePerfTools)
\ No newline at end of file
diff --git a/gtsam/CMakeLists.txt b/gtsam/CMakeLists.txt
index 1fc8e4570..dc9f40ed7 100644
--- a/gtsam/CMakeLists.txt
+++ b/gtsam/CMakeLists.txt
@@ -147,6 +147,10 @@ if (GTSAM_USE_EIGEN_MKL)
   target_include_directories(gtsam PUBLIC ${MKL_INCLUDE_DIR})
 endif()
 
+if (GOOGLE_PERFTOOLS_FOUND)
+  target_link_libraries(gtsam PRIVATE ${GPERFTOOLS_TCMALLOC} ${GPERFTOOLS_PROFILER})
+endif()
+
 # Add includes for source directories 'BEFORE' boost and any system include
 # paths so that the compiler uses GTSAM headers in our source directory instead
 # of any previously installed GTSAM headers.