diff --git a/gtsam/linear/HessianFactor.cpp b/gtsam/linear/HessianFactor.cpp
index 9c57ea5ea..3f6444990 100644
--- a/gtsam/linear/HessianFactor.cpp
+++ b/gtsam/linear/HessianFactor.cpp
@@ -58,7 +58,7 @@ namespace gtsam {
 #ifndef NDEBUG
     // Check for non-finite values
     for(size_t i=0; i<matrix_.size1(); ++i)
-      for(size_t j=0; j<matrix_.size2(); ++j)
+      for(size_t j=i; j<matrix_.size2(); ++j)
         if(!isfinite(matrix_(i,j)))
           throw invalid_argument("HessianFactor contains non-finite matrix entries.");
 #endif