diff --git a/gtsam/linear/JacobianFactor.cpp b/gtsam/linear/JacobianFactor.cpp
index 9be010ff5..2310d88f0 100644
--- a/gtsam/linear/JacobianFactor.cpp
+++ b/gtsam/linear/JacobianFactor.cpp
@@ -513,8 +513,10 @@ Vector JacobianFactor::error_vector(const VectorValues& c) const {
 
 /* ************************************************************************* */
 double JacobianFactor::error(const VectorValues& c) const {
-  Vector weighted = error_vector(c);
-  return 0.5 * weighted.dot(weighted);
+  Vector e = unweighted_error(c);
+  // Use the noise model distance function to get the correct error if available.
+  if (model_) return 0.5 * model_->distance(e);
+  return 0.5 * e.dot(e);
 }
 
 /* ************************************************************************* */
diff --git a/gtsam/nonlinear/LevenbergMarquardtOptimizer.cpp b/gtsam/nonlinear/LevenbergMarquardtOptimizer.cpp
index c85891af2..9b894db25 100644
--- a/gtsam/nonlinear/LevenbergMarquardtOptimizer.cpp
+++ b/gtsam/nonlinear/LevenbergMarquardtOptimizer.cpp
@@ -158,10 +158,13 @@ bool LevenbergMarquardtOptimizer::tryLambda(const GaussianFactorGraph& linear,
     if (params_.verbosityLM >= LevenbergMarquardtParams::TRYDELTA)
       delta.print("delta");
 
-    // cost change in the linearized system (old - new)
+    // Compute the old linearized error as it is not the same
+    // as the nonlinear error when robust noise models are used.
+    double oldLinearizedError = linear.error(VectorValues::Zero(delta));
     double newlinearizedError = linear.error(delta);
 
-    double linearizedCostChange = currentState->error - newlinearizedError;
+    // cost change in the linearized system (old - new)
+    double linearizedCostChange = oldLinearizedError - newlinearizedError;
     if (verbose)
       cout << "newlinearizedError = " << newlinearizedError
            << "  linearizedCostChange = " << linearizedCostChange << endl;
@@ -188,8 +191,8 @@ bool LevenbergMarquardtOptimizer::tryLambda(const GaussianFactorGraph& linear,
       // cost change in the original, nonlinear system (old - new)
       costChange = currentState->error - newError;
 
-      if (linearizedCostChange >
-          1e-20) {  // the (linear) error has to decrease to satisfy this condition
+      if (linearizedCostChange > std::numeric_limits<double>::epsilon() * oldLinearizedError) {
+        // the (linear) error has to decrease to satisfy this condition
         // fidelity of linearized model VS original system between
         modelFidelity = costChange / linearizedCostChange;
         // if we decrease the error in the nonlinear system and modelFidelity is above threshold