diff --git a/gtsam/hybrid/GaussianMixture.cpp b/gtsam/hybrid/GaussianMixture.cpp
index 0a0332af8..228e851bb 100644
--- a/gtsam/hybrid/GaussianMixture.cpp
+++ b/gtsam/hybrid/GaussianMixture.cpp
@@ -87,7 +87,18 @@ GaussianFactorGraphTree GaussianMixture::add(
 
 /* *******************************************************************************/
 GaussianFactorGraphTree GaussianMixture::asGaussianFactorGraphTree() const {
-  auto wrap = [](const GaussianConditional::shared_ptr &gc) {
+  auto wrap = [this](const GaussianConditional::shared_ptr &gc) {
+    const double Cgm_Kgcm = this->logConstant_ - gc->logNormalizationConstant();
+    // If there is a difference in the covariances, we need to account for that
+    // since the error is dependent on the mode.
+    if (Cgm_Kgcm > 0.0) {
+      // We add a constant factor which will be used when computing
+      // the probability of the discrete variables.
+      Vector c(1);
+      c << std::sqrt(2.0 * Cgm_Kgcm);
+      auto constantFactor = std::make_shared<JacobianFactor>(c);
+      return GaussianFactorGraph{gc, constantFactor};
+    }
     return GaussianFactorGraph{gc};
   };
   return {conditionals_, wrap};