diff --git a/gtsam/inference/EliminationTree-inst.h b/gtsam/inference/EliminationTree-inst.h
index 1ce98a435..d8791d1ba 100644
--- a/gtsam/inference/EliminationTree-inst.h
+++ b/gtsam/inference/EliminationTree-inst.h
@@ -149,9 +149,9 @@ namespace gtsam {
       if(parents[j] == none)
         roots_.push_back(nodes[j]);
 
-    // Gather remaining factors
+    // Gather remaining factors (exclude null factors)
     for(size_t i = 0; i < m; ++i)
-      if(!factorUsed[i])
+      if(!factorUsed[i] && graph[i])
         remainingFactors_.push_back(graph[i]);
   }