diff --git a/gtsam/inference/Ordering.cpp b/gtsam/inference/Ordering.cpp
index b0ac9669f..4f4b14bb5 100644
--- a/gtsam/inference/Ordering.cpp
+++ b/gtsam/inference/Ordering.cpp
@@ -163,7 +163,7 @@ namespace gtsam {
         ++ group;
     }
 
-    if(!forceOrder)
+    if(!forceOrder && !constrainFirst.empty())
       ++ group;
     BOOST_FOREACH(int& c, cmember)
       if(c == none)