diff --git a/gtsam/inference/BayesTree-inl.h b/gtsam/inference/BayesTree-inl.h
index 5eeb619e3..6178b6446 100644
--- a/gtsam/inference/BayesTree-inl.h
+++ b/gtsam/inference/BayesTree-inl.h
@@ -133,7 +133,6 @@ namespace gtsam {
   void BayesTree<CONDITIONAL>::Clique::permuteWithInverse(const Permutation& inversePermutation) {
     BayesNet<CONDITIONAL>::permuteWithInverse(inversePermutation);
     BOOST_FOREACH(Index& separatorKey, separator_) { separatorKey = inversePermutation[separatorKey]; }
-    if(cachedFactor_) cachedFactor_->permuteWithInverse(inversePermutation);
     BOOST_FOREACH(const shared_ptr& child, children_) {
       child->permuteWithInverse(inversePermutation);
     }
@@ -153,7 +152,6 @@ namespace gtsam {
 #endif
     if(changed) {
       BOOST_FOREACH(Index& separatorKey, separator_) { separatorKey = inversePermutation[separatorKey]; }
-      if(cachedFactor_) cachedFactor_->permuteWithInverse(inversePermutation);
       BOOST_FOREACH(const shared_ptr& child, children_) {
         (void)child->permuteSeparatorWithInverse(inversePermutation);
       }
diff --git a/gtsam/inference/BayesTree.h b/gtsam/inference/BayesTree.h
index 220f1b45f..a5ad7732e 100644
--- a/gtsam/inference/BayesTree.h
+++ b/gtsam/inference/BayesTree.h
@@ -58,7 +58,6 @@ namespace gtsam {
 			weak_ptr parent_;
 			std::list<shared_ptr> children_;
 			std::list<Index> separator_; /** separator keys */
-			typename CONDITIONAL::Factor::shared_ptr cachedFactor_;
 
 			friend class BayesTree<CONDITIONAL>;
 
@@ -87,9 +86,6 @@ namespace gtsam {
 			std::list<shared_ptr>& children() { return children_; }
 			const std::list<shared_ptr>& children() const { return children_; }
 
-			/** Reference the cached factor */
-			typename CONDITIONAL::Factor::shared_ptr& cachedFactor() { return cachedFactor_; }
-
 			/** The size of subtree rooted at this clique, i.e., nr of Cliques */
 			size_t treeSize() const;
 
diff --git a/gtsam/inference/JunctionTree-inl.h b/gtsam/inference/JunctionTree-inl.h
index c9938f330..0278f4f98 100644
--- a/gtsam/inference/JunctionTree-inl.h
+++ b/gtsam/inference/JunctionTree-inl.h
@@ -188,7 +188,6 @@ namespace gtsam {
 		BOOST_FOREACH(typename BayesTree::sharedClique& childRoot, children)
 			childRoot->parent_ = new_clique;
 
-		new_clique->cachedFactor() = jointFactor;
     toc("JT 2.4 Update tree");
 		return make_pair(new_clique, jointFactor);
 	}