Another attempt at fixing installation of _libgtsam_python.so. package_data is relative to package_dir, so the previous approach doesn't work when package_dir is in the source tree (and we don't want to copy the lib to source, or all of the source into lib). Using data_files method instead.

python/CMakeLists.txt

@@ -57,6 +57,10 @@ if(Boost_PYTHON${BOOST_PYTHON_VERSION_SUFFIX_UPPERCASE}_FOUND AND PYTHONLIBS_FOU
     set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in")
     set(SETUP_PY    "${CMAKE_CURRENT_BINARY_DIR}/setup.py")
 
+	# Hacky way to figure out install folder - valid for Linux & Mac
+	# default pattern: prefix/lib/pythonX.Y/site-packages from https://docs.python.org/2/install/ 
+	STRING(CONCAT PY_INSTALL_FOLDER ${CMAKE_INSTALL_PREFIX} "/lib/python" ${PYTHON_VERSION_MAJOR} "." ${PYTHON_VERSION_MINOR} "/site-packages") 
+
     configure_file(${SETUP_PY_IN} ${SETUP_PY})
 
 	# TODO(frank): possibly support a different prefix a la matlab wrapper

python/setup.py.in

@@ -10,5 +10,6 @@ setup(name='gtsam',
       url='https://collab.cc.gatech.edu/borg/gtsam',
       package_dir={ '': '${CMAKE_CURRENT_SOURCE_DIR}' },
       packages=['gtsam', 'gtsam_utils', 'gtsam_examples', 'gtsam_tests'],
-      package_data={'gtsam' : ['_libgtsam_python.so']},
+      #package_data={'gtsam' : ['_libgtsam_python.so']},  # location of .so file is relative to package_dir
+      data_files=[('${PY_INSTALL_FOLDER}/gtsam/', ['gtsam/_libgtsam_python.so'])],  # location of .so file relative to setup.py
      )