Windows build fixes & Azure CI (#1463)

This PR makes libcartographer build on Windows (including tests). Abseil was bumped to avoid a MSVC compiler bug.

I have observed two tests failing:
`MapBuilderTestByGridType/MapBuilderTestByGridType.GlobalSlam2D/1` and `MapBuilderTestByGridType/MapBuilderTestByGridType.LocalizationOnFrozenTrajectory2D/1`.

CMakeLists.txt

@@ -30,18 +30,32 @@ google_initialize_cartographer_project()
 google_enable_testing()
 
 find_package(Abseil REQUIRED)
-find_package(Boost REQUIRED COMPONENTS iostreams)
+set(BOOST_COMPONENTS iostreams)
+if(WIN32)
+  list(APPEND BOOST_COMPONENTS zlib)
+  set(Boost_USE_STATIC_LIBS FALSE)
+endif()
+find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS})
 find_package(Ceres REQUIRED COMPONENTS SuiteSparse)
 find_package(Eigen3 REQUIRED)
 find_package(LuaGoogle REQUIRED)
+# On Windows, Protobuf is incorrectly found by the bundled CMake module. Prefer native CMake config, if possible.
+set(protobuf_MODULE_COMPATIBLE TRUE CACHE INTERNAL "")
+find_package(Protobuf 3.0.0 CONFIG)
+if (NOT Protobuf_FOUND)
   find_package(Protobuf 3.0.0 REQUIRED)
+endif()
 
 if (${BUILD_GRPC})
   find_package(async_grpc REQUIRED)
 endif()
 
 include(FindPkgConfig)
+if (NOT WIN32)
   PKG_SEARCH_MODULE(CAIRO REQUIRED cairo>=1.12.16)
+else()
+  find_library(CAIRO_LIBRARIES cairo)
+endif()
 
 # Only build the documentation if we can find Sphinx.
 find_package(Sphinx)
@@ -124,7 +138,7 @@ foreach(ABS_FIL ${ALL_PROTOS})
   add_custom_command(
     OUTPUT "${PROJECT_BINARY_DIR}/${DIR}/${FIL_WE}.pb.cc"
            "${PROJECT_BINARY_DIR}/${DIR}/${FIL_WE}.pb.h"
-    COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
+    COMMAND  ${Protobuf_PROTOC_EXECUTABLE}
     ARGS --cpp_out  ${PROJECT_BINARY_DIR} -I
       ${PROJECT_SOURCE_DIR} ${ABS_FIL}
     DEPENDS ${ABS_FIL}
@@ -150,7 +164,7 @@ if(${BUILD_GRPC})
     add_custom_command(
       OUTPUT "${PROJECT_BINARY_DIR}/${DIR}/${FIL_WE}.pb.cc"
              "${PROJECT_BINARY_DIR}/${DIR}/${FIL_WE}.pb.h"
-      COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
+      COMMAND  ${Protobuf_PROTOC_EXECUTABLE}
       ARGS --cpp_out  ${PROJECT_BINARY_DIR}
         -I ${PROJECT_SOURCE_DIR}
         ${ABS_FIL}
@@ -165,7 +179,7 @@ if(${BUILD_GRPC})
 endif()
 set(INSTALL_GENERATED_HDRS ${ALL_PROTO_HDRS} ${ALL_GRPC_SERVICE_HDRS})
 
-add_library(${PROJECT_NAME} ${ALL_LIBRARY_HDRS} ${ALL_LIBRARY_SRCS})
+add_library(${PROJECT_NAME} STATIC ${ALL_LIBRARY_HDRS} ${ALL_LIBRARY_SRCS})
 
 configure_file(
   ${PROJECT_SOURCE_DIR}/cartographer/common/config.h.cmake
@@ -214,19 +228,38 @@ target_include_directories(${PROJECT_NAME} SYSTEM PUBLIC
   "${Boost_INCLUDE_DIRS}")
 target_link_libraries(${PROJECT_NAME} PUBLIC ${Boost_LIBRARIES})
 
-# We expect find_package(Ceres) to have located these for us.
+if (WIN32)
-target_link_libraries(${PROJECT_NAME} PUBLIC glog)
+  find_package(glog REQUIRED)
-target_link_libraries(${PROJECT_NAME} PUBLIC gflags)
+  set(GLOG_LIBRARY glog::glog)
+else()
+  set(GLOG_LIBRARY glog)
+endif()
 
+target_link_libraries(${PROJECT_NAME} PUBLIC ${GLOG_LIBRARY})
+target_link_libraries(${PROJECT_NAME} PUBLIC gflags)
+if(WIN32)
+  # Needed to fix conflict with MSVC's error macro.
+  target_compile_definitions(${PROJECT_NAME} PUBLIC -DGLOG_NO_ABBREVIATED_SEVERITIES)
+endif()
+if(MSVC)
+  # Needed for VS 2017 5.8
+  target_compile_definitions(${PROJECT_NAME} PUBLIC -D_ENABLE_EXTENDED_ALIGNED_STORAGE -D_USE_MATH_DEFINES)
+endif()
+
+if("${CAIRO_INCLUDE_DIRS}")
   target_include_directories(${PROJECT_NAME} SYSTEM PUBLIC
     "${CAIRO_INCLUDE_DIRS}")
+endif()
 target_link_libraries(${PROJECT_NAME} PUBLIC ${CAIRO_LIBRARIES})
 
 target_include_directories(${PROJECT_NAME} SYSTEM PUBLIC
   ${PROTOBUF_INCLUDE_DIR})
 # TODO(hrapp): This should not explicitly list pthread and use
 # PROTOBUF_LIBRARIES, but that failed on first try.
-target_link_libraries(${PROJECT_NAME} PUBLIC ${PROTOBUF_LIBRARY} pthread standalone_absl)
+target_link_libraries(${PROJECT_NAME} PUBLIC ${PROTOBUF_LIBRARY} standalone_absl)
+if (NOT WIN32)
+  target_link_libraries(${PROJECT_NAME} PUBLIC pthread)
+endif()
 if(${BUILD_GRPC})
   target_link_libraries(${PROJECT_NAME} PUBLIC grpc++)
   target_link_libraries(${PROJECT_NAME} PUBLIC async_grpc)
@@ -246,6 +279,10 @@ set(TEST_LIB
 add_library(${TEST_LIB} ${TEST_LIBRARY_HDRS} ${TEST_LIBRARY_SRCS})
 target_include_directories(${TEST_LIB} SYSTEM PRIVATE
   "${GMOCK_INCLUDE_DIRS}")
+# Needed for dynamically linked GTest on Windows.
+if (WIN32)
+  target_compile_definitions(${TEST_LIB} PUBLIC -DGTEST_LINKED_AS_SHARED_LIBRARY)
+endif()
 target_link_libraries(${TEST_LIB} PUBLIC ${GMOCK_LIBRARY})
 target_link_libraries(${TEST_LIB} PUBLIC ${PROJECT_NAME})
 set_target_properties(${TEST_LIB} PROPERTIES

azure-pipelines.yml

@@ -0,0 +1,60 @@
+# Copyright 2018 The Cartographer Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+jobs:
+- job: Build
+  pool:
+    vmImage: 'vs2017-win2016'
+  timeoutInMinutes: 360
+  steps:
+  - script: |
+      choco sources add -n=roswin -s https://roswin.azurewebsites.net/api/v2/ --priority 1
+      rem Azure VM runs out of space on C:, so use D: for ros and rosdeps
+      mkdir D:\opt && mklink /J C:\opt D:\opt
+      choco upgrade %ROS_METAPACKAGE% -y
+      robocopy "." ".\src\cartographer" /E /MOVE /XD "src" > NUL
+      call "C:\opt\ros\melodic\x64\env.bat" rosdep install --from-paths src --ignore-src -r -y
+    env:
+      ROS_METAPACKAGE: 'ros-melodic-desktop'
+    displayName: Install prerequisites
+
+  - script: |
+      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+      call "C:\opt\ros\melodic\x64\setup.bat"
+      call src\cartographer\scripts\remove_mingw_cygwin_from_path.bat
+      catkin_make_isolated --use-ninja --install
+    displayName: Build
+
+  - script: |
+      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+      call "C:\opt\ros\melodic\x64\setup.bat"
+      call src\cartographer\scripts\remove_mingw_cygwin_from_path.bat
+      cd build_isolated\cartographer\install && ctest --no-compress-output -T Test
+    displayName: Run tests
+
+  - script: |
+      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+      call "C:\opt\ros\melodic\x64\setup.bat"
+      call src\cartographer\scripts\remove_mingw_cygwin_from_path.bat
+      python src\cartographer\scripts\ctest_to_junit.py build_isolated\cartographer\install
+    displayName: Convert tests to jUnit
+    condition: always()
+
+  - task: PublishTestResults@2
+    displayName: Publish test results
+    inputs:
+      testRunner: 'jUnit'
+      testResultsFiles: '**\jUnit.xml'
+      searchFolder: '$(Build.SourcesDirectory)\build_isolated\cartographer\install\Testing'
+    condition: always()

cartographer-config.cmake.in

@@ -36,6 +36,9 @@ endif()
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CARTOGRAPHER_CMAKE_DIR}/modules)
 
 find_package(Ceres ${QUIET_OR_REQUIRED_OPTION} HINTS ${CERES_DIR_HINTS})
+if (WIN32)
+    find_package(glog REQUIRED)
+endif()
 find_package(Abseil ${QUIET_OR_REQUIRED_OPTION})
 if(CARTOGRAPHER_HAS_GRPC)
    find_package(async_grpc ${QUIET_OR_REQUIRED_OPTION})

cartographer/common/internal/testing/thread_pool_for_testing.cc

@@ -16,7 +16,9 @@
 
 #include "cartographer/common/internal/testing/thread_pool_for_testing.h"
 
+#ifndef WIN32
 #include <unistd.h>
+#endif
 #include <algorithm>
 #include <chrono>
 #include <numeric>

cartographer/common/thread_pool.cc

@@ -16,7 +16,9 @@
 
 #include "cartographer/common/thread_pool.h"
 
+#ifndef WIN32
 #include <unistd.h>
+#endif
 #include <algorithm>
 #include <chrono>
 #include <numeric>

cartographer/common/time.cc

@@ -56,10 +56,14 @@ common::Duration FromMilliseconds(const int64 milliseconds) {
 }
 
 double GetThreadCpuTimeSeconds() {
+#ifndef WIN32
   struct timespec thread_cpu_time;
   CHECK(clock_gettime(CLOCK_THREAD_CPUTIME_ID, &thread_cpu_time) == 0)
       << std::strerror(errno);
   return thread_cpu_time.tv_sec + 1e-9 * thread_cpu_time.tv_nsec;
+#else
+  return 0.;
+#endif
 }
 
 }  // namespace common

cartographer/io/proto_stream_test.cc

@@ -31,13 +31,7 @@ namespace {
 
 class ProtoStreamTest : public ::testing::Test {
  protected:
-  void SetUp() override {
+  void SetUp() override { test_directory_ = "."; }
-    const std::string tmpdir = P_tmpdir;
-    test_directory_ = tmpdir + "/proto_stream_test_XXXXXX";
-    ASSERT_NE(mkdtemp(&test_directory_[0]), nullptr) << strerror(errno);
-  }
-
-  void TearDown() override { remove(test_directory_.c_str()); }
 
   std::string test_directory_;
 };

cartographer/mapping/internal/3d/scan_matching/rotation_delta_cost_functor_3d_test.cc

@@ -16,6 +16,8 @@
 
 #include "cartographer/mapping/internal/3d/scan_matching/rotation_delta_cost_functor_3d.h"
 
+#include <array>
+
 #include "gtest/gtest.h"
 
 namespace cartographer {

cartographer/pose_graph/constraint/cost_function/acceleration_cost_3d_test.cc

@@ -16,6 +16,8 @@
 
 #include "cartographer/pose_graph/constraint/cost_function/acceleration_cost_3d.h"
 
+#include <array>
+
 #include "cartographer/testing/test_helpers.h"
 
 namespace cartographer {

cartographer/pose_graph/constraint/cost_function/rotation_cost_3d_test.cc

@@ -16,6 +16,8 @@
 
 #include "cartographer/pose_graph/constraint/cost_function/rotation_cost_3d.h"
 
+#include <array>
+
 #include "cartographer/testing/test_helpers.h"
 
 namespace cartographer {

cmake/functions.cmake

@@ -72,6 +72,10 @@ macro(google_initialize_cartographer_project)
     set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
         ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules)
   endif()
+
+  if(WIN32)
+    # TODO turn on equivalent warnings on Windows
+  else()
     set(GOOG_CXX_FLAGS "-pthread -std=c++11 -fPIC ${GOOG_CXX_FLAGS}")
 
     google_add_flag(GOOG_CXX_FLAGS "-Wall")
@@ -130,6 +134,7 @@ macro(google_initialize_cartographer_project)
       execute_process(COMMAND ${DETECT_CHANGES_CMD})
     endif()
     include(${FILES_LIST_PATH})
+  endif()
 endmacro()
 
 macro(google_enable_testing)

cmake/modules/FindAbseil.cmake

@@ -37,9 +37,9 @@ if(NOT TARGET standalone_absl)
     "${ABSEIL_PROJECT_BUILD_DIR}/absl/algorithm/${prefix}absl_algorithm${suffix}"
     "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_base${suffix}"
     "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_dynamic_annotations${suffix}"
-    "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_malloc_internal${suffix}"
+    "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_internal_malloc_internal${suffix}"
-    "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_spinlock_wait${suffix}"
+    "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_internal_spinlock_wait${suffix}"
-    "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_throw_delegate${suffix}"
+    "${ABSEIL_PROJECT_BUILD_DIR}/absl/base/${prefix}absl_internal_throw_delegate${suffix}"
     "${ABSEIL_PROJECT_BUILD_DIR}/absl/container/${prefix}absl_container${suffix}"
     "${ABSEIL_PROJECT_BUILD_DIR}/absl/container/${prefix}test_instance_tracker_lib${suffix}"
     "${ABSEIL_PROJECT_BUILD_DIR}/absl/debugging/${prefix}absl_debugging${suffix}"
@@ -64,7 +64,7 @@ if(NOT TARGET standalone_absl)
   ExternalProject_Add(${ABSEIL_PROJECT_NAME}
     PREFIX ${ABSEIL_PROJECT_NAME}
     GIT_REPOSITORY   https://github.com/abseil/abseil-cpp.git
-    GIT_TAG          5441bbe1db5d0f2ca24b5b60166367b0966790af
+    GIT_TAG          7b46e1d31a6b08b1c6da2a13e7b151a20446fa07
     INSTALL_COMMAND  ""
     BUILD_COMMAND    ${CMAKE_COMMAND} --build "${ABSEIL_PROJECT_BUILD_DIR}"
     CMAKE_CACHE_ARGS "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON;-DBUILD_TESTING:BOOL=OFF;-DCMAKE_BUILD_TYPE:STRING=Release"
@@ -81,6 +81,15 @@ if(NOT TARGET standalone_absl)
     INTERFACE_LINK_LIBRARIES
     "${ABSEIL_DEPENDENT_LIBRARIES}"
   )
+  if(MSVC)
+    # /wd4005  macro-redefinition
+    # /wd4068  unknown pragma
+    # /wd4244  conversion from 'type1' to 'type2'
+    # /wd4267  conversion from 'size_t' to 'type2'
+    # /wd4800  force value to bool 'true' or 'false' (performance warning)
+    target_compile_options(standalone_absl INTERFACE /wd4005 /wd4068 /wd4244 /wd4267 /wd4800)
+    target_compile_definitions(standalone_absl INTERFACE -DNOMINMAX -DWIN32_LEAN_AND_MEAN=1 -D_CRT_SECURE_NO_WARNINGS)
+  endif()
   add_dependencies(standalone_absl ${ABSEIL_PROJECT_NAME})
   unset(prefix)
   unset(suffix)

package.xml

@@ -37,7 +37,6 @@
   <buildtool_depend>catkin</buildtool_depend>
 
   <build_depend>git</build_depend>
-  <build_depend>g++-static</build_depend>
   <build_depend>google-mock</build_depend>
   <build_depend>python-sphinx</build_depend>
 

scripts/ctest_to_junit.py

@@ -0,0 +1,33 @@
+# Copyright 2018 The Cartographer Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from lxml import etree
+import StringIO
+import sys
+
+TAGfile = open(sys.argv[1]+"/Testing/TAG", 'r')
+dirname = TAGfile.readline().strip()
+
+xmlfile = open(sys.argv[1]+"/Testing/"+dirname+"/Test.xml", 'r')
+xslfile = open(sys.path[0] + "/ctest_to_junit.xsl", 'r')
+
+xmlcontent = xmlfile.read()
+xslcontent = xslfile.read()
+
+xmldoc = etree.parse(StringIO.StringIO(xmlcontent))
+xslt_root = etree.XML(xslcontent)
+transform = etree.XSLT(xslt_root)
+
+result_tree = transform(xmldoc)
+result_tree.write(sys.argv[1]+"/Testing/"+dirname+"/jUnit.xml")

scripts/ctest_to_junit.xsl

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2018 The Cartographer Authors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+	  http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+	<xsl:output method="xml" indent="yes" />
+	<xsl:template match="/">
+		<testsuites>
+			<xsl:variable name="buildName" select="//Site/@BuildName"/>
+			<xsl:variable name="numberOfTests" select="count(//Site/Testing/Test)"/>
+			<xsl:variable name="numberOfFailures" select="count(//Site/Testing/Test[@Status!='passed'])" />
+			<testsuite name="CTest"
+				tests="{$numberOfTests}" time="0"
+				failures="{$numberOfFailures}"  errors="0"
+				skipped="0">
+			<xsl:for-each select="//Site/Testing/Test">
+					<xsl:variable name="testName" select="translate(Name, '-', '_')"/>
+					<xsl:variable name="duration" select="Results/NamedMeasurement[@name='Execution Time']/Value"/>
+					<xsl:variable name="status" select="@Status"/>
+					<xsl:variable name="output" select="Results/Measurement/Value"/>
+					<xsl:variable name="className" select="translate(Path, '/.', '.')"/>
+					<testcase classname="projectroot{$className}"
+						name="{$testName}"
+						time="{$duration}">
+						<xsl:if test="@Status!='passed'">
+							<failure>
+								<xsl:value-of select="$output" />
+							</failure>
+						</xsl:if>
+						<system-out>
+							<xsl:value-of select="$output" />
+						</system-out>
+					</testcase>
+				</xsl:for-each>
+			</testsuite>
+		</testsuites>
+	</xsl:template>
+</xsl:stylesheet>

scripts/remove_mingw_cygwin_from_path.bat

@@ -0,0 +1,20 @@
+rem  Copyright 2018 The Cartographer Authors
+rem  
+rem  Licensed under the Apache License, Version 2.0 (the "License");
+rem  you may not use this file except in compliance with the License.
+rem  You may obtain a copy of the License at
+rem  
+rem       http://www.apache.org/licenses/LICENSE-2.0
+rem  
+rem  Unless required by applicable law or agreed to in writing, software
+rem  distributed under the License is distributed on an "AS IS" BASIS,
+rem  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem  See the License for the specific language governing permissions and
+rem  limitations under the License.
+
+rem  Remove git bash's MinGW/Cygwin stuff from PATH, as it interferes with CMake's
+rem  detection of Boost when using ninja.
+set PATH=%PATH:C:\tools\mingw64\bin;=%
+set PATH=%PATH:C:\Program Files\Git\bin;=%
+set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
+set PATH=%PATH:C:\Program Files\Git\mingw64\bin;=%