Unified pbstream tool (#1326)

Consolidating pbstream tools into a single tool

* adds two subcommands for now (info and migrate)
* removes commandline flags for filenames in favor of commandline args
* updates documentation for migration tool

CMakeLists.txt

@@ -180,14 +180,9 @@ google_binary(cartographer_compute_relations_metrics
     cartographer/ground_truth/compute_relations_metrics_main.cc
 )
 
-google_binary(cartographer_migrate_serialization_format
+google_binary(cartographer_pbstream
   SRCS
-    cartographer/io/migrate_serialization_format_main.cc
+  cartographer/io/pbstream_main.cc
-)
-
-google_binary(cartographer_pbstream_info
-  SRCS
-  cartographer/io/pbstream_info_main.cc
 )
 
 if(${BUILD_GRPC})

cartographer/io/internal/mapping_state_serialization.h

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #ifndef CARTOGRAPHER_IO_INTERNAL_MAPPING_STATE_SERIALIZATION_H_
 #define CARTOGRAPHER_IO_INTERNAL_MAPPING_STATE_SERIALIZATION_H_
 

cartographer/io/internal/pbstream_info.cc

@@ -14,17 +14,24 @@
  * limitations under the License.
  */
 
+#include "cartographer/io/internal/pbstream_info.h"
+
+#include <map>
+#include <sstream>
+#include <string>
+
 #include "cartographer/io/proto_stream.h"
 #include "cartographer/io/proto_stream_deserializer.h"
 #include "gflags/gflags.h"
 #include "glog/logging.h"
 
-DEFINE_string(pbstream_filename, "", "Proto stream file.");
 DEFINE_bool(all_debug_strings, false,
             "Print debug strings of all serialized data.");
 
+using cartographer::mapping::proto::SerializedData;
+
 namespace cartographer {
-namespace mapping {
+namespace io {
 namespace {
 
 void Run(const std::string& pbstream_filename, bool all_debug_strings) {
@@ -33,13 +40,13 @@ void Run(const std::string& pbstream_filename, bool all_debug_strings) {
   io::ProtoStreamDeserializer deserializer(&reader);
   const auto header = deserializer.header();
   LOG(INFO) << "Header: " << header.DebugString();
-  for (const proto::TrajectoryBuilderOptionsWithSensorIds& trajectory_options :
+  for (const mapping::proto::TrajectoryBuilderOptionsWithSensorIds&
-       deserializer.all_trajectory_builder_options()
+           trajectory_options : deserializer.all_trajectory_builder_options()
-           .options_with_sensor_ids()) {
+                                    .options_with_sensor_ids()) {
     LOG(INFO) << "Trajectory options: " << trajectory_options.DebugString();
   }
-  const proto::PoseGraph pose_graph = deserializer.pose_graph();
+  const mapping::proto::PoseGraph pose_graph = deserializer.pose_graph();
-  for (const proto::Trajectory& trajectory : pose_graph.trajectory()) {
+  for (const mapping::proto::Trajectory& trajectory : pose_graph.trajectory()) {
     LOG(INFO) << "Trajectory id: " << trajectory.trajectory_id()
               << " has #nodes " << trajectory.node_size() << " has #submaps "
               << trajectory.submap_size();
@@ -48,16 +55,15 @@ void Run(const std::string& pbstream_filename, bool all_debug_strings) {
     LOG(INFO) << "Pose graph: " << pose_graph.DebugString();
   }
 
-  const std::map<proto::SerializedData::DataCase, std::string>
+  const std::map<SerializedData::DataCase, std::string> data_case_to_name = {
-      data_case_to_name = {
+      {SerializedData::kSubmap, "submap"},
-          {proto::SerializedData::kSubmap, "submap"},
+      {SerializedData::kNode, "node"},
-          {proto::SerializedData::kNode, "node"},
+      {SerializedData::kTrajectoryData, "trajectory_data"},
-          {proto::SerializedData::kTrajectoryData, "trajectory_data"},
+      {SerializedData::kImuData, "imu_data"},
-          {proto::SerializedData::kImuData, "imu_data"},
+      {SerializedData::kOdometryData, "odometry_data"},
-          {proto::SerializedData::kOdometryData, "odometry_data"},
+      {SerializedData::kFixedFramePoseData, "fixed_frame_pose_data"},
-          {proto::SerializedData::kFixedFramePoseData, "fixed_frame_pose_data"},
+      {SerializedData::kLandmarkData, "landmark_data"},
-          {proto::SerializedData::kLandmarkData, "landmark_data"},
+  };
-      };
   // Initialize so zero counts of these are also reported.
   std::map<std::string, int> data_counts = {
       {"submap_2d", 0},
@@ -65,7 +71,7 @@ void Run(const std::string& pbstream_filename, bool all_debug_strings) {
       {"submap_3d", 0},
       {"submap_3d_high_resolution_hybrid_grid", 0},
   };
-  proto::SerializedData proto;
+  SerializedData proto;
   while (deserializer.ReadNextSerializedData(&proto)) {
     if (all_debug_strings) {
       LOG(INFO) << "Serialized data: " << proto.DebugString();
@@ -77,7 +83,7 @@ void Run(const std::string& pbstream_filename, bool all_debug_strings) {
     }
     const std::string& data_name = it->second;
     ++data_counts[data_name];
-    if (proto.data_case() == proto::SerializedData::kSubmap) {
+    if (proto.data_case() == SerializedData::kSubmap) {
       if (proto.mutable_submap()->has_submap_2d()) {
         ++data_counts["submap_2d"];
         if (proto.mutable_submap()->mutable_submap_2d()->has_grid()) {
@@ -100,23 +106,22 @@ void Run(const std::string& pbstream_filename, bool all_debug_strings) {
               << entry.second;
   }
 }
-
 }  // namespace
-}  // namespace mapping
-}  // namespace cartographer
 
-int main(int argc, char** argv) {
+int pbstream_info(int argc, char* argv[]) {
-  google::InitGoogleLogging(argv[0]);
+  std::stringstream ss;
-  FLAGS_logtostderr = true;
+  ss << "\n\n"
-  google::SetUsageMessage(
+     << "Reads a pbstream file and summarizes its contents.\n\n"
-      "\n\n"
+     << "Usage: " << argv[0] << " " << argv[1]
-      "Reads a pbstream file and summarizes its contents.\n");
+     << " <pbstream_filename> [flags]\n";
-  google::ParseCommandLineFlags(&argc, &argv, true);
+  google::SetUsageMessage(ss.str());
-
+  if (argc < 3) {
-  if (FLAGS_pbstream_filename.empty()) {
     google::ShowUsageWithFlagsRestrict(argv[0], "pbstream_info");
     return EXIT_FAILURE;
   }
-  ::cartographer::mapping::Run(FLAGS_pbstream_filename,
+  Run(argv[2], FLAGS_all_debug_strings);
-                               FLAGS_all_debug_strings);
+  return EXIT_SUCCESS;
 }
+
+}  // namespace io
+}  // namespace cartographer

cartographer/io/internal/pbstream_info.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2018 The Cartographer Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CARTOGRAPHER_IO_INTERNAL_PBSTREAM_INFO_H_
+#define CARTOGRAPHER_IO_INTERNAL_PBSTREAM_INFO_H_
+
+namespace cartographer {
+namespace io {
+
+// info subtool for pbstream swiss army knife. The command line arguments are
+// assumed to be parsed and removed from the remaining arguments already.
+int pbstream_info(int argc, char* argv[]);
+
+}  // namespace io
+}  // namespace cartographer
+
+#endif  // CARTOGRAPHER_IO_INTERNAL_PBSTREAM_INFO_H_

cartographer/io/internal/pbstream_migrate.cc

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2018 The Cartographer Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sstream>
+
+#include "cartographer/io/proto_stream.h"
+#include "cartographer/io/serialization_format_migration.h"
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+namespace cartographer {
+namespace io {
+
+int pbstream_migrate(int argc, char** argv) {
+  std::stringstream ss;
+  ss << "\n\nTool for migrating files that use the serialization output of "
+        "Cartographer 0.3, to the new serialization format, which includes a "
+        "header (Version 1)."
+     << "\nUsage: " << argv[0] << " " << argv[1]
+     << " <input_filename> <output_filename>";
+  google::SetUsageMessage(ss.str());
+
+  if (argc < 4) {
+    google::ShowUsageWithFlagsRestrict(argv[0], "pbstream_migrate");
+    return EXIT_FAILURE;
+  }
+  cartographer::io::ProtoStreamReader input(argv[2]);
+  cartographer::io::ProtoStreamWriter output(argv[3]);
+  LOG(INFO) << "Migrating old serialization format in \"" << argv[2]
+            << "\" to new serialization format in \"" << argv[3] << "\"";
+  cartographer::io::MigrateStreamFormatToVersion1(&input, &output);
+
+  return EXIT_SUCCESS;
+}
+
+}  // namespace io
+}  // namespace cartographer

cartographer/io/internal/pbstream_migrate.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2018 The Cartographer Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CARTOGRAPHER_IO_INTERNAL_PBSTREAM_MIGRATE_H_
+#define CARTOGRAPHER_IO_INTERNAL_PBSTREAM_MIGRATE_H_
+
+namespace cartographer {
+namespace io {
+
+// 'pbstream migrate' entry point. Commandline flags are assumed to be already
+// parsed and removed from the remaining arguments.
+int pbstream_migrate(int argc, char** argv);
+
+}  // namespace io
+}  // namespace cartographer
+
+#endif  // CARTOGRAPHER_IO_INTERNAL_PBSTREAM_MIGRATE_H_

cartographer/io/migrate_serialization_format_main.cc

@@ -1,52 +0,0 @@
-/*
- * Copyright 2018 The Cartographer Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "cartographer/io/proto_stream.h"
-#include "cartographer/io/serialization_format_migration.h"
-#include "gflags/gflags.h"
-#include "glog/logging.h"
-
-DEFINE_string(
-    original_pbstream_file, "",
-    "Path to the pbstream file that will be migrated to the new version.");
-DEFINE_string(output_pbstream_file, "",
-              "Output filename for the migrated pbstream.");
-
-int main(int argc, char** argv) {
-  google::InitGoogleLogging(argv[0]);
-  FLAGS_logtostderr = true;
-  google::SetUsageMessage(
-      "\n\n"
-      "Tool for migrating files that use the serialization output of "
-      "Cartographer 0.3, to the new serialization format, which includes a "
-      "header (Version 1).");
-  google::ParseCommandLineFlags(&argc, &argv, true);
-
-  if (FLAGS_original_pbstream_file.empty() ||
-      FLAGS_output_pbstream_file.empty()) {
-    google::ShowUsageWithFlagsRestrict(argv[0], "migrate_serialization_format");
-    return EXIT_FAILURE;
-  }
-  cartographer::io::ProtoStreamReader input(FLAGS_original_pbstream_file);
-  cartographer::io::ProtoStreamWriter output(FLAGS_output_pbstream_file);
-  LOG(INFO) << "Migrating old serialization format in \""
-            << FLAGS_original_pbstream_file
-            << "\" to new serialization format in \""
-            << FLAGS_output_pbstream_file << "\"";
-  cartographer::io::MigrateStreamFormatToVersion1(&input, &output);
-
-  return EXIT_SUCCESS;
-}

cartographer/io/pbstream_main.cc

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 The Cartographer Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <unordered_set>
+
+#include "cartographer/io/internal/pbstream_info.h"
+#include "cartographer/io/internal/pbstream_migrate.h"
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+int main(int argc, char** argv) {
+  google::InitGoogleLogging(argv[0]);
+
+  FLAGS_logtostderr = true;
+  google::SetUsageMessage(
+      "Swiss Army knife for pbstreams.\n\n"
+      "Currently supported subcommands are:\n"
+      "\tinfo    - Prints summary of pbstream.\n"
+      "\tmigrate - Migrates old pbstream (w/o header) to new pbstream format.");
+  google::ParseCommandLineFlags(&argc, &argv, true);
+
+  if (argc < 2) {
+    google::ShowUsageWithFlagsRestrict(argv[0], "pbstream_info_main");
+    return EXIT_FAILURE;
+  } else if (std::string(argv[1]) == "info") {
+    return ::cartographer::io::pbstream_info(argc, argv);
+  } else if (std::string(argv[1]) == "migrate") {
+    return ::cartographer::io::pbstream_migrate(argc, argv);
+  } else {
+    LOG(INFO) << "Unknown subtool: \"" << argv[1];
+    google::ShowUsageWithFlagsRestrict(argv[0], "pbstream_info_main");
+    return EXIT_FAILURE;
+  }
+}

docs/source/pbstream_migration.rst

@@ -24,16 +24,14 @@ In order to enable users to reuse previously generated pbstream files, we
 provide a migration tool which converts pbstreams from Cartographer 0.3 to the
 new serialization format used in Cartographer 1.0.
 
-The tool is shipped as part of Cartographer (`source`_) and once built can be
+The tool is shipped as part of Cartographer's pbstream tool (`source`_) and once
-invoked as follows:::
+built can be invoked as follows:::
 
-  cartographer_migrate_serialization_format \
+  cartographer_pbstream migrate old.pbstream new.pbstream
-        --original_pbstream_file=old.pbstream \
-        --output_pbstream_file=new.pbstream
 
-The tool assumes that the pbstream provided via the ``--original_pbstream_file``
+The tool assumes that the first pbstream provided as commandline argument,
-argument, follows the serialization format of Cartographer 0.3. The resulting
+follows the serialization format of Cartographer 0.3. The resulting
-1.0 pbstream will be saved to the ``--output_pbstream_file`` location.
+1.0 pbstream will be saved to the second commandline argument location.
 
 .. _RFC-0021: https://github.com/googlecartographer/rfcs/blob/master/text/0021-serialization-format.md
-.. _source: https://github.com/googlecartographer/cartographer/blob/master/cartographer/io/migrate_serialization_format_main.cc
+.. _source: https://github.com/googlecartographer/cartographer/blob/master/cartographer/io/pbstream_main.cc