Refactor the documentation update script. (#179)

cartographer/mapping/proto/sparse_pose_graph_options.proto

@@ -39,4 +39,4 @@ message SparsePoseGraphOptions {
   // Rate at which we sample a single trajectory's scans for global
   // localization.
   optional double global_sampling_ratio = 5;
-};
+}

cartographer/mapping_2d/scan_matching/proto/fast_correlative_scan_matcher_options.proto

@@ -27,4 +27,4 @@ message FastCorrelativeScanMatcherOptions {
 
   // Number of precomputed grids to use.
   optional int32 branch_and_bound_depth = 2;
-};
+}

cartographer/sensor/proto/adaptive_voxel_filter_options.proto

@@ -26,4 +26,4 @@ message AdaptiveVoxelFilterOptions {
 
   // Points further away from the origin are removed.
   optional float max_range = 3;
-};
+}

docs/source/configuration.rst

@@ -20,7 +20,7 @@ Configuration
 .. needed and run scripts/update_configuration_doc.py.
 
 cartographer.common.proto.CeresSolverOptions
---------------------------------------------
+============================================
 
 bool use_nonmonotonic_steps
   Configure the Ceres solver. See the Ceres documentation for more
@@ -34,7 +34,7 @@ int32 num_threads
 
 
 cartographer.kalman_filter.proto.PoseTrackerOptions
----------------------------------------------------
+===================================================
 
 double position_model_variance
   Model variances depend linearly on time.
@@ -57,7 +57,7 @@ int32 num_odometry_states
 
 
 cartographer.mapping.proto.MapBuilderOptions
---------------------------------------------
+============================================
 
 bool use_trajectory_builder_2d
   Not yet documented.
@@ -79,7 +79,7 @@ SparsePoseGraphOptions sparse_pose_graph_options
 
 
 cartographer.mapping.proto.SparsePoseGraphOptions
--------------------------------------------------
+=================================================
 
 int32 optimize_every_n_scans
   Online loop closure: If positive, will run the loop closure while the map
@@ -101,7 +101,7 @@ double global_sampling_ratio
 
 
 cartographer.mapping.sparse_pose_graph.proto.ConstraintBuilderOptions
----------------------------------------------------------------------
+=====================================================================
 
 double sampling_ratio
   A constraint will be added if the proportion of added constraints to
@@ -140,7 +140,7 @@ mapping_3d.scan_matching.proto.CeresScanMatcherOptions ceres_scan_matcher_option
 
 
 cartographer.mapping.sparse_pose_graph.proto.OptimizationProblemOptions
------------------------------------------------------------------------
+=======================================================================
 
 double huber_scale
   Scaling parameter for Huber loss function.
@@ -165,7 +165,7 @@ common.proto.CeresSolverOptions ceres_solver_options
 
 
 cartographer.mapping_2d.proto.LaserFanInserterOptions
------------------------------------------------------
+=====================================================
 
 double hit_probability
   Probability change for a hit (this will be converted to odds and therefore
@@ -181,7 +181,7 @@ bool insert_free_space
 
 
 cartographer.mapping_2d.proto.LocalTrajectoryBuilderOptions
------------------------------------------------------------
+===========================================================
 
 float laser_min_range
   Laser returns outside these ranges will be dropped.
@@ -238,7 +238,7 @@ bool use_imu_data
 
 
 cartographer.mapping_2d.proto.SubmapsOptions
---------------------------------------------
+============================================
 
 double resolution
   Resolution of the map in meters.
@@ -259,7 +259,7 @@ LaserFanInserterOptions laser_fan_inserter_options
 
 
 cartographer.mapping_2d.scan_matching.proto.CeresScanMatcherOptions
--------------------------------------------------------------------
+===================================================================
 
 double occupied_space_weight
   Scaling parameters for each cost functor.
@@ -279,7 +279,7 @@ common.proto.CeresSolverOptions ceres_solver_options
 
 
 cartographer.mapping_2d.scan_matching.proto.FastCorrelativeScanMatcherOptions
------------------------------------------------------------------------------
+=============================================================================
 
 double linear_search_window
   Minimum linear search window in which the best possible scan alignment
@@ -294,7 +294,7 @@ int32 branch_and_bound_depth
 
 
 cartographer.mapping_2d.scan_matching.proto.RealTimeCorrelativeScanMatcherOptions
----------------------------------------------------------------------------------
+=================================================================================
 
 double linear_search_window
   Minimum linear search window in which the best possible scan alignment
@@ -312,7 +312,7 @@ double rotation_delta_cost_weight
 
 
 cartographer.mapping_3d.proto.KalmanLocalTrajectoryBuilderOptions
------------------------------------------------------------------
+=================================================================
 
 bool use_online_correlative_scan_matching
   Whether to solve the online scan matching first using the correlative scan
@@ -332,7 +332,7 @@ double odometer_rotational_variance
 
 
 cartographer.mapping_3d.proto.LaserFanInserterOptions
------------------------------------------------------
+=====================================================
 
 double hit_probability
   Probability change for a hit (this will be converted to odds and therefore
@@ -348,11 +348,11 @@ int32 num_free_space_voxels
 
 
 cartographer.mapping_3d.proto.LocalTrajectoryBuilderOptions
------------------------------------------------------------
+===========================================================
 
 
 cartographer.mapping_3d.proto.MotionFilterOptions
--------------------------------------------------
+=================================================
 
 double max_time_seconds
   Threshold above which a new scan is inserted based on time.
@@ -365,7 +365,7 @@ double max_angle_radians
 
 
 cartographer.mapping_3d.proto.OptimizingLocalTrajectoryBuilderOptions
----------------------------------------------------------------------
+=====================================================================
 
 double high_resolution_grid_weight
   Not yet documented.
@@ -390,7 +390,7 @@ double odometry_rotation_weight
 
 
 cartographer.mapping_3d.proto.SubmapsOptions
---------------------------------------------
+============================================
 
 double high_resolution
   Resolution of the 'high_resolution' map in meters used for local SLAM and
@@ -414,7 +414,7 @@ LaserFanInserterOptions laser_fan_inserter_options
 
 
 cartographer.mapping_3d.scan_matching.proto.CeresScanMatcherOptions
--------------------------------------------------------------------
+===================================================================
 
 double translation_weight
   Scaling parameters for each cost functor.
@@ -434,7 +434,7 @@ common.proto.CeresSolverOptions ceres_solver_options
 
 
 cartographer.mapping_3d.scan_matching.proto.FastCorrelativeScanMatcherOptions
------------------------------------------------------------------------------
+=============================================================================
 
 int32 branch_and_bound_depth
   Number of precomputed grids to use.
@@ -463,7 +463,7 @@ double angular_search_window
 
 
 cartographer.sensor.proto.AdaptiveVoxelFilterOptions
-----------------------------------------------------
+====================================================
 
 float max_length
   'max_length' of a voxel edge.

scripts/update_configuration_doc.py

@@ -47,44 +47,82 @@ SUFFIX = """
 """
 NODOC = 'Not yet documented.'
 
-def GenerateDocumentation(output_dict, proto_file_name):
+
-  copyright = True
+def ForEachProtoFile(root, callback):
-  message = None
+  for dirpath, dirnames, filenames in os.walk(root):
-  content = [];
+    for name in filenames:
-  package = None
+      if name.endswith('.proto'):
-  multiline = None
+        path = os.path.join(dirpath, name)
-  print("Reading '%s'..." % proto_file_name)
+        print("Found '%s'..." % path)
-  for line in io.open(proto_file_name, encoding='UTF-8'):
+        assert not os.path.islink(path)
+        callback(io.open(path, encoding='UTF-8'))
+
+
+class Message(object):
+  def __init__(self, name, preceding_comments):
+    self.name = name
+    self.preceding_comments = preceding_comments
+    self.trailing_comments = None
+    self.options = []
+
+  def AddTrailingComments(self, comments):
+    self.trailing_comments = comments
+
+  def AddOption(self, name, comments):
+    self.options.append((name, comments))
+
+
+def ParseProtoFile(proto_file):
+  """Computes the list of Message objects of the option messages in a file."""
+  line_iter = iter(proto_file)
+
+  # We ignore the license header and search for the 'package' line.
+  for line in line_iter:
     line = line.strip()
-    if copyright:
-      if not line.startswith('//'):
-        copyright = False
-      continue
-    if package is None:
     if line.startswith('package'):
       assert line[-1] == ';'
       package = line[7:-1].strip()
-      continue
+      break
-    if line.startswith('//'):
+    else:
-      content_line = line[2:].strip()
+      assert '}' not in line
-      if not content_line.startswith('NEXT ID:'):
+
-        content.append(content_line)
+  message_list = []
-      continue
+  while True:
-    if message is None:
+    # Search for the next options message and capture preceding comments.
-      if line.startswith('message') and line.endswith('Options {'):
+    message_comments = []
-        message = package + '.' + line[7:-1].strip()
+    for line in line_iter:
-        print(" Found '%s'." % message)
+      line = line.strip()
-        assert message not in output_dict
+      if '}' in line:
-        message_list = [message, '=' * len(message), '']
+        # The preceding comments were for a different message it seems.
-        output_dict[message] = message_list
+        message_comments = []
-        message_list.extend(content)
+      elif line.startswith('//'):
-        content = []
+        # We keep comments preceding an options message.
-      continue
+        comment = line[2:].strip()
-    elif line.endswith('}'):
+        if not comment.startswith('NEXT ID:'):
-      message_list.extend(content)
+          message_comments.append(comment)
-      content = []
+      elif line.startswith('message') and line.endswith('Options {'):
-      message_list = None
+        message_name = package + '.' + line[7:-1].strip()
-      message = None
+        break
+    else:
+      # We reached the end of file.
+      break
+    print(" Found '%s'." % message_name)
+    message = Message(message_name, message_comments)
+    message_list.append(message)
+
+    # We capture the contents of this message.
+    option_comments = []
+    multiline = None
+    for line in line_iter:
+      line = line.strip()
+      if '}' in line:
+        # We reached the end of this message.
+        message.AddTrailingComments(option_comments)
+        break
+      elif line.startswith('//'):
+        comment = line[2:].strip()
+        if not comment.startswith('NEXT ID:'):
+          option_comments.append(comment)
       else:
         assert not line.startswith('required')
         if multiline is None:
@@ -97,30 +135,43 @@ def GenerateDocumentation(output_dict, proto_file_name):
         if not multiline.endswith(';'):
           continue
         assert len(multiline) < 200
-      option = multiline[8:-1].strip().rstrip('0123456789').strip()
+        option_name = multiline[8:-1].strip().rstrip('0123456789').strip()
-      assert option.endswith('=')
+        assert option_name.endswith('=')
-      option = option[:-1].strip();
+        option_name = option_name[:-1].strip();
-      print("  Option '%s'." % option)
+        print("  Option '%s'." % option_name)
         multiline = None
-      message_list.append(option)
+        message.AddOption(option_name, option_comments)
-      if len(content) == 0:
+        option_comments = []
-        content.append(NODOC)
+
-      for option_description_line in content:
+  return message_list
-        message_list.append('  ' + option_description_line)
+
-      content = []
+
-      message_list.append('')
+def GenerateDocumentation(output_dict, proto_file):
+  for message in ParseProtoFile(proto_file):
+    content = [message.name, '=' * len(message.name), '']
+    assert message.name not in output_dict
+    output_dict[message.name] = content
+    if message.preceding_comments:
+      content.extend(preceding_comments)
+      content.append('')
+    for option_name, option_comments in message.options:
+      content.append(option_name)
+      if not option_comments:
+        option_comments.append(NODOC)
+      for comment in option_comments:
+        content.append('  ' + comment)
+      content.append('')
+    if message.trailing_comments:
+      content.extend(message.trailing_comments)
+      content.append('')
 
 
 def GenerateDocumentationRecursively(output_file, root):
   """Recursively generates documentation, sorts and writes it."""
   output_dict = {}
-  for root, dirs, files in os.walk(root):
+  def callback(proto_file):
-    for name in files:
+    GenerateDocumentation(output_dict, proto_file)
-      if name.endswith('.proto'):
+  ForEachProtoFile(root, callback)
-        path = os.path.join(root, name)
-        assert not os.path.islink(path)
-        GenerateDocumentation(output_dict, path)
-
   output = ['\n'.join(doc) for key, doc in sorted(list(output_dict.items()))]
   print('\n\n'.join(output), file=output_file)