Refactor the documentation update script. (#179)

cartographer/mapping/proto/sparse_pose_graph_options.proto

@@ -39,4 +39,4 @@ message SparsePoseGraphOptions {
   // Rate at which we sample a single trajectory's scans for global
   // localization.
   optional double global_sampling_ratio = 5;
-};
+}

cartographer/mapping_2d/scan_matching/proto/fast_correlative_scan_matcher_options.proto

@@ -27,4 +27,4 @@ message FastCorrelativeScanMatcherOptions {
 
   // Number of precomputed grids to use.
   optional int32 branch_and_bound_depth = 2;
-};
+}

cartographer/sensor/proto/adaptive_voxel_filter_options.proto

@@ -26,4 +26,4 @@ message AdaptiveVoxelFilterOptions {
 
   // Points further away from the origin are removed.
   optional float max_range = 3;
-};
+}

docs/source/configuration.rst

@@ -20,7 +20,7 @@ Configuration
 .. needed and run scripts/update_configuration_doc.py.
 
 cartographer.common.proto.CeresSolverOptions
---------------------------------------------
+============================================
 
 bool use_nonmonotonic_steps
   Configure the Ceres solver. See the Ceres documentation for more
@@ -34,7 +34,7 @@ int32 num_threads
 
 
 cartographer.kalman_filter.proto.PoseTrackerOptions
----------------------------------------------------
+===================================================
 
 double position_model_variance
   Model variances depend linearly on time.
@@ -57,7 +57,7 @@ int32 num_odometry_states
 
 
 cartographer.mapping.proto.MapBuilderOptions
---------------------------------------------
+============================================
 
 bool use_trajectory_builder_2d
   Not yet documented.
@@ -79,7 +79,7 @@ SparsePoseGraphOptions sparse_pose_graph_options
 
 
 cartographer.mapping.proto.SparsePoseGraphOptions
--------------------------------------------------
+=================================================
 
 int32 optimize_every_n_scans
   Online loop closure: If positive, will run the loop closure while the map
@@ -101,7 +101,7 @@ double global_sampling_ratio
 
 
 cartographer.mapping.sparse_pose_graph.proto.ConstraintBuilderOptions
----------------------------------------------------------------------
+=====================================================================
 
 double sampling_ratio
   A constraint will be added if the proportion of added constraints to
@@ -140,7 +140,7 @@ mapping_3d.scan_matching.proto.CeresScanMatcherOptions ceres_scan_matcher_option
 
 
 cartographer.mapping.sparse_pose_graph.proto.OptimizationProblemOptions
------------------------------------------------------------------------
+=======================================================================
 
 double huber_scale
   Scaling parameter for Huber loss function.
@@ -165,7 +165,7 @@ common.proto.CeresSolverOptions ceres_solver_options
 
 
 cartographer.mapping_2d.proto.LaserFanInserterOptions
------------------------------------------------------
+=====================================================
 
 double hit_probability
   Probability change for a hit (this will be converted to odds and therefore
@@ -181,7 +181,7 @@ bool insert_free_space
 
 
 cartographer.mapping_2d.proto.LocalTrajectoryBuilderOptions
------------------------------------------------------------
+===========================================================
 
 float laser_min_range
   Laser returns outside these ranges will be dropped.
@@ -238,7 +238,7 @@ bool use_imu_data
 
 
 cartographer.mapping_2d.proto.SubmapsOptions
---------------------------------------------
+============================================
 
 double resolution
   Resolution of the map in meters.
@@ -259,7 +259,7 @@ LaserFanInserterOptions laser_fan_inserter_options
 
 
 cartographer.mapping_2d.scan_matching.proto.CeresScanMatcherOptions
--------------------------------------------------------------------
+===================================================================
 
 double occupied_space_weight
   Scaling parameters for each cost functor.
@@ -279,7 +279,7 @@ common.proto.CeresSolverOptions ceres_solver_options
 
 
 cartographer.mapping_2d.scan_matching.proto.FastCorrelativeScanMatcherOptions
------------------------------------------------------------------------------
+=============================================================================
 
 double linear_search_window
   Minimum linear search window in which the best possible scan alignment
@@ -294,7 +294,7 @@ int32 branch_and_bound_depth
 
 
 cartographer.mapping_2d.scan_matching.proto.RealTimeCorrelativeScanMatcherOptions
----------------------------------------------------------------------------------
+=================================================================================
 
 double linear_search_window
   Minimum linear search window in which the best possible scan alignment
@@ -312,7 +312,7 @@ double rotation_delta_cost_weight
 
 
 cartographer.mapping_3d.proto.KalmanLocalTrajectoryBuilderOptions
------------------------------------------------------------------
+=================================================================
 
 bool use_online_correlative_scan_matching
   Whether to solve the online scan matching first using the correlative scan
@@ -332,7 +332,7 @@ double odometer_rotational_variance
 
 
 cartographer.mapping_3d.proto.LaserFanInserterOptions
------------------------------------------------------
+=====================================================
 
 double hit_probability
   Probability change for a hit (this will be converted to odds and therefore
@@ -348,11 +348,11 @@ int32 num_free_space_voxels
 
 
 cartographer.mapping_3d.proto.LocalTrajectoryBuilderOptions
------------------------------------------------------------
+===========================================================
 
 
 cartographer.mapping_3d.proto.MotionFilterOptions
--------------------------------------------------
+=================================================
 
 double max_time_seconds
   Threshold above which a new scan is inserted based on time.
@@ -365,7 +365,7 @@ double max_angle_radians
 
 
 cartographer.mapping_3d.proto.OptimizingLocalTrajectoryBuilderOptions
----------------------------------------------------------------------
+=====================================================================
 
 double high_resolution_grid_weight
   Not yet documented.
@@ -390,7 +390,7 @@ double odometry_rotation_weight
 
 
 cartographer.mapping_3d.proto.SubmapsOptions
---------------------------------------------
+============================================
 
 double high_resolution
   Resolution of the 'high_resolution' map in meters used for local SLAM and
@@ -414,7 +414,7 @@ LaserFanInserterOptions laser_fan_inserter_options
 
 
 cartographer.mapping_3d.scan_matching.proto.CeresScanMatcherOptions
--------------------------------------------------------------------
+===================================================================
 
 double translation_weight
   Scaling parameters for each cost functor.
@@ -434,7 +434,7 @@ common.proto.CeresSolverOptions ceres_solver_options
 
 
 cartographer.mapping_3d.scan_matching.proto.FastCorrelativeScanMatcherOptions
------------------------------------------------------------------------------
+=============================================================================
 
 int32 branch_and_bound_depth
   Number of precomputed grids to use.
@@ -463,7 +463,7 @@ double angular_search_window
 
 
 cartographer.sensor.proto.AdaptiveVoxelFilterOptions
-----------------------------------------------------
+====================================================
 
 float max_length
   'max_length' of a voxel edge.

scripts/update_configuration_doc.py

@@ -47,80 +47,131 @@ SUFFIX = """
 """
 NODOC = 'Not yet documented.'
 
-def GenerateDocumentation(output_dict, proto_file_name):
-  copyright = True
-  message = None
-  content = [];
-  package = None
-  multiline = None
-  print("Reading '%s'..." % proto_file_name)
-  for line in io.open(proto_file_name, encoding='UTF-8'):
+
+def ForEachProtoFile(root, callback):
+  for dirpath, dirnames, filenames in os.walk(root):
+    for name in filenames:
+      if name.endswith('.proto'):
+        path = os.path.join(dirpath, name)
+        print("Found '%s'..." % path)
+        assert not os.path.islink(path)
+        callback(io.open(path, encoding='UTF-8'))
+
+
+class Message(object):
+  def __init__(self, name, preceding_comments):
+    self.name = name
+    self.preceding_comments = preceding_comments
+    self.trailing_comments = None
+    self.options = []
+
+  def AddTrailingComments(self, comments):
+    self.trailing_comments = comments
+
+  def AddOption(self, name, comments):
+    self.options.append((name, comments))
+
+
+def ParseProtoFile(proto_file):
+  """Computes the list of Message objects of the option messages in a file."""
+  line_iter = iter(proto_file)
+
+  # We ignore the license header and search for the 'package' line.
+  for line in line_iter:
     line = line.strip()
-    if copyright:
-      if not line.startswith('//'):
-        copyright = False
-      continue
-    if package is None:
-      if line.startswith('package'):
-        assert line[-1] == ';'
-        package = line[7:-1].strip()
-      continue
-    if line.startswith('//'):
-      content_line = line[2:].strip()
-      if not content_line.startswith('NEXT ID:'):
-        content.append(content_line)
-      continue
-    if message is None:
-      if line.startswith('message') and line.endswith('Options {'):
-        message = package + '.' + line[7:-1].strip()
-        print(" Found '%s'." % message)
-        assert message not in output_dict
-        message_list = [message, '=' * len(message), '']
-        output_dict[message] = message_list
-        message_list.extend(content)
-        content = []
-      continue
-    elif line.endswith('}'):
-      message_list.extend(content)
-      content = []
-      message_list = None
-      message = None
+    if line.startswith('package'):
+      assert line[-1] == ';'
+      package = line[7:-1].strip()
+      break
     else:
-      assert not line.startswith('required')
-      if multiline is None:
-        if line.startswith('optional'):
-          multiline = line
-        else:
-          continue
+      assert '}' not in line
+
+  message_list = []
+  while True:
+    # Search for the next options message and capture preceding comments.
+    message_comments = []
+    for line in line_iter:
+      line = line.strip()
+      if '}' in line:
+        # The preceding comments were for a different message it seems.
+        message_comments = []
+      elif line.startswith('//'):
+        # We keep comments preceding an options message.
+        comment = line[2:].strip()
+        if not comment.startswith('NEXT ID:'):
+          message_comments.append(comment)
+      elif line.startswith('message') and line.endswith('Options {'):
+        message_name = package + '.' + line[7:-1].strip()
+        break
+    else:
+      # We reached the end of file.
+      break
+    print(" Found '%s'." % message_name)
+    message = Message(message_name, message_comments)
+    message_list.append(message)
+
+    # We capture the contents of this message.
+    option_comments = []
+    multiline = None
+    for line in line_iter:
+      line = line.strip()
+      if '}' in line:
+        # We reached the end of this message.
+        message.AddTrailingComments(option_comments)
+        break
+      elif line.startswith('//'):
+        comment = line[2:].strip()
+        if not comment.startswith('NEXT ID:'):
+          option_comments.append(comment)
       else:
-        multiline += ' ' + line
-      if not multiline.endswith(';'):
-        continue
-      assert len(multiline) < 200
-      option = multiline[8:-1].strip().rstrip('0123456789').strip()
-      assert option.endswith('=')
-      option = option[:-1].strip();
-      print("  Option '%s'." % option)
-      multiline = None
-      message_list.append(option)
-      if len(content) == 0:
-        content.append(NODOC)
-      for option_description_line in content:
-        message_list.append('  ' + option_description_line)
-      content = []
-      message_list.append('')
+        assert not line.startswith('required')
+        if multiline is None:
+          if line.startswith('optional'):
+            multiline = line
+          else:
+            continue
+        else:
+          multiline += ' ' + line
+        if not multiline.endswith(';'):
+          continue
+        assert len(multiline) < 200
+        option_name = multiline[8:-1].strip().rstrip('0123456789').strip()
+        assert option_name.endswith('=')
+        option_name = option_name[:-1].strip();
+        print("  Option '%s'." % option_name)
+        multiline = None
+        message.AddOption(option_name, option_comments)
+        option_comments = []
+
+  return message_list
+
+
+def GenerateDocumentation(output_dict, proto_file):
+  for message in ParseProtoFile(proto_file):
+    content = [message.name, '=' * len(message.name), '']
+    assert message.name not in output_dict
+    output_dict[message.name] = content
+    if message.preceding_comments:
+      content.extend(preceding_comments)
+      content.append('')
+    for option_name, option_comments in message.options:
+      content.append(option_name)
+      if not option_comments:
+        option_comments.append(NODOC)
+      for comment in option_comments:
+        content.append('  ' + comment)
+      content.append('')
+    if message.trailing_comments:
+      content.extend(message.trailing_comments)
+      content.append('')
 
 
 def GenerateDocumentationRecursively(output_file, root):
   """Recursively generates documentation, sorts and writes it."""
   output_dict = {}
-  for root, dirs, files in os.walk(root):
-    for name in files:
-      if name.endswith('.proto'):
-        path = os.path.join(root, name)
-        assert not os.path.islink(path)
-        GenerateDocumentation(output_dict, path)
-
+  def callback(proto_file):
+    GenerateDocumentation(output_dict, proto_file)
+  ForEachProtoFile(root, callback)
   output = ['\n'.join(doc) for key, doc in sorted(list(output_dict.items()))]
   print('\n\n'.join(output), file=output_file)