Collect metrics with Prometheus C++ Client (#916)

RFC=[0014](https://github.com/googlecartographer/rfcs/blob/master/text/0014-monitoring.md)

CMakeLists.txt

@@ -60,12 +60,20 @@ file(GLOB_RECURSE ALL_EXECUTABLES "*_main.cc")
 list(REMOVE_ITEM ALL_LIBRARY_SRCS ${ALL_TESTS})
 list(REMOVE_ITEM ALL_LIBRARY_SRCS ${ALL_EXECUTABLES})
 file(GLOB_RECURSE ALL_GRPC_FILES "cartographer_grpc/*")
+file(GLOB_RECURSE ALL_PROMETHEUS_FILES "cartographer_grpc/metrics/prometheus/*")
+list(REMOVE_ITEM ALL_GRPC_FILES ${ALL_PROMETHEUS_FILES})
 if (NOT ${BUILD_GRPC})
   list(REMOVE_ITEM ALL_LIBRARY_HDRS ${ALL_GRPC_FILES})
   list(REMOVE_ITEM ALL_LIBRARY_SRCS ${ALL_GRPC_FILES})
   list(REMOVE_ITEM ALL_TESTS ${ALL_GRPC_FILES})
   list(REMOVE_ITEM ALL_EXECUTABLES ${ALL_GRPC_FILES})
 endif()
+if (NOT ${BUILD_PROMETHEUS})
+  list(REMOVE_ITEM ALL_LIBRARY_HDRS ${ALL_PROMETHEUS_FILES})
+  list(REMOVE_ITEM ALL_LIBRARY_SRCS ${ALL_PROMETHEUS_FILES})
+  list(REMOVE_ITEM ALL_TESTS ${ALL_PROMETHEUS_FILES})
+  list(REMOVE_ITEM ALL_EXECUTABLES ${ALL_PROMETHEUS_FILES})
+endif()
 set(INSTALL_SOURCE_HDRS ${ALL_LIBRARY_HDRS})
 file(GLOB_RECURSE INTERNAL_HDRS "cartographer/internal/*.h"
   "cartographer_grpc/internal/*.h")

cartographer_grpc/metrics/prometheus/family_factory.cc

@@ -0,0 +1,182 @@
+/*
+ * Copyright 2018 The Cartographer Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cartographer_grpc/metrics/prometheus/family_factory.h"
+
+#include "cartographer/common/make_unique.h"
+#include "prometheus/counter.h"
+#include "prometheus/family.h"
+#include "prometheus/gauge.h"
+#include "prometheus/histogram.h"
+
+namespace cartographer_grpc {
+namespace metrics {
+namespace prometheus {
+
+namespace {
+
+using BucketBoundaries = cartographer::metrics::Histogram::BucketBoundaries;
+
+class Counter : public cartographer::metrics::Counter {
+ public:
+  explicit Counter(::prometheus::Counter* prometheus)
+      : prometheus_(prometheus) {}
+
+  void Increment() override { prometheus_->Increment(); }
+  void Increment(double by_value) override { prometheus_->Increment(by_value); }
+
+ private:
+  ::prometheus::Counter* prometheus_;
+};
+
+class CounterFamily
+    : public cartographer::metrics::Family<cartographer::metrics::Counter> {
+ public:
+  explicit CounterFamily(
+      ::prometheus::Family<::prometheus::Counter>* prometheus)
+      : prometheus_(prometheus) {}
+
+  Counter* Add(const std::map<std::string, std::string>& labels) override {
+    ::prometheus::Counter* counter = &prometheus_->Add(labels);
+    auto wrapper = cartographer::common::make_unique<Counter>(counter);
+    auto* ptr = wrapper.get();
+    wrappers_.emplace_back(std::move(wrapper));
+    return ptr;
+  }
+
+ private:
+  ::prometheus::Family<::prometheus::Counter>* prometheus_;
+  std::vector<std::unique_ptr<Counter>> wrappers_;
+};
+
+class Gauge : public cartographer::metrics::Gauge {
+ public:
+  explicit Gauge(::prometheus::Gauge* prometheus) : prometheus_(prometheus) {}
+
+  void Decrement() override { prometheus_->Decrement(); }
+  void Decrement(double by_value) override { prometheus_->Decrement(by_value); }
+  void Increment() override { prometheus_->Increment(); }
+  void Increment(double by_value) override { prometheus_->Increment(by_value); }
+  void Set(double value) override { prometheus_->Set(value); }
+
+ private:
+  ::prometheus::Gauge* prometheus_;
+};
+
+class GaugeFamily
+    : public cartographer::metrics::Family<cartographer::metrics::Gauge> {
+ public:
+  explicit GaugeFamily(::prometheus::Family<::prometheus::Gauge>* prometheus)
+      : prometheus_(prometheus) {}
+
+  Gauge* Add(const std::map<std::string, std::string>& labels) override {
+    ::prometheus::Gauge* gauge = &prometheus_->Add(labels);
+    auto wrapper = cartographer::common::make_unique<Gauge>(gauge);
+    auto* ptr = wrapper.get();
+    wrappers_.emplace_back(std::move(wrapper));
+    return ptr;
+  }
+
+ private:
+  ::prometheus::Family<::prometheus::Gauge>* prometheus_;
+  std::vector<std::unique_ptr<Gauge>> wrappers_;
+};
+
+class Histogram : public cartographer::metrics::Histogram {
+ public:
+  explicit Histogram(::prometheus::Histogram* prometheus)
+      : prometheus_(prometheus) {}
+
+  void Observe(double value) override { prometheus_->Observe(value); }
+
+ private:
+  ::prometheus::Histogram* prometheus_;
+};
+
+class HistogramFamily
+    : public cartographer::metrics::Family<cartographer::metrics::Histogram> {
+ public:
+  HistogramFamily(::prometheus::Family<::prometheus::Histogram>* prometheus,
+                  const BucketBoundaries& boundaries)
+      : prometheus_(prometheus), boundaries_(boundaries) {}
+
+  Histogram* Add(const std::map<std::string, std::string>& labels) override {
+    ::prometheus::Histogram* histogram = &prometheus_->Add(labels, boundaries_);
+    auto wrapper = cartographer::common::make_unique<Histogram>(histogram);
+    auto* ptr = wrapper.get();
+    wrappers_.emplace_back(std::move(wrapper));
+    return ptr;
+  }
+
+ private:
+  ::prometheus::Family<::prometheus::Histogram>* prometheus_;
+  std::vector<std::unique_ptr<Histogram>> wrappers_;
+  const BucketBoundaries boundaries_;
+};
+
+}  // namespace
+
+FamilyFactory::FamilyFactory()
+    : registry_(std::make_shared<::prometheus::Registry>()) {}
+
+cartographer::metrics::Family<cartographer::metrics::Counter>*
+FamilyFactory::NewCounterFamily(const std::string& name,
+                                const std::string& description) {
+  auto& family = ::prometheus::BuildCounter()
+                     .Name(name)
+                     .Help(description)
+                     .Register(*registry_);
+  auto wrapper = cartographer::common::make_unique<CounterFamily>(&family);
+  auto* ptr = wrapper.get();
+  counters_.emplace_back(std::move(wrapper));
+  return ptr;
+}
+
+cartographer::metrics::Family<cartographer::metrics::Gauge>*
+FamilyFactory::NewGaugeFamily(const std::string& name,
+                              const std::string& description) {
+  auto& family = ::prometheus::BuildGauge()
+                     .Name(name)
+                     .Help(description)
+                     .Register(*registry_);
+  auto wrapper = cartographer::common::make_unique<GaugeFamily>(&family);
+  auto* ptr = wrapper.get();
+  gauges_.emplace_back(std::move(wrapper));
+  return ptr;
+}
+
+cartographer::metrics::Family<cartographer::metrics::Histogram>*
+FamilyFactory::NewHistogramFamily(const std::string& name,
+                                  const std::string& description,
+                                  const BucketBoundaries& boundaries) {
+  auto& family = ::prometheus::BuildHistogram()
+                     .Name(name)
+                     .Help(description)
+                     .Register(*registry_);
+  auto wrapper =
+      cartographer::common::make_unique<HistogramFamily>(&family, boundaries);
+  auto* ptr = wrapper.get();
+  histograms_.emplace_back(std::move(wrapper));
+  return ptr;
+}
+
+std::weak_ptr<::prometheus::Collectable> FamilyFactory::GetCollectable() const {
+  return registry_;
+}
+
+}  // namespace prometheus
+}  // namespace metrics
+}  // namespace cartographer_grpc

cartographer_grpc/metrics/prometheus/family_factory.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2018 The Cartographer Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CARTOGRAPHER_GRPC_METRICS_PROMETHEUS_FAMILY_FACTORY_H_
+#define CARTOGRAPHER_GRPC_METRICS_PROMETHEUS_FAMILY_FACTORY_H_
+
+#include <memory>
+#include <string>
+
+#include "cartographer/metrics/family_factory.h"
+#include "prometheus/registry.h"
+
+namespace cartographer_grpc {
+namespace metrics {
+namespace prometheus {
+
+class FamilyFactory : public cartographer::metrics::FamilyFactory {
+ public:
+  FamilyFactory();
+
+  cartographer::metrics::Family<cartographer::metrics::Counter>*
+  NewCounterFamily(const std::string& name,
+                   const std::string& description) override;
+  cartographer::metrics::Family<cartographer::metrics::Gauge>* NewGaugeFamily(
+      const std::string& name, const std::string& description) override;
+  cartographer::metrics::Family<cartographer::metrics::Histogram>*
+  NewHistogramFamily(const std::string& name, const std::string& description,
+                     const cartographer::metrics::Histogram::BucketBoundaries&
+                         boundaries) override;
+
+  std::weak_ptr<::prometheus::Collectable> GetCollectable() const;
+
+ private:
+  std::vector<std::unique_ptr<
+      cartographer::metrics::Family<cartographer::metrics::Counter>>>
+      counters_;
+  std::vector<std::unique_ptr<
+      cartographer::metrics::Family<cartographer::metrics::Gauge>>>
+      gauges_;
+  std::vector<std::unique_ptr<
+      cartographer::metrics::Family<cartographer::metrics::Histogram>>>
+      histograms_;
+  std::shared_ptr<::prometheus::Registry> registry_;
+};
+
+}  // namespace prometheus
+}  // namespace metrics
+}  // namespace cartographer_grpc
+
+#endif  // CARTOGRAPHER_GRPC_METRICS_PROMETHEUS_FAMILY_FACTORY_H_

cartographer_grpc/metrics/prometheus/metrics_test.cc

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2018 The Cartographer Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cartographer/metrics/family_factory.h"
+#include "cartographer/metrics/register.h"
+#include "cartographer_grpc/metrics/prometheus/family_factory.h"
+#include "glog/logging.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "prometheus/exposer.h"
+
+namespace cartographer_grpc {
+namespace metrics {
+namespace prometheus {
+namespace {
+
+static auto* kCounter = cartographer::metrics::Counter::Null();
+static auto* kGauge = cartographer::metrics::Gauge::Null();
+static auto* kScoresMetric = cartographer::metrics::Histogram::Null();
+
+const char kLabelKey[] = "kind";
+const char kLabelValue[] = "score";
+const std::array<double, 5> kObserveScores = {{-1, 0.11, 0.2, 0.5, 2}};
+
+class Algorithm {
+ public:
+  static void RegisterMetrics(cartographer::metrics::FamilyFactory* factory) {
+    auto boundaries = cartographer::metrics::Histogram::FixedWidth(0.05, 20);
+    auto* scores_family = factory->NewHistogramFamily(
+        "/algorithm/scores", "Scores achieved", boundaries);
+    kScoresMetric = scores_family->Add({{kLabelKey, kLabelValue}});
+  }
+  void Run() {
+    for (double score : kObserveScores) {
+      kScoresMetric->Observe(score);
+    }
+  }
+};
+
+TEST(MetricsTest, CollectCounter) {
+  FamilyFactory factory;
+  auto* counter_family = factory.NewCounterFamily("/test/hits", "Hits");
+  kCounter = counter_family->Add({{kLabelKey, kLabelValue}});
+  kCounter->Increment();
+  kCounter->Increment(5);
+  double expected_value = 1 + 5;
+  std::vector<::io::prometheus::client::MetricFamily> collected;
+  {
+    std::shared_ptr<::prometheus::Collectable> collectable;
+    CHECK(collectable = factory.GetCollectable().lock());
+    collected = collectable->Collect();
+  }
+  ASSERT_EQ(collected.size(), 1);
+  ASSERT_EQ(collected[0].metric_size(), 1);
+  EXPECT_THAT(
+      collected[0].metric(0).label(),
+      testing::AllOf(
+          testing::ElementsAre(testing::Property(
+              &io::prometheus::client::LabelPair::name, kLabelKey)),
+          testing::ElementsAre(testing::Property(
+              &io::prometheus::client::LabelPair::value, kLabelValue))));
+  EXPECT_THAT(collected[0].metric(0).counter().value(),
+              testing::DoubleEq(expected_value));
+}
+
+TEST(MetricsTest, CollectGauge) {
+  FamilyFactory factory;
+  auto* gauge_family =
+      factory.NewGaugeFamily("/test/queue/length", "Length of some queue");
+  kGauge = gauge_family->Add({{kLabelKey, kLabelValue}});
+  kGauge->Increment();
+  kGauge->Increment(5);
+  kGauge->Decrement();
+  kGauge->Decrement(2);
+  double expected_value = 1 + 5 - 1 - 2;
+  std::vector<::io::prometheus::client::MetricFamily> collected;
+  {
+    std::shared_ptr<::prometheus::Collectable> collectable;
+    CHECK(collectable = factory.GetCollectable().lock());
+    collected = collectable->Collect();
+  }
+  ASSERT_EQ(collected.size(), 1);
+  ASSERT_EQ(collected[0].metric_size(), 1);
+  EXPECT_THAT(
+      collected[0].metric(0).label(),
+      testing::AllOf(
+          testing::ElementsAre(testing::Property(
+              &io::prometheus::client::LabelPair::name, kLabelKey)),
+          testing::ElementsAre(testing::Property(
+              &io::prometheus::client::LabelPair::value, kLabelValue))));
+  EXPECT_THAT(collected[0].metric(0).gauge().value(),
+              testing::DoubleEq(expected_value));
+}
+
+TEST(MetricsTest, CollectHistogram) {
+  FamilyFactory registry;
+  Algorithm::RegisterMetrics(&registry);
+
+  Algorithm algorithm;
+  algorithm.Run();
+  std::vector<::io::prometheus::client::MetricFamily> collected;
+  {
+    std::shared_ptr<::prometheus::Collectable> collectable;
+    CHECK(collectable = registry.GetCollectable().lock());
+    collected = collectable->Collect();
+  }
+  ASSERT_EQ(collected.size(), 1);
+  ASSERT_EQ(collected[0].metric_size(), 1);
+  EXPECT_THAT(
+      collected[0].metric(0).label(),
+      testing::AllOf(
+          testing::ElementsAre(testing::Property(
+              &io::prometheus::client::LabelPair::name, kLabelKey)),
+          testing::ElementsAre(testing::Property(
+              &io::prometheus::client::LabelPair::value, kLabelValue))));
+  EXPECT_THAT(collected[0].metric(0).histogram().sample_count(),
+              testing::Eq(kObserveScores.size()));
+  EXPECT_EQ(collected[0].metric(0).histogram().bucket(0).cumulative_count(), 1);
+}
+
+}  // namespace
+}  // namespace prometheus
+}  // namespace metrics
+}  // namespace cartographer_grpc