Repo created

libs/routing/routing_quality/routing_quality_tool/CMakeLists.txt

@@ -0,0 +1,22 @@
+project(routing_quality_tool)
+
+set(SRC
+  benchmark_results.cpp
+  benchmark_results.hpp
+  benchmark_stat.cpp
+  benchmark_stat.hpp
+  error_type_counter.cpp
+  error_type_counter.hpp
+  routing_quality_tool.cpp
+  utils.cpp
+  utils.hpp
+)
+
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  routing_quality
+  routing_api
+  kml
+  gflags::gflags
+)

libs/routing/routing_quality/routing_quality_tool/benchmark_results.cpp

@@ -0,0 +1,18 @@
+#include "routing/routing_quality/routing_quality_tool/benchmark_results.hpp"
+
+namespace routing_quality::routing_quality_tool
+{
+double BenchmarkResults::GetAverageBuildTime() const
+{
+  if (m_buildTimes.empty())
+    return 0.0;
+
+  return m_summaryBuildTimeSeconds / static_cast<double>(m_buildTimes.size());
+}
+
+void BenchmarkResults::PushBuildTime(double time)
+{
+  m_summaryBuildTimeSeconds += time;
+  m_buildTimes.emplace_back(time);
+}
+}  // namespace routing_quality::routing_quality_tool

libs/routing/routing_quality/routing_quality_tool/benchmark_results.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "routing/routing_callbacks.hpp"
+
+#include <map>
+#include <string>
+#include <vector>
+
+namespace routing_quality::routing_quality_tool
+{
+class BenchmarkResults
+{
+public:
+  double GetAverageBuildTime() const;
+  void PushBuildTime(double time);
+
+  std::vector<double> const & GetBuildTimes() const { return m_buildTimes; }
+
+private:
+  double m_summaryBuildTimeSeconds = 0.0;
+  // m_buildTimes[i] stores build time of i-th route.
+  std::vector<double> m_buildTimes;
+};
+
+struct TimeInfo
+{
+  TimeInfo(double oldTime, double newTime) : m_oldTime(oldTime), m_newTime(newTime) {}
+  double m_oldTime;
+  double m_newTime;
+};
+}  // namespace routing_quality::routing_quality_tool

libs/routing/routing_quality/routing_quality_tool/benchmark_stat.cpp

@@ -0,0 +1,303 @@
+#include "routing/routing_quality/routing_quality_tool/benchmark_stat.hpp"
+
+#include "routing/routing_quality/routing_quality_tool/benchmark_results.hpp"
+#include "routing/routing_quality/routing_quality_tool/error_type_counter.hpp"
+#include "routing/routing_quality/routing_quality_tool/utils.hpp"
+
+#include "routing/routing_callbacks.hpp"
+
+#include "geometry/point2d.hpp"
+
+#include "base/assert.hpp"
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+
+namespace
+{
+using namespace routing;
+using namespace routes_builder;
+using namespace routing_quality::routing_quality_tool;
+
+bool IsErrorCode(RouterResultCode code)
+{
+  return code != RouterResultCode::NoError;
+}
+
+void LogIfNotConsistent(RoutesBuilder::Result const & oldRes, RoutesBuilder::Result const & newRes)
+{
+  auto const start = mercator::ToLatLon(oldRes.m_params.m_checkpoints.GetStart());
+  auto const finish = mercator::ToLatLon(oldRes.m_params.m_checkpoints.GetFinish());
+  CHECK(!oldRes.m_routes.empty(), ());
+  CHECK(!newRes.m_routes.empty(), ());
+  auto const & oldRoute = oldRes.m_routes.back();
+  auto const & newRoute = newRes.m_routes.back();
+
+  bool const sameETA = AlmostEqualAbs(oldRoute.m_eta, newRoute.m_eta, 1.0);
+  bool const sameDistance = AlmostEqualAbs(oldRoute.m_distance, newRoute.m_distance, 1.0);
+  if (!sameETA || !sameDistance)
+  {
+    LOG(LINFO, ("old ETA:", oldRoute.m_eta, "old distance:", oldRoute.m_distance, "new ETA:", newRoute.m_eta,
+                "new distance:", newRoute.m_distance, start, finish));
+  }
+}
+
+/// \brief Helps to compare route time building for routes group by old time building.
+void FillInfoAboutBuildTimeGroupByPreviousResults(std::vector<std::string> & labels,
+                                                  std::vector<std::vector<double>> & bars,
+                                                  std::vector<TimeInfo> && times)
+{
+  bars.clear();
+  labels.clear();
+
+  std::sort(times.begin(), times.end(), [](auto const & a, auto const & b) { return a.m_oldTime < b.m_oldTime; });
+
+  size_t constexpr kSteps = 10;
+  size_t const step = times.size() / kSteps;
+
+  size_t startFrom = 0;
+  size_t curCount = 0;
+  bars.resize(2);
+  double curSumOld = 0;
+  double curSumNew = 0;
+  for (size_t i = 0; i < times.size(); ++i)
+  {
+    if (curCount < step && i + 1 != times.size())
+    {
+      ++curCount;
+      curSumOld += times[i].m_oldTime;
+      curSumNew += times[i].m_newTime;
+      continue;
+    }
+
+    double const curLeft = times[startFrom].m_oldTime;
+    startFrom = i + 1;
+    double const curRight = times[i].m_oldTime;
+    labels.emplace_back("[" + strings::to_string_dac(curLeft, 2 /* dac */) + "s, " +
+                        strings::to_string_dac(curRight, 2 /* dac */) + "s]\\n" + "Routes count:\\n" +
+                        std::to_string(curCount));
+    if (curCount != 0)
+    {
+      curSumOld /= curCount;
+      curSumNew /= curCount;
+    }
+    double const k = curSumOld == 0.0 ? 0.0 : curSumNew / curSumOld;
+    bars[0].emplace_back(100.0);
+    bars[1].emplace_back(100.0 * k);
+    curCount = 0;
+  }
+}
+
+std::vector<double> GetBoostPercents(BenchmarkResults const & oldResults, BenchmarkResults const & newResults)
+{
+  std::vector<double> boostPercents;
+  for (size_t i = 0; i < oldResults.GetBuildTimes().size(); ++i)
+  {
+    auto const oldTime = oldResults.GetBuildTimes()[i];
+    auto const newTime = newResults.GetBuildTimes()[i];
+    if (AlmostEqualAbs(oldTime, newTime, 1e-2))
+      continue;
+
+    auto const diffPercent = (oldTime - newTime) / oldTime * 100.0;
+    boostPercents.emplace_back(diffPercent);
+  }
+
+  return boostPercents;
+}
+}  // namespace
+
+namespace routing_quality::routing_quality_tool
+{
+using namespace routing;
+using namespace routes_builder;
+// Shows distribution of routes time building.
+static std::string const kPythonDistTimeBuilding = "show_route_time_building_dist.py";
+// Shows graph of "how many routes in percents build in less time than some T".
+static std::string const kPythonGraphTimeAndCount = "show_time_count_graph.py";
+// Bar graph of routing errors. Labels - string representation of errors, heights - number of such
+// errors.
+static std::string const kPythonBarError = "show_errors_bar.py";
+// Shows percent of boost distribution. Where boost percent equals:
+// 100.0 * (old_time - new_time) / old_time
+static std::string const kPythonBarBoostPercentDistr = "show_boost_distr.py";
+// Shows distribution of non-zero difference of ETA between old and new mapsme version.
+static std::string const kPythonEtaDiff = "eta_diff.py";
+// The same as above but in percents.
+static std::string const kPythonEtaDiffPercent = "eta_diff_percent.py";
+// Groups routes by previous time building and draws two types of bars. The first one (old mapsme)
+// has the same height in all groups and the second ones' height is proportional less or more
+// according to difference in average time building between old and new version. The example you can
+// see here: https://github.com/mapsme/omim/pull/12401
+static std::string const kPythonSmartDistr = "show_smart_boost_distr.py";
+
+void RunBenchmarkStat(std::vector<std::pair<RoutesBuilder::Result, std::string>> const & mapsmeResults,
+                      std::string const & dirForResults)
+{
+  BenchmarkResults benchmarkResults;
+  ErrorTypeCounter errorTypeCounter;
+  for (auto const & resultItem : mapsmeResults)
+  {
+    auto const & result = resultItem.first;
+    errorTypeCounter.PushError(result.m_code);
+    if (result.m_code != RouterResultCode::NoError)
+      continue;
+
+    CHECK(!result.m_routes.empty(), ());
+    benchmarkResults.PushBuildTime(result.m_buildTimeSeconds);
+  }
+
+  auto pythonScriptPath = base::JoinPath(dirForResults, kPythonDistTimeBuilding);
+  CreatePythonScriptForDistribution(pythonScriptPath, "Route building time, seconds", benchmarkResults.GetBuildTimes());
+
+  LOG(LINFO, ("Average route time building:", benchmarkResults.GetAverageBuildTime(), "seconds."));
+
+  auto times = benchmarkResults.GetBuildTimes();
+  std::sort(times.begin(), times.end());
+  std::vector<m2::PointD> countToTimes(times.size());
+  for (size_t i = 0; i < countToTimes.size(); ++i)
+  {
+    countToTimes[i].x = times[i];
+    countToTimes[i].y = static_cast<double>(i + 1) / times.size() * 100.0;
+  }
+
+  pythonScriptPath = base::JoinPath(dirForResults, kPythonGraphTimeAndCount);
+  CreatePythonGraphByPointsXY(pythonScriptPath, "Building time, seconds" /* xlabel */,
+                              "Percent of routes built less than" /* ylabel */, {countToTimes},
+                              {"mapsme"} /* legends */);
+
+  pythonScriptPath = base::JoinPath(dirForResults, kPythonBarError);
+
+  std::vector<std::string> labels;
+  std::vector<double> heights;
+  FillLabelsAndErrorTypeDistribution(labels, heights, errorTypeCounter);
+
+  CreatePythonBarByMap(pythonScriptPath, labels, {heights}, {"mapsme"} /* legends */, "Type of errors" /* xlabel */,
+                       "Number of errors" /* ylabel */);
+}
+
+void RunBenchmarkComparison(std::vector<std::pair<RoutesBuilder::Result, std::string>> && mapsmeResults,
+                            std::vector<std::pair<RoutesBuilder::Result, std::string>> && mapsmeOldResults,
+                            std::string const & dirForResults)
+{
+  BenchmarkResults benchmarkResults;
+  BenchmarkResults benchmarkOldResults;
+  ErrorTypeCounter errorTypeCounter;
+  ErrorTypeCounter errorTypeCounterOld;
+  std::vector<double> etaDiffsPercent;
+  std::vector<double> etaDiffs;
+
+  std::vector<TimeInfo> times;
+
+  for (size_t i = 0; i < mapsmeResults.size(); ++i)
+  {
+    auto const & mapsmeResult = mapsmeResults[i].first;
+
+    std::pair<RoutesBuilder::Result, std::string> mapsmeOldResultPair;
+    if (!FindAnotherResponse(mapsmeResult, mapsmeOldResults, mapsmeOldResultPair))
+    {
+      LOG(LDEBUG, ("Can not find pair for:", i));
+      continue;
+    }
+
+    auto const & mapsmeOldResult = mapsmeOldResultPair.first;
+
+    errorTypeCounter.PushError(mapsmeResult.m_code);
+    errorTypeCounterOld.PushError(mapsmeOldResult.m_code);
+
+    if (IsErrorCode(mapsmeResult.m_code) && !IsErrorCode(mapsmeOldResult.m_code))
+    {
+      auto const start = mercator::ToLatLon(mapsmeResult.m_params.m_checkpoints.GetStart());
+      auto const finish = mercator::ToLatLon(mapsmeResult.m_params.m_checkpoints.GetFinish());
+      LOG_FORCE(LWARNING, ("New version returns error code:", mapsmeResult.m_code,
+                           "but old returns NoError for:", start, finish));
+    }
+
+    if (IsErrorCode(mapsmeResult.m_code) || IsErrorCode(mapsmeOldResult.m_code))
+      continue;
+
+    LogIfNotConsistent(mapsmeOldResult, mapsmeResult);
+
+    CHECK(!mapsmeOldResult.m_routes.empty() && !mapsmeResult.m_routes.empty(), ());
+    auto const etaDiff = (mapsmeOldResult.m_routes.back().m_eta - mapsmeResult.m_routes.back().m_eta);
+    auto const etaDiffPercent = etaDiff / mapsmeOldResult.m_routes.back().m_eta * 100.0;
+
+    etaDiffs.emplace_back(etaDiff);
+    etaDiffsPercent.emplace_back(etaDiffPercent);
+
+    auto const oldTime = mapsmeOldResult.m_buildTimeSeconds;
+    auto const newTime = mapsmeResult.m_buildTimeSeconds;
+    auto const diffPercent = (oldTime - newTime) / oldTime * 100.0;
+    // Warn about routes building time degradation.
+    double constexpr kSlowdownPercent = -10.0;
+    if (diffPercent < kSlowdownPercent)
+    {
+      auto const start = mercator::ToLatLon(mapsmeResult.m_params.m_checkpoints.GetStart());
+      auto const finish = mercator::ToLatLon(mapsmeResult.m_params.m_checkpoints.GetFinish());
+      LOG(LINFO, ("oldTime:", oldTime, "newTime:", newTime, "diffPercent:", diffPercent, start, finish));
+    }
+
+    benchmarkResults.PushBuildTime(mapsmeResult.m_buildTimeSeconds);
+    benchmarkOldResults.PushBuildTime(mapsmeOldResult.m_buildTimeSeconds);
+
+    times.emplace_back(mapsmeOldResult.m_buildTimeSeconds, mapsmeResult.m_buildTimeSeconds);
+  }
+
+  LOG(LINFO, ("Comparing", benchmarkResults.GetBuildTimes().size(), "routes."));
+
+  auto const oldAverage = benchmarkOldResults.GetAverageBuildTime();
+  auto const newAverage = benchmarkResults.GetAverageBuildTime();
+  auto const averageTimeDiff = (oldAverage - newAverage) / oldAverage * 100.0;
+  LOG(LINFO, ("Average route time building. "
+              "Old version:",
+              oldAverage, "New version:", newAverage, "(", -averageTimeDiff, "% )"));
+
+  std::vector<std::vector<m2::PointD>> graphics;
+  for (auto const & results : {benchmarkOldResults, benchmarkResults})
+  {
+    auto times = results.GetBuildTimes();
+    std::sort(times.begin(), times.end());
+    std::vector<m2::PointD> countToTimes(times.size());
+    for (size_t i = 0; i < countToTimes.size(); ++i)
+    {
+      countToTimes[i].x = times[i];
+      countToTimes[i].y = static_cast<double>(i + 1) / times.size() * 100.0;
+    }
+
+    graphics.emplace_back(std::move(countToTimes));
+  }
+
+  auto pythonScriptPath = base::JoinPath(dirForResults, kPythonGraphTimeAndCount);
+  CreatePythonGraphByPointsXY(pythonScriptPath, "Building time, seconds" /* xlabel */,
+                              "Percent of routes built less than" /* ylabel */, graphics,
+                              {"old mapsme", "new mapsme"} /* legends */);
+
+  std::vector<std::string> labels;
+  std::vector<std::vector<double>> errorsCount;
+  FillLabelsAndErrorTypeDistribution(labels, errorsCount, errorTypeCounter, errorTypeCounterOld);
+
+  pythonScriptPath = base::JoinPath(dirForResults, kPythonBarError);
+  CreatePythonBarByMap(pythonScriptPath, labels, errorsCount, {"old mapsme", "new mapsme"} /* legends */,
+                       "Type of errors" /* xlabel */, "Number of errors" /* ylabel */);
+
+  auto const boostPercents = GetBoostPercents(benchmarkOldResults, benchmarkResults);
+  pythonScriptPath = base::JoinPath(dirForResults, kPythonBarBoostPercentDistr);
+  CreatePythonScriptForDistribution(pythonScriptPath, "Boost percent" /* title */, boostPercents);
+
+  pythonScriptPath = base::JoinPath(dirForResults, kPythonEtaDiff);
+  CreatePythonScriptForDistribution(pythonScriptPath, "ETA diff distribution" /* title */, etaDiffs);
+
+  pythonScriptPath = base::JoinPath(dirForResults, kPythonEtaDiffPercent);
+  CreatePythonScriptForDistribution(pythonScriptPath, "ETA diff percent distribution" /* title */, etaDiffsPercent);
+
+  std::vector<std::vector<double>> bars;
+  FillInfoAboutBuildTimeGroupByPreviousResults(labels, bars, std::move(times));
+  pythonScriptPath = base::JoinPath(dirForResults, kPythonSmartDistr);
+  CreatePythonBarByMap(pythonScriptPath, labels, bars, {"old mapsme", "new mapsme"} /* legends */,
+                       "Intervals of groups (build time in old mapsme)" /* xlabel */,
+                       "Boost\\nRight column is so lower/higher than the left\\n how much the average build time "
+                       "has decreased in each group)" /* ylabel */,
+                       false /* drawPercents */);
+}
+}  // namespace routing_quality::routing_quality_tool

libs/routing/routing_quality/routing_quality_tool/benchmark_stat.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "routing/routes_builder/routes_builder.hpp"
+
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace routing_quality::routing_quality_tool
+{
+using RouteResult = routing::routes_builder::RoutesBuilder::Result;
+void RunBenchmarkStat(std::vector<std::pair<RouteResult, std::string>> const & mapsmeResults,
+                      std::string const & dirForResults);
+
+void RunBenchmarkComparison(std::vector<std::pair<RouteResult, std::string>> && mapsmeResults,
+                            std::vector<std::pair<RouteResult, std::string>> && mapsmeOldResults,
+                            std::string const & dirForResults);
+}  // namespace routing_quality::routing_quality_tool

libs/routing/routing_quality/routing_quality_tool/error_type_counter.cpp

@@ -0,0 +1,49 @@
+#include "routing/routing_quality/routing_quality_tool/error_type_counter.hpp"
+
+#include "base/assert.hpp"
+
+namespace routing_quality::routing_quality_tool
+{
+void ErrorTypeCounter::PushError(routing::RouterResultCode code)
+{
+  ++m_errorCounter[routing::ToString(code)];
+}
+
+void ErrorTypeCounter::PushError(api::ResultCode code)
+{
+  routing::RouterResultCode routingCode = routing::RouterResultCode::InternalError;
+  switch (code)
+  {
+  case api::ResultCode::ResponseOK: routingCode = routing::RouterResultCode::NoError; break;
+  case api::ResultCode::Error: routingCode = routing::RouterResultCode::RouteNotFound; break;
+  }
+  CHECK_NOT_EQUAL(routingCode, routing::RouterResultCode::InternalError,
+                  ("Wrong value of api::ResultCode:", static_cast<int>(code)));
+  PushError(routingCode);
+}
+
+void FillLabelsAndErrorTypeDistribution(std::vector<std::string> & labels, std::vector<double> & errorsTypeDistribution,
+                                        ErrorTypeCounter const & counter)
+{
+  errorsTypeDistribution.clear();
+
+  for (auto const & [errorName, errorCount] : counter.GetErrorsDistribution())
+  {
+    labels.emplace_back(errorName);
+    errorsTypeDistribution.emplace_back(errorCount);
+  }
+}
+
+void FillLabelsAndErrorTypeDistribution(std::vector<std::string> & labels,
+                                        std::vector<std::vector<double>> & errorsTypeDistribution,
+                                        ErrorTypeCounter const & counter, ErrorTypeCounter const & counterOld)
+{
+  errorsTypeDistribution.clear();
+  errorsTypeDistribution.resize(2);
+
+  FillLabelsAndErrorTypeDistribution(labels, errorsTypeDistribution[0], counter);
+
+  for (auto const & [_, errorCount] : counterOld.GetErrorsDistribution())
+    errorsTypeDistribution[1].emplace_back(errorCount);
+}
+}  // namespace routing_quality::routing_quality_tool

libs/routing/routing_quality/routing_quality_tool/error_type_counter.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "routing/routing_quality/api/api.hpp"
+
+#include "routing/routing_callbacks.hpp"
+
+#include <cstddef>
+#include <map>
+#include <string>
+
+namespace routing_quality::routing_quality_tool
+{
+class ErrorTypeCounter
+{
+public:
+  void PushError(routing::RouterResultCode code);
+  void PushError(api::ResultCode code);
+  std::map<std::string, size_t> const & GetErrorsDistribution() const { return m_errorCounter; }
+
+private:
+  // string representation of RouterResultCode to number of such codes.
+  std::map<std::string, size_t> m_errorCounter;
+};
+
+void FillLabelsAndErrorTypeDistribution(std::vector<std::string> & labels, std::vector<double> & errorsTypeDistribution,
+                                        ErrorTypeCounter const & counter);
+
+void FillLabelsAndErrorTypeDistribution(std::vector<std::string> & labels,
+                                        std::vector<std::vector<double>> & errorsTypeDistribution,
+                                        ErrorTypeCounter const & counter, ErrorTypeCounter const & counterOld);
+}  // namespace routing_quality::routing_quality_tool

libs/routing/routing_quality/routing_quality_tool/routing_quality_tool.cpp

@@ -0,0 +1,336 @@
+#include "routing/routing_quality/routing_quality_tool/benchmark_stat.hpp"
+#include "routing/routing_quality/routing_quality_tool/error_type_counter.hpp"
+#include "routing/routing_quality/routing_quality_tool/utils.hpp"
+
+#include "routing/routing_quality/api/api.hpp"
+
+#include "routing/routing_quality/waypoints.hpp"
+
+#include "routing/routes_builder/routes_builder.hpp"
+
+#include "platform/platform.hpp"
+
+#include "base/exception.hpp"
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+
+#include <algorithm>
+#include <exception>
+#include <limits>
+#include <utility>
+#include <vector>
+
+#include <gflags/gflags.h>
+
+DEFINE_string(mapsme_old_results, "", "Path to directory with previous mapsme router results.");
+DEFINE_string(mapsme_results, "", "Path to directory with mapsme router results.");
+DEFINE_string(api_results, "", "Path to directory with api router results.");
+
+DEFINE_string(save_results, "", "The directory where results of tool will be saved.");
+
+DEFINE_double(kml_percent, 0.0,
+              "The percent of routes for which kml file will be generated."
+              "With kml files you can make screenshots with a desktop app");
+
+DEFINE_bool(benchmark_stat, false, "Dump statistics about route time building.");
+
+namespace
+{
+// Shows distribution of simularity in comparison mode.
+static std::string const kPythonDistribution = "show_distribution.py";
+static std::string const kPythonBarDistributionError = "show_errors_distribution.py";
+
+double constexpr kBadETADiffPercent = std::numeric_limits<double>::max();
+
+void PrintHelpAndExit()
+{
+  std::stringstream usage;
+  usage << std::boolalpha;
+  usage << R"(Description:
+This tool takes two paths to directory with routes, that were dumped by routes_builder_tool and calculate some metrics. Here possible options of execution:
+
+    --mapsme_results and --api_results are required to compare mapsme and api.
+    or
+    --mapsme_results and --mapsme_old_results are required to compare mapsme and old mapsme version.
+    or
+    --mapsme_results and --benchmark_stat are required to calculate benchmark statistic of mapsme.
+    or
+    --mapsme_results and --mapsme_old_results and --benchmark_stat are required to calculate different statistics of comparison mapsme and old mapsme version.
+
+    --save_results is required for saving results in this directory.
+    --kml_percent may be used in non-benchamrk mode for dumping kmls and visual comparison different routes.
+)";
+
+  auto const addStringInfo = [&usage](auto const & arg, auto const & argValue)
+  {
+    using T = decltype(argValue);
+    usage << "\n\t" << arg << " is";
+    if (argValue == T{})
+      usage << " not set.";
+    else
+      usage << " set to: " << argValue;
+  };
+
+  addStringInfo("--mapsme_results", FLAGS_mapsme_results);
+  addStringInfo("--api_results", FLAGS_api_results);
+  addStringInfo("--mapsme_old_results", FLAGS_mapsme_old_results);
+  addStringInfo("--benchmark_stat", FLAGS_benchmark_stat);
+  addStringInfo("--save_results", FLAGS_save_results);
+  addStringInfo("--kml_percent", FLAGS_kml_percent);
+
+  usage << "\n\nType --help for usage.\n";
+  std::cout << usage.str();
+  exit(0);
+}
+
+bool HasHelpFlags(int argc, char ** argv)
+{
+  for (int i = 0; i < argc; ++i)
+  {
+    auto const value = std::string(argv[i]);
+    if (value == "--help" || value == "-h")
+      return true;
+  }
+  return false;
+}
+}  // namespace
+
+using namespace routing;
+using namespace routes_builder;
+using namespace routing_quality;
+using namespace routing_quality_tool;
+
+void PrintResults(std::vector<Result> && results, RoutesSaver & routesSaver)
+{
+  double sumSimilarity = 0.0;
+  double sumETADiffPercent = 0.0;
+  double goodETANumber = 0.0;
+  for (auto const & result : results)
+  {
+    sumSimilarity += result.m_similarity;
+    if (result.m_etaDiffPercent != kBadETADiffPercent)
+    {
+      sumETADiffPercent += result.m_etaDiffPercent;
+      goodETANumber += 1;
+    }
+  }
+
+  LOG(LINFO, ("Matched routes:", results.size()));
+  LOG(LINFO, ("Average similarity:", sumSimilarity / results.size()));
+  LOG(LINFO, ("Average eta difference by fullmathed routes:", sumETADiffPercent / goodETANumber, "%"));
+
+  auto const pythonScriptPath = base::JoinPath(FLAGS_save_results, kPythonDistribution);
+
+  std::vector<double> values;
+  values.reserve(results.size());
+  for (auto const & result : results)
+    values.emplace_back(result.m_similarity);
+
+  CreatePythonScriptForDistribution(pythonScriptPath, "Simularity distribution", values);
+
+  SimilarityCounter similarityCounter(routesSaver);
+
+  std::sort(results.begin(), results.end());
+  for (auto const & result : results)
+    similarityCounter.Push(result);
+
+  similarityCounter.CreateKmlFiles(FLAGS_kml_percent, results);
+}
+
+bool IsMapsmeVsApi()
+{
+  return !FLAGS_mapsme_results.empty() && !FLAGS_api_results.empty();
+}
+
+bool IsMapsmeVsMapsme()
+{
+  return !FLAGS_mapsme_results.empty() && !FLAGS_mapsme_old_results.empty() && !FLAGS_benchmark_stat;
+}
+
+bool IsMapsmeBenchmarkStat()
+{
+  return !FLAGS_mapsme_results.empty() && FLAGS_benchmark_stat && FLAGS_mapsme_old_results.empty();
+}
+
+bool IsMapsmeVsMapsmeBenchmarkStat()
+{
+  return !FLAGS_mapsme_results.empty() && FLAGS_benchmark_stat && !FLAGS_mapsme_old_results.empty();
+}
+
+void CheckDirExistence(std::string const & dir)
+{
+  CHECK(Platform::IsDirectory(dir), ("Can not find directory:", dir));
+}
+
+template <typename AnotherResult>
+void RunComparison(std::vector<std::pair<RoutesBuilder::Result, std::string>> && mapsmeResults,
+                   std::vector<std::pair<AnotherResult, std::string>> && anotherResults)
+{
+  ErrorTypeCounter mapsmeErrorCounter;
+  ErrorTypeCounter anotherErrorCounter;
+  ComparisonType type = IsMapsmeVsApi() ? ComparisonType::MapsmeVsApi : ComparisonType::MapsmeVsMapsme;
+  RoutesSaver routesSaver(FLAGS_save_results, type);
+  std::vector<Result> results;
+  size_t apiErrors = 0;
+
+  for (size_t i = 0; i < mapsmeResults.size(); ++i)
+  {
+    auto const & mapsmeResult = mapsmeResults[i].first;
+    auto const & mapsmeFile = mapsmeResults[i].second;
+
+    std::pair<AnotherResult, std::string> anotherResultPair;
+    if (!FindAnotherResponse(mapsmeResult, anotherResults, anotherResultPair))
+    {
+      LOG(LDEBUG, ("Can not find pair for:", i));
+      continue;
+    }
+
+    auto const & anotherResult = anotherResultPair.first;
+    auto const & anotherFile = anotherResultPair.second;
+
+    auto const & startLatLon = mercator::ToLatLon(anotherResult.GetStartPoint());
+    auto const & finishLatLon = mercator::ToLatLon(anotherResult.GetFinishPoint());
+
+    mapsmeErrorCounter.PushError(mapsmeResult.m_code);
+    anotherErrorCounter.PushError(anotherResult.m_code);
+
+    if (!mapsmeResult.IsCodeOK() && anotherResult.IsCodeOK())
+    {
+      routesSaver.PushError(mapsmeResult.m_code, startLatLon, finishLatLon);
+      continue;
+    }
+
+    if (anotherResult.GetRoutes().empty() || !anotherResult.IsCodeOK())
+    {
+      routesSaver.PushRivalError(startLatLon, finishLatLon);
+      ++apiErrors;
+      continue;
+    }
+
+    auto maxSimilarity = std::numeric_limits<Similarity>::min();
+    double etaDiff = kBadETADiffPercent;
+    auto const & mapsmeRoute = mapsmeResult.GetRoutes().back();
+    for (auto const & route : anotherResult.GetRoutes())
+    {
+      auto const similarity =
+          metrics::CompareByNumberOfMatchedWaypoints(mapsmeRoute.m_followedPolyline, route.GetWaypoints());
+
+      if (maxSimilarity < similarity)
+      {
+        maxSimilarity = similarity;
+        if (maxSimilarity == 1.0)
+        {
+          etaDiff =
+              100.0 * std::abs(route.GetETA() - mapsmeRoute.GetETA()) / std::max(route.GetETA(), mapsmeRoute.GetETA());
+        }
+      }
+    }
+
+    results.emplace_back(mapsmeFile, anotherFile, startLatLon, finishLatLon, maxSimilarity, etaDiff);
+  }
+
+  std::string const anotherSourceName = IsMapsmeVsMapsme() ? "old mapsme," : "api,";
+  LOG(LINFO, (apiErrors, "routes can not build via", anotherSourceName, "but mapsme do built them."));
+
+  PrintResults(std::move(results), routesSaver);
+
+  std::vector<std::string> errorLabels;
+  std::vector<std::vector<double>> errorsCount;
+  FillLabelsAndErrorTypeDistribution(errorLabels, errorsCount, mapsmeErrorCounter, anotherErrorCounter);
+
+  auto const pythonScriptPath = base::JoinPath(FLAGS_save_results, kPythonBarDistributionError);
+  CreatePythonBarByMap(pythonScriptPath, errorLabels, errorsCount,
+                       {"mapsme", IsMapsmeVsMapsme() ? "old mapsme" : "api"} /* legends */,
+                       "Type of errors" /* xlabel */, "Number of errors" /* ylabel */);
+}
+
+void CheckArgs()
+{
+  bool const modeIsChosen =
+      IsMapsmeVsApi() || IsMapsmeVsMapsme() || IsMapsmeBenchmarkStat() || IsMapsmeVsMapsmeBenchmarkStat();
+  if (!modeIsChosen)
+    PrintHelpAndExit();
+
+  CHECK(!FLAGS_save_results.empty(),
+        ("\n\n\t--save_results is required. Tool will save results there.", "\n\nType --help for usage."));
+}
+
+int Main(int argc, char ** argv)
+{
+  if (HasHelpFlags(argc, argv))
+    PrintHelpAndExit();
+
+  gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
+
+  CheckArgs();
+
+  if (Platform::IsFileExistsByFullPath(FLAGS_save_results))
+    CheckDirExistence(FLAGS_save_results);
+  else
+    CHECK_EQUAL(Platform::MkDir(FLAGS_save_results), Platform::EError::ERR_OK, ());
+
+  CheckDirExistence(FLAGS_mapsme_results);
+
+  CHECK(0.0 <= FLAGS_kml_percent && FLAGS_kml_percent <= 100.0, ("--kml_percent should be in interval: [0.0, 100.0]."));
+
+  LOG(LINFO, ("Start loading mapsme results."));
+  auto mapsmeResults = LoadResults<RoutesBuilder::Result>(FLAGS_mapsme_results);
+  LOG(LINFO, ("Receive:", mapsmeResults.size(), "routes from --mapsme_results."));
+
+  if (IsMapsmeVsApi())
+  {
+    LOG(LINFO, ("Start loading api results."));
+    auto apiResults = LoadResults<api::Response>(FLAGS_api_results);
+    LOG(LINFO, ("Receive:", apiResults.size(), "routes from --api_results."));
+    RunComparison(std::move(mapsmeResults), std::move(apiResults));
+  }
+  else if (IsMapsmeVsMapsmeBenchmarkStat())
+  {
+    LOG(LINFO, ("Benchmark different mapsme versions. Start loading old mapsme results."));
+    auto oldMapsmeResults = LoadResults<RoutesBuilder::Result>(FLAGS_mapsme_old_results);
+    LOG(LINFO, ("Receive:", oldMapsmeResults.size(), "routes from --mapsme_old_results."));
+    RunBenchmarkComparison(std::move(mapsmeResults), std::move(oldMapsmeResults), FLAGS_save_results);
+  }
+  else if (IsMapsmeVsMapsme())
+  {
+    LOG(LINFO, ("Start loading another mapsme results."));
+    auto oldMapsmeResults = LoadResults<RoutesBuilder::Result>(FLAGS_mapsme_old_results);
+    LOG(LINFO, ("Receive:", oldMapsmeResults.size(), "routes from --mapsme_old_results."));
+    RunComparison(std::move(mapsmeResults), std::move(oldMapsmeResults));
+  }
+  else if (IsMapsmeBenchmarkStat())
+  {
+    LOG(LINFO, ("Running in benchmark stat mode."));
+    RunBenchmarkStat(mapsmeResults, FLAGS_save_results);
+  }
+  else
+  {
+    UNREACHABLE();
+  }
+
+  return 0;
+}
+
+int main(int argc, char ** argv)
+{
+  try
+  {
+    return Main(argc, argv);
+  }
+  catch (RootException const & e)
+  {
+    LOG(LCRITICAL, ("Core exception:", e.Msg()));
+  }
+  catch (std::exception const & e)
+  {
+    LOG(LCRITICAL, ("Std exception:", e.what()));
+  }
+  catch (...)
+  {
+    LOG(LCRITICAL, ("Unknown exception."));
+  }
+
+  LOG(LINFO, ("Done."));
+
+  return 0;
+}

libs/routing/routing_quality/routing_quality_tool/utils.cpp

@@ -0,0 +1,514 @@
+#include "routing/routing_quality/routing_quality_tool/utils.hpp"
+
+#include "kml/serdes.hpp"
+#include "kml/types.hpp"
+
+#include "geometry/point_with_altitude.hpp"
+
+#include "base/assert.hpp"
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+
+#include <array>
+#include <iomanip>
+#include <utility>
+
+using namespace routing::routes_builder;
+
+namespace
+{
+using namespace routing_quality;
+
+void PrintWithSpaces(std::string const & str, size_t maxN)
+{
+  std::cout << str;
+  if (maxN <= str.size())
+    return;
+
+  maxN -= str.size();
+  for (size_t i = 0; i < maxN; ++i)
+    std::cout << " ";
+}
+
+std::vector<geometry::PointWithAltitude> ConvertToPointsWithAltitudes(std::vector<ms::LatLon> const & latlons)
+{
+  std::vector<geometry::PointWithAltitude> result;
+  result.reserve(latlons.size());
+  for (size_t i = 0; i < latlons.size(); ++i)
+    result.emplace_back(mercator::FromLatLon(latlons[i]), geometry::kDefaultAltitudeMeters);
+
+  return result;
+}
+
+std::string MakeStringFromPercent(double percent)
+{
+  std::stringstream ss;
+  ss << std::setprecision(2);
+  ss << percent;
+  std::string percentStr;
+  ss >> percentStr;
+  percentStr += "%";
+
+  return percentStr;
+}
+
+kml::BookmarkData CreateBookmark(m2::PointD const & point, bool isStart)
+{
+  kml::BookmarkData bookmarkData;
+  bookmarkData.m_color = {isStart ? kml::PredefinedColor::Red : kml::PredefinedColor::Blue, 0};
+  bookmarkData.m_point = point;
+
+  return bookmarkData;
+}
+
+template <typename AnotherResult>
+void SaveKmlFileDataTo(RoutesBuilder::Result const & mapsmeResult, AnotherResult const & apiResult,
+                       std::string const & kmlFile)
+{
+  static std::array<uint32_t, 5> const kColors = {
+      0xff0000ff,  // Red
+      0x0000ffff,  // Blue
+      0x00ff00ff,  // Green
+      0xffa500ff,  // Orange
+      0xa52a2aff   // Brown
+  };
+
+  kml::FileData kml;
+
+  size_t colorNumber = 0;
+  auto const addTrack = [&](kml::MultiGeometry::LineT && line)
+  {
+    kml::TrackData track;
+    track.m_geometry.m_lines.push_back(std::move(line));
+
+    CHECK_LESS(colorNumber, kColors.size(), ());
+    track.m_layers = {{5.0 /* lineWidth */, {kml::PredefinedColor::None, kColors[colorNumber++]}}};
+    kml.m_tracksData.emplace_back(std::move(track));
+  };
+
+  auto const & start = apiResult.GetStartPoint();
+  auto const & finish = apiResult.GetFinishPoint();
+
+  kml.m_bookmarksData.emplace_back(CreateBookmark(start, true /* isStart */));
+  kml.m_bookmarksData.emplace_back(CreateBookmark(finish, false /* isStart */));
+
+  auto const & resultPoints = mapsmeResult.GetRoutes().back().m_followedPolyline.GetPolyline().GetPoints();
+
+  kml::MultiGeometry::LineT mmTrack;
+  mmTrack.reserve(resultPoints.size());
+  for (auto const & pt : resultPoints)
+    mmTrack.emplace_back(pt, geometry::kDefaultAltitudeMeters);
+
+  addTrack(std::move(mmTrack));
+
+  for (auto const & route : apiResult.GetRoutes())
+    addTrack(ConvertToPointsWithAltitudes(route.GetWaypoints()));
+
+  kml::SerializerKml ser(kml);
+  FileWriter sink(kmlFile);
+  ser.Serialize(sink);
+}
+
+template <typename T>
+std::string CreatePythonArray(std::vector<T> const & data, bool isString = false)
+{
+  std::stringstream ss;
+  ss << "[";
+  for (auto const & item : data)
+  {
+    if (isString)
+      ss << "\"";
+    ss << item;
+    if (isString)
+      ss << "\"";
+    ss << ",";
+  }
+
+  auto result = ss.str();
+  if (data.empty())
+    result += "]";
+  else
+    result.back() = ']';
+
+  return result;
+}
+}  // namespace
+
+namespace routing_quality
+{
+namespace routing_quality_tool
+{
+Result::Result(std::string mapsmeDumpPath, std::string anotherDumpPath, ms::LatLon const & start,
+               ms::LatLon const & finish, double similarity, double etaDiffPercent)
+  : m_mapsmeDumpPath(std::move(mapsmeDumpPath))
+  , m_anotherDumpPath(std::move(anotherDumpPath))
+  , m_start(start)
+  , m_finish(finish)
+  , m_similarity(similarity)
+  , m_etaDiffPercent(etaDiffPercent)
+{}
+
+bool Result::operator<(Result const & rhs) const
+{
+  return m_similarity < rhs.m_similarity;
+}
+
+RoutesSaver::RoutesSaver(std::string targetDir, ComparisonType comparisonType)
+  : m_targetDir(std::move(targetDir))
+  , m_comparisonType(comparisonType)
+{
+  OpenOutputStream();
+}
+
+void RoutesSaver::PushRoute(Result const & result)
+{
+  WriteStartAndFinish(m_output, result.m_start, result.m_finish);
+  m_output << result.m_mapsmeDumpPath << " " << result.m_anotherDumpPath << std::endl;
+}
+
+void RoutesSaver::PushError(routing::RouterResultCode code, ms::LatLon const & start, ms::LatLon const & finish)
+{
+  CHECK_NOT_EQUAL(code, routing::RouterResultCode::NoError, ("Only errors codes."));
+
+  auto & ofstream = m_errorRoutes[code];
+  if (!ofstream.is_open())
+  {
+    std::string const fullpath = base::JoinPath(m_targetDir, DebugPrint(code) + ".routes");
+
+    ofstream.open(fullpath);
+    CHECK(ofstream.good(), ("Can not open:", fullpath, "for writing."));
+
+    ofstream << std::setprecision(20);
+    LOG(LINFO, ("Save routes with error:", code, "to:", fullpath));
+  }
+
+  WriteStartAndFinish(ofstream, start, finish);
+}
+
+void RoutesSaver::PushRivalError(ms::LatLon const & start, ms::LatLon const & finish)
+{
+  static std::string const kApiErrorsFile = "api_errors.routes";
+  if (!m_apiErrors.is_open())
+  {
+    std::string const & fullpath = base::JoinPath(m_targetDir, kApiErrorsFile);
+    m_apiErrors.open(fullpath);
+    CHECK(m_apiErrors.good(), ("Can not open:", fullpath, "for writing."));
+    m_apiErrors << std::setprecision(20);
+    LOG(LINFO, ("Routes that api can not build, but mapsme do, placed:", fullpath));
+  }
+
+  WriteStartAndFinish(m_apiErrors, start, finish);
+}
+
+void RoutesSaver::TurnToNextFile()
+{
+  ++m_fileNumber;
+
+  if (m_output.is_open())
+    m_output.close();
+
+  OpenOutputStream();
+}
+
+std::string RoutesSaver::GetCurrentPath() const
+{
+  std::string const fullpath = base::JoinPath(m_targetDir, std::to_string(m_fileNumber) + ".routes");
+
+  return fullpath;
+}
+
+std::string const & RoutesSaver::GetTargetDir() const
+{
+  return m_targetDir;
+}
+
+ComparisonType RoutesSaver::GetComparsionType() const
+{
+  return m_comparisonType;
+}
+
+void RoutesSaver::OpenOutputStream()
+{
+  std::string const & fullpath = GetCurrentPath();
+  m_output.open(fullpath);
+  CHECK(m_output.good(), ("Can not open:", fullpath, "for writing."));
+  m_output << std::setprecision(20);
+}
+
+void RoutesSaver::WriteStartAndFinish(std::ofstream & output, ms::LatLon const & start, ms::LatLon const & finish)
+{
+  output << start.m_lat << " " << start.m_lon << " " << finish.m_lat << " " << finish.m_lon << std::endl;
+}
+
+// This function creates python script that shows the distribution error.
+void CreatePythonScriptForDistribution(std::string const & pythonScriptPath, std::string const & title,
+                                       std::vector<double> const & values)
+{
+  std::ofstream python(pythonScriptPath);
+  CHECK(python.good(), ("Can not open:", pythonScriptPath, "for writing."));
+
+  std::string pythonArray = "[";
+  for (auto const & value : values)
+    pythonArray += std::to_string(value) + ",";
+
+  if (values.empty())
+    pythonArray += "]";
+  else
+    pythonArray.back() = ']';
+
+  python << R"(
+import numpy as np
+import matplotlib.pyplot as plt
+
+a = np.hstack()" +
+                pythonArray + R"()
+plt.hist(a, bins='auto')  # arguments are passed to np.histogram
+plt.title(")" + title +
+                R"(")
+plt.show()
+)";
+
+  LOG(LINFO, ("Run: python", pythonScriptPath, "to look at:", title));
+}
+
+void CreatePythonGraphByPointsXY(std::string const & pythonScriptPath, std::string const & xlabel,
+                                 std::string const & ylabel, std::vector<std::vector<m2::PointD>> const & graphics,
+                                 std::vector<std::string> const & legends)
+{
+  CHECK_EQUAL(legends.size(), graphics.size(), ());
+  std::ofstream python(pythonScriptPath);
+  CHECK(python.good(), ("Can not open:", pythonScriptPath, "for writing."));
+
+  std::string pythonArrayX = "[";
+  std::string pythonArrayY = "[";
+  std::string legendsArray = "[";
+  for (size_t i = 0; i < graphics.size(); ++i)
+  {
+    auto const & points = graphics[i];
+    pythonArrayX += "[";
+    pythonArrayY += "[";
+    legendsArray += "\"" + legends[i] + "\",";
+    for (auto const & point : points)
+    {
+      pythonArrayX += std::to_string(point.x) + ",";
+      pythonArrayY += std::to_string(point.y) + ",";
+    }
+
+    pythonArrayX += "],";
+    pythonArrayY += "],";
+  }
+
+  pythonArrayX.back() = ']';
+  pythonArrayY.back() = ']';
+  legendsArray.back() = ']';
+
+  python << R"(
+import pylab
+
+legends = )" + legendsArray +
+                R"(
+xlist = )" + pythonArrayX +
+                R"(
+ylist = )" + pythonArrayY +
+                R"(
+for (x, y, l) in zip(xlist, ylist, legends):
+  pylab.plot(x, y, label=l)
+
+pylab.xlabel(")" +
+                xlabel + R"(")
+pylab.ylabel(")" +
+                ylabel + R"(")
+pylab.legend()
+pylab.tight_layout()
+pylab.show()
+)";
+
+  LOG(LINFO, ("Run: python", pythonScriptPath, "to look at:", ylabel, "versus", xlabel));
+}
+
+void CreatePythonBarByMap(std::string const & pythonScriptPath, std::vector<std::string> const & barLabels,
+                          std::vector<std::vector<double>> const & barHeights, std::vector<std::string> const & legends,
+                          std::string const & xlabel, std::string const & ylabel, bool drawPercents)
+{
+  std::ofstream python(pythonScriptPath);
+  CHECK(python.good(), ("Can not open:", pythonScriptPath, "for writing."));
+
+  std::string labelsArray = CreatePythonArray(barLabels, true /* isString */);
+  std::string legendsArray = CreatePythonArray(legends, true /* isString */);
+  std::string counts = "[";
+  for (auto const & heights : barHeights)
+    counts += CreatePythonArray(heights) + ",";
+
+  if (barHeights.empty())
+    counts += "]";
+  else
+    counts.back() = ']';
+
+  std::string const formatString =
+      drawPercents ? "f'{round(height, 2)}({round(height / summ * 100, 2)}%)'" : "f'{round(height, 2)}'";
+
+  python << R"(
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+
+bar_width = 0.35
+labels = )" + labelsArray +
+                R"(
+legends = )" + legendsArray +
+                R"(
+counts = )" + counts +
+                R"(
+
+x = np.arange(len(labels))  # the label locations
+width = 0.35  # the width of the bars
+
+fig, ax = plt.subplots()
+bars = []
+for i in range(len(counts)):
+    bar = ax.bar(x + i * bar_width, counts[i], bar_width, label=legends[i])
+    bars.append(bar)
+
+ax.set_ylabel(')" +
+                ylabel + R"(')
+ax.set_title(')" +
+                xlabel + R"(')
+pos = (bar_width * (len(counts) - 1)) / 2
+ax.set_xticks(x + pos)
+ax.set_xticklabels(labels)
+ax.legend()
+
+def autolabel(rects, counts_ith):
+    summ = 0
+    for count in counts_ith:
+        summ += count
+
+    for rect in rects:
+        height = rect.get_height()
+        ax.annotate()" +
+                formatString + R"(,
+                    xy=(rect.get_x() + rect.get_width() / 2, height),
+                    xytext=(0, 3),  # 3 points vertical offset
+                    textcoords="offset points",
+                    ha='center', va='bottom')
+
+for i in range(len(counts)):
+    autolabel(bars[i], counts[i])
+fig.tight_layout()
+plt.show()
+)";
+
+  LOG(LINFO, ("Run: python", pythonScriptPath, "to look at bar:", ylabel, "versus", xlabel));
+}
+
+/// \brief |SimilarityCounter| groups routes that we compare by similarity, here we tune these groups.
+// static
+std::vector<SimilarityCounter::Interval> const SimilarityCounter::kIntervals = {
+    {"[0.0, 0.0]", 0, 0 + 1e-5}, {"[0.0, 0.1)", 0, 0.1},   {"[0.1, 0.2)", 0.1, 0.2}, {"[0.2, 0.3)", 0.2, 0.3},
+    {"[0.3, 0.6)", 0.3, 0.6},    {"[0.6, 0.8)", 0.6, 0.8}, {"[0.8, 1.0)", 0.8, 1.0}, {"[1.0, 1.0]", 1.0, 1.0 + 1e-5},
+};
+
+SimilarityCounter::SimilarityCounter(RoutesSaver & routesSaver) : m_routesSaver(routesSaver)
+{
+  for (auto const & interval : kIntervals)
+    m_routesCounter.emplace_back(interval.m_name, 0);
+}
+
+SimilarityCounter::~SimilarityCounter()
+{
+  size_t maxNumberLength = 0;
+  size_t maxKeyLength = 0;
+  size_t sum = 0;
+  for (auto const & item : m_routesCounter)
+  {
+    sum += item.m_routesNumber;
+    maxNumberLength = std::max(maxNumberLength, std::to_string(item.m_routesNumber).size());
+    maxKeyLength = std::max(maxKeyLength, item.m_intervalName.size());
+  }
+
+  for (auto const & item : m_routesCounter)
+  {
+    auto const percent = static_cast<double>(item.m_routesNumber) / sum * 100.0;
+    auto const percentStr = MakeStringFromPercent(percent);
+
+    PrintWithSpaces(item.m_intervalName, maxKeyLength + 1);
+    std::cout << " => ";
+    PrintWithSpaces(std::to_string(item.m_routesNumber), maxNumberLength + 1);
+    std::cout << "( " << percentStr << " )" << std::endl;
+  }
+}
+
+void SimilarityCounter::Push(Result const & result)
+{
+  auto left = kIntervals[m_currentInterval].m_left;
+  auto right = kIntervals[m_currentInterval].m_right;
+  while (!(left <= result.m_similarity && result.m_similarity < right))
+  {
+    ++m_currentInterval;
+    m_routesSaver.TurnToNextFile();
+    CHECK_LESS(m_currentInterval, m_routesCounter.size(), ());
+    left = kIntervals[m_currentInterval].m_left;
+    right = kIntervals[m_currentInterval].m_right;
+  }
+
+  CHECK_LESS(m_currentInterval, m_routesCounter.size(), ());
+  if (m_routesCounter[m_currentInterval].m_routesNumber == 0)
+  {
+    LOG(LINFO, ("Save routes with:", m_routesCounter[m_currentInterval].m_intervalName,
+                "similarity to:", m_routesSaver.GetCurrentPath()));
+  }
+
+  CHECK_LESS(m_currentInterval, m_routesCounter.size(), ());
+  ++m_routesCounter[m_currentInterval].m_routesNumber;
+  m_routesSaver.PushRoute(result);
+}
+
+void SimilarityCounter::CreateKmlFiles(double percent, std::vector<Result> const & results)
+{
+  size_t realResultIndex = 0;
+  for (size_t intervalId = 0; intervalId < m_routesCounter.size(); ++intervalId)
+  {
+    std::string savePath = base::JoinPath(m_routesSaver.GetTargetDir(), std::to_string(intervalId));
+
+    auto const currentSize = m_routesCounter[intervalId].m_routesNumber;
+    auto const resultSize = static_cast<size_t>(currentSize * percent / 100.0);
+    if (resultSize == 0)
+      continue;
+
+    auto const mkdirRes = Platform::MkDir(savePath);
+    CHECK(mkdirRes == Platform::EError::ERR_OK || mkdirRes == Platform::EError::ERR_FILE_ALREADY_EXISTS,
+          ("Cannot create dir:", savePath));
+
+    LOG(LINFO, ("Saving", resultSize, "kmls for:", m_routesCounter[intervalId].m_intervalName, "to:", savePath));
+
+    auto const step = static_cast<size_t>(currentSize / resultSize);
+
+    for (size_t i = 0; i < currentSize; ++i)
+    {
+      if (i % step != 0)
+      {
+        ++realResultIndex;
+        continue;
+      }
+
+      CHECK_LESS(realResultIndex, results.size(), ());
+      auto const mapsmeResult = RoutesBuilder::Result::Load(results[realResultIndex].m_mapsmeDumpPath);
+
+      std::string const kmlFile = base::JoinPath(savePath, std::to_string(i) + ".kml");
+      if (m_routesSaver.GetComparsionType() == ComparisonType::MapsmeVsApi)
+      {
+        auto const apiResult = api::Response::Load(results[realResultIndex].m_anotherDumpPath);
+        SaveKmlFileDataTo(mapsmeResult, apiResult, kmlFile);
+      }
+      else
+      {
+        auto const mapsmeAnotherResult = RoutesBuilder::Result::Load(results[realResultIndex].m_anotherDumpPath);
+        SaveKmlFileDataTo(mapsmeResult, mapsmeAnotherResult, kmlFile);
+      }
+
+      ++realResultIndex;
+    }
+  }
+}
+}  // namespace routing_quality_tool
+}  // namespace routing_quality

libs/routing/routing_quality/routing_quality_tool/utils.hpp

@@ -0,0 +1,184 @@
+#pragma once
+
+#include "routing/routes_builder/routes_builder.hpp"
+#include "routing/routing_quality/api/api.hpp"
+
+#include "routing/routing_callbacks.hpp"
+
+#include "platform/platform.hpp"
+
+#include "geometry/latlon.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+
+#include <algorithm>
+#include <fstream>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace routing_quality
+{
+namespace routing_quality_tool
+{
+enum class ComparisonType
+{
+  MapsmeVsMapsme,
+  MapsmeVsApi
+};
+
+template <typename Result>
+std::vector<std::pair<Result, std::string>> LoadResults(std::string const & dir)
+{
+  std::vector<std::pair<Result, std::string>> result;
+
+  std::vector<std::string> files;
+  Platform::GetFilesByExt(dir, Result::kDumpExtension, files);
+  std::sort(files.begin(), files.end());
+
+  double lastPercent = 0.0;
+  double count = 0;
+
+  for (auto const & file : files)
+  {
+    double const curPercent = (count + 1) / files.size() * 100.0;
+    if (curPercent - lastPercent > 5.0 || count + 1 == files.size())
+    {
+      lastPercent = curPercent;
+      LOG(LINFO, ("Progress:", lastPercent, "%"));
+    }
+
+    auto const fullPath = base::JoinPath(dir, file);
+    result.emplace_back(Result::Load(fullPath), fullPath);
+    ++count;
+  }
+
+  return result;
+}
+
+template <typename AnotherResult>
+bool AreRoutesWithSameEnds(routing::routes_builder::RoutesBuilder::Result const & mapsmeResult,
+                           AnotherResult const & anotherResult)
+{
+  CHECK_EQUAL(mapsmeResult.m_params.m_checkpoints.GetPoints().size(), 2, ());
+  CHECK_EQUAL(mapsmeResult.GetVehicleType(), anotherResult.GetVehicleType(), ());
+
+  auto const & start = mapsmeResult.GetStartPoint();
+  auto const & finish = mapsmeResult.GetFinishPoint();
+
+  auto const & anotherStart = anotherResult.GetStartPoint();
+  auto const & anotherFinish = anotherResult.GetFinishPoint();
+
+  double constexpr kEps = 1e-10;
+  return AlmostEqualAbs(start, anotherStart, kEps) && AlmostEqualAbs(finish, anotherFinish, kEps);
+}
+
+template <typename AnotherResult>
+bool FindAnotherResponse(routing::routes_builder::RoutesBuilder::Result const & mapsmeResult,
+                         std::vector<std::pair<AnotherResult, std::string>> const & apiResults,
+                         std::pair<AnotherResult, std::string> & anotherResult)
+{
+  for (auto const & result : apiResults)
+  {
+    if (!AreRoutesWithSameEnds(mapsmeResult, result.first))
+      continue;
+
+    anotherResult = result;
+    return true;
+  }
+
+  return false;
+}
+
+struct Result
+{
+  Result(std::string mapsmeDumpPath, std::string anotherDumpPath, ms::LatLon const & start, ms::LatLon const & finish,
+         double similarity, double etaDiffPercent);
+
+  bool operator<(Result const & rhs) const;
+
+  std::string m_mapsmeDumpPath;
+  std::string m_anotherDumpPath;
+
+  ms::LatLon m_start;
+  ms::LatLon m_finish;
+
+  // value in range: [0, 1]
+  double m_similarity;
+  double m_etaDiffPercent;
+};
+
+class RoutesSaver
+{
+public:
+  RoutesSaver(std::string targetDir, ComparisonType comparisonType);
+
+  void PushRoute(Result const & result);
+  void PushError(routing::RouterResultCode code, ms::LatLon const & start, ms::LatLon const & finish);
+  void PushRivalError(ms::LatLon const & start, ms::LatLon const & finish);
+  void TurnToNextFile();
+  std::string GetCurrentPath() const;
+  std::string const & GetTargetDir() const;
+  ComparisonType GetComparsionType() const;
+
+private:
+  void OpenOutputStream();
+  void WriteStartAndFinish(std::ofstream & output, ms::LatLon const & start, ms::LatLon const & finish);
+
+  std::ofstream m_apiErrors;
+  std::string m_targetDir;
+  uint32_t m_fileNumber = 0;
+  std::ofstream m_output;
+  std::map<routing::RouterResultCode, std::ofstream> m_errorRoutes;
+  ComparisonType m_comparisonType;
+};
+
+class SimilarityCounter
+{
+public:
+  struct Interval
+  {
+    std::string m_name;
+    // [m_left, m_right)
+    double m_left;
+    double m_right;
+  };
+
+  struct Item
+  {
+    Item(std::string name, uint32_t n) : m_intervalName(std::move(name)), m_routesNumber(n) {}
+
+    std::string m_intervalName;
+    uint32_t m_routesNumber = 0;
+  };
+
+  static std::vector<Interval> const kIntervals;
+
+  explicit SimilarityCounter(RoutesSaver & routesSaver);
+  ~SimilarityCounter();
+
+  void Push(Result const & result);
+  void CreateKmlFiles(double percent, std::vector<Result> const & results);
+
+private:
+  RoutesSaver & m_routesSaver;
+  size_t m_currentInterval = 0;
+  std::vector<Item> m_routesCounter;
+};
+
+void CreatePythonScriptForDistribution(std::string const & pythonScriptPath, std::string const & title,
+                                       std::vector<double> const & values);
+
+void CreatePythonGraphByPointsXY(std::string const & pythonScriptPath, std::string const & xlabel,
+                                 std::string const & ylabel, std::vector<std::vector<m2::PointD>> const & graphics,
+                                 std::vector<std::string> const & legends);
+
+/// \brief Create python file, that show bar graph, where labels of bars are keys of |stat| and
+/// heights area values of |stat|.
+void CreatePythonBarByMap(std::string const & pythonScriptPath, std::vector<std::string> const & barLabels,
+                          std::vector<std::vector<double>> const & barHeights, std::vector<std::string> const & legends,
+                          std::string const & xlabel, std::string const & ylabel, bool drawPercents = true);
+}  // namespace routing_quality_tool
+}  // namespace routing_quality