Repo created

libs/routing/routes_builder/routes_builder_tool/CMakeLists.txt

@@ -0,0 +1,15 @@
+project(routes_builder_tool)
+
+set(SRC
+  routes_builder_tool.cpp
+  utils.cpp
+  utils.hpp
+)
+
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  routes_builder
+  routing_api
+  gflags::gflags
+)

libs/routing/routes_builder/routes_builder_tool/routes_builder_tool.cpp

@@ -0,0 +1,139 @@
+#include "routing/routes_builder/routes_builder_tool/utils.hpp"
+
+#include "routing/routes_builder/routes_builder.hpp"
+
+#include "platform/platform.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+
+#include <exception>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <gflags/gflags.h>
+
+DEFINE_uint64(threads, 0, "The number of threads. std::thread::hardware_concurrency() is used by default.");
+
+DEFINE_string(routes_file, "",
+              "Path to file with routes in format: \n\t"
+              "first_start_lat first_start_lon first_finish_lat first_finish_lon\n\t"
+              "second_start_lat second_start_lon second_finish_lat second_finish_lon\n\t"
+              "...");
+
+DEFINE_string(dump_path, "",
+              "Path where routes will be dumped after building."
+              "Useful for intermediate results, because routes building "
+              "is a long process.");
+
+DEFINE_string(data_path, "", "Data path.");
+DEFINE_string(resources_path, "", "Resources path.");
+
+DEFINE_string(api_name, "", "Api name, current options: mapbox,google");
+DEFINE_string(api_token, "", "Token for chosen api.");
+
+DEFINE_uint64(start_from, 0, "The line number from which the tool should start reading.");
+
+DEFINE_int32(timeout, 10 * 60,
+             "Timeout in seconds for each route building. "
+             "0 means without timeout (default: 10 minutes).");
+
+DEFINE_bool(verbose, false, "Verbose logging (default: false)");
+
+DEFINE_int32(launches_number, 1, "Number of launches of routes buildings. Needs for benchmarking (default: 1)");
+DEFINE_string(vehicle_type, "car", "Vehicle type: car|pedestrian|bicycle|transit. (Only for mapsme).");
+
+using namespace routing;
+using namespace routes_builder;
+using namespace routing_quality;
+
+namespace
+{
+bool IsLocalBuild()
+{
+  return !FLAGS_routes_file.empty() && FLAGS_api_name.empty() && FLAGS_api_token.empty();
+}
+
+bool IsApiBuild()
+{
+  return !FLAGS_routes_file.empty() && !FLAGS_api_name.empty() && !FLAGS_api_token.empty();
+}
+
+void CheckDirExistence(std::string const & dir)
+{
+  CHECK(Platform::IsDirectory(dir), ("Can not find directory:", dir));
+}
+}  // namespace
+
+int Main(int argc, char ** argv)
+{
+  gflags::SetUsageMessage("This tool provides routes building for them further analyze.");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  CHECK_GREATER_OR_EQUAL(FLAGS_timeout, 0, ("Timeout should be greater than zero."));
+
+  CHECK(!FLAGS_routes_file.empty(), ("\n\n\t--routes_file is required.", "\n\nType --help for usage."));
+
+  if (!FLAGS_data_path.empty())
+    GetPlatform().SetWritableDirForTests(FLAGS_data_path);
+
+  if (!FLAGS_resources_path.empty())
+    GetPlatform().SetResourceDir(FLAGS_resources_path);
+
+  CHECK(IsLocalBuild() || IsApiBuild(),
+        ("\n\n\t--routes_file empty is:", FLAGS_routes_file.empty(), "\n\t--api_name empty is:", FLAGS_api_name.empty(),
+         "\n\t--api_token empty is:", FLAGS_api_token.empty(), "\n\nType --help for usage."));
+
+  CHECK(!FLAGS_dump_path.empty(),
+        ("\n\n\t--dump_path is empty. It makes no sense to run this tool. No result will be saved.",
+         "\n\nType --help for usage."));
+
+  CHECK_GREATER_OR_EQUAL(FLAGS_launches_number, 1, ());
+
+  if (Platform::IsFileExistsByFullPath(FLAGS_dump_path))
+    CheckDirExistence(FLAGS_dump_path);
+  else
+    CHECK_EQUAL(Platform::MkDir(FLAGS_dump_path), Platform::EError::ERR_OK, ());
+
+  if (IsLocalBuild())
+  {
+    auto const launchesNumber = static_cast<uint32_t>(FLAGS_launches_number);
+    if (launchesNumber > 1)
+      LOG(LINFO, ("Benchmark mode is activated. Each route will be built", launchesNumber, "times."));
+
+    BuildRoutes(FLAGS_routes_file, FLAGS_dump_path, FLAGS_start_from, FLAGS_threads, FLAGS_timeout, FLAGS_vehicle_type,
+                FLAGS_verbose, launchesNumber);
+  }
+
+  if (IsApiBuild())
+  {
+    auto api = CreateRoutingApi(FLAGS_api_name, FLAGS_api_token);
+    BuildRoutesWithApi(std::move(api), FLAGS_routes_file, FLAGS_dump_path, FLAGS_start_from);
+  }
+
+  return 0;
+}
+
+int main(int argc, char ** argv)
+{
+  try
+  {
+    Main(argc, argv);
+  }
+  catch (RootException const & e)
+  {
+    LOG(LERROR, ("Core exception:", e.Msg()));
+  }
+  catch (std::exception const & e)
+  {
+    LOG(LERROR, ("Std exception:", e.what()));
+  }
+  catch (...)
+  {
+    LOG(LERROR, ("Unknown exception."));
+  }
+
+  LOG(LINFO, ("Done."));
+  return 0;
+}

libs/routing/routes_builder/routes_builder_tool/utils.cpp

@@ -0,0 +1,277 @@
+#include "routing/routes_builder/routes_builder_tool/utils.hpp"
+
+#include "routing/routing_quality/api/google/google_api.hpp"
+#include "routing/routing_quality/api/mapbox/mapbox_api.hpp"
+
+#include "routing/checkpoints.hpp"
+#include "routing/vehicle_mask.hpp"
+
+#include "platform/platform.hpp"
+
+#include "geometry/latlon.hpp"
+#include "geometry/mercator.hpp"
+
+#include "base/assert.hpp"
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+#include "base/timer.hpp"
+
+#include <algorithm>
+#include <array>
+#include <chrono>
+#include <future>
+#include <iostream>
+#include <optional>
+#include <thread>
+#include <tuple>
+
+namespace routing
+{
+namespace routes_builder
+{
+using namespace routing_quality;
+
+namespace
+{
+size_t GetNumberOfLines(std::string const & filename)
+{
+  std::ifstream input(filename);
+  CHECK(input.good(), ("Error during opening:", filename));
+
+  size_t count = 0;
+  std::string line;
+  while (std::getline(input, line))
+    ++count;
+
+  return count;
+}
+
+routing::VehicleType ConvertVehicleTypeFromString(std::string const & str)
+{
+  if (str == "car")
+    return routing::VehicleType::Car;
+  if (str == "pedestrian")
+    return routing::VehicleType::Pedestrian;
+  if (str == "bicycle")
+    return routing::VehicleType::Bicycle;
+  if (str == "transit")
+    return routing::VehicleType::Transit;
+
+  CHECK(false, ("Unknown vehicle type:", str));
+  UNREACHABLE();
+}
+}  // namespace
+
+void BuildRoutes(std::string const & routesPath, std::string const & dumpPath, uint64_t startFrom,
+                 uint64_t threadsNumber, uint32_t timeoutPerRouteSeconds, std::string const & vehicleTypeStr,
+                 bool verbose, uint32_t launchesNumber)
+{
+  CHECK(Platform::IsFileExistsByFullPath(routesPath), ("Can not find file:", routesPath));
+  CHECK(!dumpPath.empty(), ("Empty dumpPath."));
+
+  std::ifstream input(routesPath);
+  CHECK(input.good(), ("Error during opening:", routesPath));
+
+  if (!threadsNumber)
+  {
+    auto const hardwareConcurrency = std::thread::hardware_concurrency();
+    threadsNumber = hardwareConcurrency > 0 ? hardwareConcurrency : 2;
+  }
+
+  RoutesBuilder routesBuilder(threadsNumber);
+
+  std::vector<std::future<RoutesBuilder::Result>> tasks;
+  double lastPercent = 0.0;
+
+  auto const vehicleType = ConvertVehicleTypeFromString(vehicleTypeStr);
+  {
+    RoutesBuilder::Params params;
+    params.m_type = vehicleType;
+    params.m_timeoutSeconds = timeoutPerRouteSeconds;
+    params.m_launchesNumber = launchesNumber;
+
+    base::ScopedLogLevelChanger changer(verbose ? base::LogLevel::LINFO : base::LogLevel::LERROR);
+    ms::LatLon start;
+    ms::LatLon finish;
+    size_t startFromCopy = startFrom;
+    while (input >> start.m_lat >> start.m_lon >> finish.m_lat >> finish.m_lon)
+    {
+      if (startFromCopy > 0)
+      {
+        --startFromCopy;
+        continue;
+      }
+
+      auto const startPoint = mercator::FromLatLon(start);
+      auto const finishPoint = mercator::FromLatLon(finish);
+
+      params.m_checkpoints = Checkpoints(std::vector<m2::PointD>({startPoint, finishPoint}));
+      tasks.emplace_back(routesBuilder.ProcessTaskAsync(params));
+    }
+
+    LOG_FORCE(LINFO, ("Created:", tasks.size(), "tasks, vehicle type:", vehicleType));
+    base::Timer timer;
+    for (size_t i = 0; i < tasks.size(); ++i)
+    {
+      size_t shiftIndex = i + startFrom;
+      auto & task = tasks[i];
+      task.wait();
+
+      auto const result = task.get();
+      if (result.m_code == RouterResultCode::Cancelled)
+        LOG_FORCE(LINFO, ("Route:", i, "(", i + 1, "line of file) was building too long."));
+
+      std::string const fullPath =
+          base::JoinPath(dumpPath, std::to_string(shiftIndex) + RoutesBuilder::Result::kDumpExtension);
+
+      RoutesBuilder::Result::Dump(result, fullPath);
+
+      double const curPercent = static_cast<double>(shiftIndex + 1) / (tasks.size() + startFrom) * 100.0;
+
+      if (curPercent - lastPercent > 1.0 || shiftIndex + 1 == tasks.size())
+      {
+        lastPercent = curPercent;
+        LOG_FORCE(LINFO, ("Progress:", lastPercent, "%"));
+      }
+    }
+    LOG_FORCE(LINFO, ("BuildRoutes() took:", timer.ElapsedSeconds(), "seconds."));
+  }
+}
+
+std::optional<std::tuple<ms::LatLon, ms::LatLon, int32_t>> ParseApiLine(std::ifstream & input)
+{
+  std::string line;
+  if (!std::getline(input, line))
+    return {};
+
+  std::stringstream ss;
+  ss << line;
+
+  ms::LatLon start;
+  ms::LatLon finish;
+  ss >> start.m_lat >> start.m_lon >> finish.m_lat >> finish.m_lon;
+
+  int32_t utcOffset = 0;
+  if (!(ss >> utcOffset))
+    utcOffset = 0;
+
+  return {{start, finish, utcOffset}};
+}
+
+void BuildRoutesWithApi(std::unique_ptr<api::RoutingApi> routingApi, std::string const & routesPath,
+                        std::string const & dumpPath, int64_t startFrom)
+{
+  std::ifstream input(routesPath);
+  CHECK(input.good(), ("Error during opening:", routesPath));
+
+  size_t constexpr kResponseBuffer = 50;
+  std::array<api::Response, kResponseBuffer> result;
+
+  api::Params params;
+  params.m_type = api::VehicleType::Car;
+
+  size_t rps = 0;
+  base::HighResTimer timer;
+  auto const getElapsedMilliSeconds = [&timer]()
+  {
+    auto const ms = timer.ElapsedMilliseconds();
+    LOG(LDEBUG, ("Elapsed:", ms, "ms"));
+    return ms;
+  };
+
+  auto const drop = [&]()
+  {
+    rps = 0;
+    timer.Reset();
+  };
+
+  auto const sleepIfNeed = [&]()
+  {
+    // Greater than 1 second.
+    if (getElapsedMilliSeconds() > 1000)
+    {
+      drop();
+      return;
+    }
+
+    ++rps;
+    if (rps >= routingApi->GetMaxRPS())
+    {
+      LOG(LDEBUG, ("Sleep for 2 seconds."));
+      std::this_thread::sleep_for(std::chrono::seconds(2));
+      drop();
+    }
+  };
+
+  size_t count = 0;
+  size_t prevDumpedNumber = 0;
+  auto const dump = [&]()
+  {
+    for (size_t i = 0; i < count; ++i)
+    {
+      std::string filepath =
+          base::JoinPath(dumpPath, std::to_string(prevDumpedNumber + i) + api::Response::kDumpExtension);
+
+      api::Response::Dump(filepath, result[i]);
+    }
+
+    prevDumpedNumber += count;
+    count = 0;
+  };
+
+  size_t const tasksNumber = GetNumberOfLines(routesPath);
+  LOG(LINFO, ("Receive:", tasksNumber, "for api:", routingApi->GetApiName()));
+  double lastPercent = 0.0;
+
+  ms::LatLon start;
+  ms::LatLon finish;
+  int32_t utcOffset;
+  while (auto parsedData = ParseApiLine(input))
+  {
+    if (startFrom > 0)
+    {
+      --startFrom;
+      ++count;
+      if (count == kResponseBuffer)
+      {
+        prevDumpedNumber += count;
+        count = 0;
+      }
+
+      continue;
+    }
+
+    if (count == kResponseBuffer)
+      dump();
+
+    double const curPercent = static_cast<double>(prevDumpedNumber + count) / tasksNumber * 100.0;
+    if (curPercent - lastPercent > 1.0)
+    {
+      lastPercent = curPercent;
+      LOG(LINFO, ("Progress:", lastPercent, "%"));
+    }
+
+    std::tie(start, finish, utcOffset) = *parsedData;
+    params.m_waypoints = routing::Checkpoints(mercator::FromLatLon(start), mercator::FromLatLon(finish));
+
+    result[count] = routingApi->CalculateRoute(params, utcOffset);
+
+    sleepIfNeed();
+    ++count;
+  }
+
+  dump();
+}
+
+std::unique_ptr<routing_quality::api::RoutingApi> CreateRoutingApi(std::string const & name, std::string const & token)
+{
+  if (name == "mapbox")
+    return std::make_unique<routing_quality::api::mapbox::MapboxApi>(token);
+  if (name == "google")
+    return std::make_unique<routing_quality::api::google::GoogleApi>(token);
+
+  LOG(LERROR, ("Unsupported api name:", name));
+  UNREACHABLE();
+}
+}  // namespace routes_builder
+}  // namespace routing

libs/routing/routes_builder/routes_builder_tool/utils.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "routing/routing_quality/api/api.hpp"
+
+#include "routing/routes_builder/routes_builder.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace routing
+{
+namespace routes_builder
+{
+void BuildRoutes(std::string const & routesPath, std::string const & dumpPath, uint64_t startFrom,
+                 uint64_t threadsNumber, uint32_t timeoutPerRouteSeconds, std::string const & vehicleType, bool verbose,
+                 uint32_t launchesNumber);
+
+void BuildRoutesWithApi(std::unique_ptr<routing_quality::api::RoutingApi> routingApi, std::string const & routesPath,
+                        std::string const & dumpPath, int64_t startFrom);
+
+std::unique_ptr<routing_quality::api::RoutingApi> CreateRoutingApi(std::string const & name, std::string const & token);
+}  // namespace routes_builder
+}  // namespace routing