Repo created

tools/track_analyzing/track_analyzer/CMakeLists.txt

@@ -0,0 +1,26 @@
+project(track_analyzer)
+
+set(SRC
+  cmd_balance_csv.cpp
+  cmd_balance_csv.hpp
+  cmd_cpp_track.cpp
+  cmd_gpx.cpp
+  cmd_match.cpp
+  cmd_table.cpp
+  cmd_track.cpp
+  cmd_tracks.cpp
+  cmd_unmatched_tracks.cpp
+  crossroad_checker.cpp
+  crossroad_checker.hpp
+  track_analyzer.cpp
+  utils.cpp
+  utils.hpp
+)
+
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  map
+  track_analyzing
+  gflags::gflags
+)

tools/track_analyzing/track_analyzer/cmd_balance_csv.cpp

@@ -0,0 +1,289 @@
+#include "track_analyzing/track_analyzer/cmd_balance_csv.hpp"
+
+#include "track_analyzing/track_analyzer/utils.hpp"
+
+#include "storage/storage.hpp"
+
+#include "coding/csv_reader.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+#include "base/math.hpp"
+
+#include <algorithm>
+#include <fstream>
+#include <ostream>
+#include <random>
+#include <utility>
+
+namespace
+{
+double constexpr kSumFractionEps = 1e-5;
+size_t constexpr kTableColumns = 19;
+}  // namespace
+
+namespace track_analyzing
+{
+using namespace std;
+
+void FillTable(basic_istream<char> & tableCsvStream, MwmToDataPoints & matchedDataPoints, vector<TableRow> & table)
+{
+  for (auto const & row :
+       coding::CSVRunner(coding::CSVReader(tableCsvStream, true /* hasHeader */, ',' /* delimiter */)))
+  {
+    CHECK_EQUAL(row.size(), kTableColumns, (row));
+    auto const & mwmName = row[kMwmNameCsvColumn];
+    matchedDataPoints[mwmName] += 1;
+    table.push_back(row);
+  }
+}
+
+void RemoveKeysSmallValue(MwmToDataPoints & checkedMap, MwmToDataPoints & additionalMap, uint64_t ignoreDataPointNumber)
+{
+  CHECK(AreKeysEqual(additionalMap, checkedMap),
+        ("Mwms in |checkedMap| and in |additionalMap| should have the same set of keys."));
+
+  for (auto it = checkedMap.begin(); it != checkedMap.end();)
+  {
+    auto const dataPointNumberAfterMatching = it->second;
+    if (dataPointNumberAfterMatching > ignoreDataPointNumber)
+    {
+      ++it;
+      continue;
+    }
+
+    auto const & mwmName = it->first;
+    additionalMap.erase(mwmName);
+    it = checkedMap.erase(it);
+  }
+}
+
+MwmToDataPointFraction GetMwmToDataPointFraction(MwmToDataPoints const & numberMapping)
+{
+  CHECK(!numberMapping.empty(), ());
+  uint64_t const totalDataPointNumber = ValueSum(numberMapping);
+
+  MwmToDataPointFraction fractionMapping;
+  for (auto const & kv : numberMapping)
+    fractionMapping.emplace(kv.first, static_cast<double>(kv.second) / totalDataPointNumber);
+
+  return fractionMapping;
+}
+
+MwmToDataPoints CalcsMatchedDataPointsToKeepDistribution(MwmToDataPoints const & matchedDataPoints,
+                                                         MwmToDataPointFraction const & distributionFractions)
+{
+  CHECK(!matchedDataPoints.empty(), ());
+  CHECK(AreKeysEqual(matchedDataPoints, distributionFractions),
+        ("Mwms in |matchedDataPoints| and in |distributionFractions| should have the same set of keys."));
+  CHECK(AlmostEqualAbs(ValueSum(distributionFractions), 1.0, kSumFractionEps), (ValueSum(distributionFractions)));
+
+  auto const matchedFractions = GetMwmToDataPointFraction(matchedDataPoints);
+
+  double maxRatio = 0.0;
+  double maxRationDistributionFraction = 0.0;
+  uint64_t maxRationMatchedDataPointNumber = 0;
+  string maxMwmName;
+  // First, let's find such mwm that all |matchedDataPoints| of it may be used and
+  // the distribution set in |distributionFractions| will be kept. It's an mwm
+  // on which the maximum of ratio
+  // (percent of data points in the distribution) / (percent of matched data points)
+  // is reached.
+  for (auto const & kv : matchedDataPoints)
+  {
+    auto const & mwmName = kv.first;
+    auto const matchedDataPointNumber = kv.second;
+    auto const itDistr = distributionFractions.find(mwmName);
+    CHECK(itDistr != distributionFractions.cend(), (mwmName));
+    auto const distributionFraction = itDistr->second;
+    auto const itMatched = matchedFractions.find(mwmName);
+    CHECK(itMatched != matchedFractions.cend(), (mwmName));
+    double const matchedFraction = itMatched->second;
+
+    double const ratio = distributionFraction / matchedFraction;
+    if (ratio > maxRatio)
+    {
+      maxRatio = ratio;
+      maxRationDistributionFraction = distributionFraction;
+      maxRationMatchedDataPointNumber = matchedDataPointNumber;
+      maxMwmName = mwmName;
+    }
+  }
+  LOG(LINFO, ("Max ration mwm name:", maxMwmName, "max ration:", maxRatio));
+  CHECK_GREATER_OR_EQUAL(maxRatio, 1.0, ());
+
+  // Having data points number in mwm on which the maximum is reached and
+  // fraction of data point in the distribution for this mwm (less or equal 1.0) it's possible to
+  // calculate the total matched points number which may be used to keep the distribution.
+  auto const totalMatchedPointNumberToKeepDistribution =
+      static_cast<uint64_t>(maxRationMatchedDataPointNumber / maxRationDistributionFraction);
+  CHECK_LESS_OR_EQUAL(totalMatchedPointNumberToKeepDistribution, ValueSum(matchedDataPoints), ());
+
+  // Having total maximum matched point number which let to keep the distribution
+  // and which fraction for each mwm should be used to correspond to the distribution
+  // it's possible to find matched data point number which may be used for each mwm
+  // to fulfil the distribution.
+  MwmToDataPoints matchedDataPointsToKeepDistribution;
+  for (auto const & kv : distributionFractions)
+  {
+    auto const & mwm = kv.first;
+    auto const fraction = kv.second;
+    auto const matchedDataPointsToKeepDistributionForMwm =
+        static_cast<uint64_t>(fraction * totalMatchedPointNumberToKeepDistribution);
+    matchedDataPointsToKeepDistribution.emplace(mwm, matchedDataPointsToKeepDistributionForMwm);
+
+    if (matchedDataPointsToKeepDistributionForMwm == 0)
+    {
+      LOG(LWARNING, ("Zero points should be put to", mwm, "to keep the distribution. Distribution fraction:", fraction,
+                     "totalMatchedPointNumberToKeepDistribution:", totalMatchedPointNumberToKeepDistribution));
+    }
+  }
+  return matchedDataPointsToKeepDistribution;
+}
+
+MwmToDataPoints BalancedDataPointNumber(MwmToDataPoints && matchedDataPoints, MwmToDataPoints && distribution,
+                                        uint64_t ignoreDataPointsNumber)
+{
+  // Removing every mwm from |distribution| and |matchedDataPoints| if it has
+  // |ignoreDataPointsNumber| data points or less in |matchedDataPoints|.
+  RemoveKeysSmallValue(matchedDataPoints, distribution, ignoreDataPointsNumber);
+
+  // Calculating how much percent data points in each mwm of |distribution|.
+  auto const distributionFraction = GetMwmToDataPointFraction(distribution);
+
+  CHECK(AreKeysEqual(distribution, matchedDataPoints),
+        ("Mwms in |checkedMap| and in |additionalMap| should have the same set of keys."));
+  // The distribution defined in |distribution| and the distribution after matching
+  // may be different. (They are most likely different.) So to keep the distribution
+  // defined in |distribution| after matching only a part of matched data points
+  // should be used.
+  // Returning mapping from mwm to balanced data point number with information about
+  // how many data points form |distributionCsvStream| should be used for every mwm.
+  return CalcsMatchedDataPointsToKeepDistribution(matchedDataPoints, distributionFraction);
+}
+
+void FilterTable(MwmToDataPoints const & balancedDataPointNumbers, vector<TableRow> & table)
+{
+  map<string, vector<size_t>> tableIdx;
+  for (size_t idx = 0; idx < table.size(); ++idx)
+  {
+    auto const & row = table[idx];
+    tableIdx[row[kMwmNameCsvColumn]].push_back(idx);
+  }
+
+  // Shuffling data point indexes and reducing the size of index vector for each mwm according to
+  // |balancedDataPointNumbers|.
+  random_device rd;
+  mt19937 g(rd());
+  for (auto & kv : tableIdx)
+  {
+    auto const & mwmName = kv.first;
+    auto & mwmRecordsIndices = kv.second;
+    auto it = balancedDataPointNumbers.find(mwmName);
+    if (it == balancedDataPointNumbers.end())
+    {
+      mwmRecordsIndices.resize(0);  // No indices.
+      continue;
+    }
+
+    auto const balancedDataPointNumber = it->second;
+    CHECK_LESS_OR_EQUAL(balancedDataPointNumber, mwmRecordsIndices.size(), (mwmName));
+    shuffle(mwmRecordsIndices.begin(), mwmRecordsIndices.end(), g);
+    mwmRecordsIndices.resize(balancedDataPointNumber);
+  }
+
+  // Refilling |table| with part of its items to corresponds |balancedDataPointNumbers| distribution.
+  vector<TableRow> balancedTable;
+  for (auto const & kv : tableIdx)
+    for (auto const idx : kv.second)
+      balancedTable.emplace_back(std::move(table[idx]));
+  table = std::move(balancedTable);
+}
+
+void BalanceDataPoints(basic_istream<char> & tableCsvStream, basic_istream<char> & distributionCsvStream,
+                       uint64_t ignoreDataPointsNumber, vector<TableRow> & balancedTable)
+{
+  LOG(LINFO, ("Balancing data points..."));
+  // Filling a map mwm to DataPoints number according to distribution csv file.
+  MwmToDataPoints distribution;
+  MappingFromCsv(distributionCsvStream, distribution);
+  uint64_t const totalDistributionDataPointsNumber = ValueSum(distribution);
+
+  balancedTable.clear();
+  MwmToDataPoints matchedDataPoints;
+  FillTable(tableCsvStream, matchedDataPoints, balancedTable);
+  uint64_t const totalMatchedDataPointsNumber = ValueSum(matchedDataPoints);
+  LOG(LINFO, ("Total matched data points number:", totalMatchedDataPointsNumber));
+
+  if (matchedDataPoints.empty())
+  {
+    CHECK(balancedTable.empty(), ());
+    return;
+  }
+
+  // Removing every mwm from |distribution| if there is no such mwm in |matchedDataPoints|.
+  for (auto it = distribution.begin(); it != distribution.end();)
+    if (matchedDataPoints.count(it->first) == 0)
+      it = distribution.erase(it);
+    else
+      ++it;
+
+  CHECK(AreKeysEqual(distribution, matchedDataPoints),
+        ("Mwms in |distribution| and in |matchedDataPoints| should have the same set of keys.", distribution,
+         matchedDataPoints));
+
+  // Calculating how many points should have every mwm to keep the |distribution|.
+  MwmToDataPoints const balancedDataPointNumber =
+      BalancedDataPointNumber(std::move(matchedDataPoints), std::move(distribution), ignoreDataPointsNumber);
+  uint64_t const totalBalancedDataPointsNumber = ValueSum(balancedDataPointNumber);
+  LOG(LINFO, ("Total balanced data points number:", totalBalancedDataPointsNumber));
+
+  // |balancedTable| is filled now with all the items from |tableCsvStream|.
+  // Removing some items form |tableCsvStream| (if it's necessary) to correspond to
+  // the distribution.
+  FilterTable(balancedDataPointNumber, balancedTable);
+  LOG(LINFO, ("Data points have balanced. Total distribution data points number:", totalDistributionDataPointsNumber,
+              "Total matched data points number:", totalMatchedDataPointsNumber,
+              "Total balanced data points number:", totalBalancedDataPointsNumber));
+}
+
+void CmdBalanceCsv(string const & csvPath, string const & distributionPath, uint64_t ignoreDataPointsNumber)
+{
+  LOG(LINFO, ("Balancing csv file", csvPath, "with distribution set in", distributionPath, ". If an mwm has",
+              ignoreDataPointsNumber, "data points or less it will not be considered."));
+  ifstream table(csvPath);
+  CHECK(table.is_open(), ("Cannot open", csvPath));
+  ifstream distribution(distributionPath);
+  CHECK(distribution.is_open(), ("Cannot open", distributionPath));
+  vector<TableRow> balancedTable;
+
+  BalanceDataPoints(table, distribution, ignoreDataPointsNumber, balancedTable);
+
+  // Calculating statistics.
+  storage::Storage storage;
+  storage.RegisterAllLocalMaps();
+  Stats stats;
+  for (auto const & record : balancedTable)
+  {
+    CHECK_EQUAL(record.size(), kTableColumns, (record));
+    auto const & mwmName = record[kMwmNameCsvColumn];
+    // Note. One record in csv means one data point.
+    stats.AddDataPoints(mwmName, storage, 1 /* data points number */);
+  }
+  stats.Log();
+
+  WriteCsvTableHeader(cout);
+  for (auto const & record : balancedTable)
+  {
+    for (size_t i = 0; i < kTableColumns; ++i)
+    {
+      auto const & field = record[i];
+      cout << field;
+      if (i != kTableColumns - 1)
+        cout << ",";
+    }
+    cout << '\n';
+  }
+  LOG(LINFO, ("Balanced."));
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_balance_csv.hpp

@@ -0,0 +1,91 @@
+#pragma once
+
+#include "track_analyzing/track_analyzer/utils.hpp"
+
+#include "base/logging.hpp"
+
+#include <cstdint>
+#include <map>
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace track_analyzing
+{
+// Mwm to data points number in the mwm mapping.
+using MwmToDataPoints = Stats::NameToCountMapping;
+// Csv table row.
+using TableRow = std::vector<std::string>;
+// Mwm to data points percent in the mwm mapping. It's assumed that the sum of the values
+// should be equal to 100.0 percent.
+using MwmToDataPointFraction = std::map<std::string, double>;
+
+size_t constexpr kMwmNameCsvColumn = 1;
+
+template <typename MapCont>
+typename MapCont::mapped_type ValueSum(MapCont const & mapCont)
+{
+  typename MapCont::mapped_type valueSum = typename MapCont::mapped_type();
+  for (auto const & kv : mapCont)
+    valueSum += kv.second;
+  return valueSum;
+}
+
+/// \returns true if |map1| and |map2| have the same key and false otherwise.
+template <typename Map1, typename Map2>
+bool AreKeysEqual(Map1 const & map1, Map2 const & map2)
+{
+  if (map1.size() != map2.size())
+  {
+    LOG(LINFO, ("AreKeysEqual() returns false. map1.size():", map1.size(), "map2.size()", map2.size()));
+    return false;
+  }
+
+  return std::equal(map1.begin(), map1.end(), map2.begin(), [](auto const & kv1, auto const & kv2)
+  {
+    if (kv1.first == kv2.first)
+      return true;
+
+    LOG(LINFO, ("Keys:", kv1.first, "and", kv2.first, "are not equal."));
+    return false;
+  });
+}
+
+/// \brief Fills a map |mwmToDataPoints| mwm to DataPoints number according to |tableCsvStream| and
+/// fills |matchedDataPoints| with all the content of |tableCsvStream|.
+void FillTable(std::basic_istream<char> & tableCsvStream, MwmToDataPoints & matchedDataPoints,
+               std::vector<TableRow> & table);
+
+/// \brief Removing every mwm from |checkedMap| and |additionalMap| if it has
+/// |ignoreDataPointsNumber| data points or less in |checkedMap|.
+void RemoveKeysSmallValue(MwmToDataPoints & checkedMap, MwmToDataPoints & additionalMap,
+                          uint64_t ignoreDataPointNumber);
+
+/// \returns mapping from mwm to fraction of data points contained in the mwm.
+MwmToDataPointFraction GetMwmToDataPointFraction(MwmToDataPoints const & numberMapping);
+
+/// \returns mapping mwm to matched data points fulfilling two conditions:
+/// *  number of data points in the returned mapping is less than or equal to
+///    number of data points in |matchedDataPoints| for every mwm
+/// * distribution defined by |distributionFraction| is kept
+MwmToDataPoints CalcsMatchedDataPointsToKeepDistribution(MwmToDataPoints const & matchedDataPoints,
+                                                         MwmToDataPointFraction const & distributionFractions);
+
+/// \returns mapping with number of matched data points for every mwm to correspond to |distribution|.
+MwmToDataPoints BalancedDataPointNumber(MwmToDataPoints && matchedDataPoints, MwmToDataPoints && distribution,
+                                        uint64_t ignoreDataPointsNumber);
+
+/// \breif Leaves in |table| only number of items according to |balancedDataPointNumbers|.
+/// \note |table| may have a significant size. It may be several tens millions records.
+void FilterTable(MwmToDataPoints const & balancedDataPointNumbers, std::vector<TableRow> & table);
+
+/// \brief Fills |balancedTable| with TableRow(s) according to the distribution set in
+/// |distributionCsvStream|.
+void BalanceDataPoints(std::basic_istream<char> & tableCsvStream, std::basic_istream<char> & distributionCsvStream,
+                       uint64_t ignoreDataPointsNumber, std::vector<TableRow> & balancedTable);
+
+/// \brief Removes some data point for every mwm from |csvPath| to correspond distribution
+/// set in |distributionPath|. If an mwm in |csvPath| contains data points less of equal to
+/// |ignoreDataPointsNumber| than the mwm will not be taken into acount while balancing.
+void CmdBalanceCsv(std::string const & csvPath, std::string const & distributionPath, uint64_t ignoreDataPointsNumber);
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_cpp_track.cpp

@@ -0,0 +1,27 @@
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/utils.hpp"
+
+#include "storage/routing_helpers.hpp"
+#include "storage/storage.hpp"
+
+using namespace routing;
+using namespace std;
+
+namespace track_analyzing
+{
+void CmdCppTrack(string const & trackFile, string const & mwmName, string const & user, size_t trackIdx)
+{
+  storage::Storage storage;
+  auto const numMwmIds = CreateNumMwmIds(storage);
+  MwmToMatchedTracks mwmToMatchedTracks;
+  ReadTracks(numMwmIds, trackFile, mwmToMatchedTracks);
+
+  MatchedTrack const & track = GetMatchedTrack(mwmToMatchedTracks, *numMwmIds, mwmName, user, trackIdx);
+
+  auto const backupPrecision = cout.precision();
+  cout.precision(8);
+  for (MatchedTrackPoint const & point : track)
+    cout << "  {" << point.GetDataPoint().m_latLon.m_lat << ", " << point.GetDataPoint().m_latLon.m_lon << "}," << endl;
+  cout.precision(backupPrecision);
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_gpx.cpp

@@ -0,0 +1,63 @@
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/track_analyzer/utils.hpp"
+
+#include "routing_common/num_mwm_id.hpp"
+
+#include "storage/routing_helpers.hpp"
+#include "storage/storage.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+
+#include <cstdlib>
+#include <fstream>
+#include <memory>
+#include <string>
+
+namespace track_analyzing
+{
+using namespace routing;
+using namespace std;
+
+void CmdGPX(string const & logFile, string const & outputDirName, string const & userID)
+{
+  if (outputDirName.empty())
+  {
+    LOG(LERROR, ("Converting to GPX error: the path to the output files is empty"));
+    return;
+  }
+
+  storage::Storage storage;
+  shared_ptr<NumMwmIds> numMwmIds = CreateNumMwmIds(storage);
+  MwmToTracks mwmToTracks;
+  ParseTracks(logFile, numMwmIds, mwmToTracks);
+  for (auto const & kv : mwmToTracks)
+  {
+    auto const & mwmName = numMwmIds->GetFile(kv.first).GetName();
+    size_t i = 0;
+    for (auto const & track : kv.second)
+    {
+      if (!userID.empty() && track.first != userID)
+        continue;
+
+      auto const path = base::JoinPath(outputDirName, mwmName + to_string(i) + ".gpx");
+      ofstream ofs(path, ofstream::out);
+      ofs << "<gpx>\n";
+      ofs << "<metadata>\n";
+      ofs << "<desc>" << track.first << "</desc>\n";
+      ofs << "</metadata>\n";
+      for (auto const & point : track.second)
+      {
+        ofs << "<wpt lat=\"" << point.m_latLon.m_lat << "\" lon=\"" << point.m_latLon.m_lon << "\">"
+            << "</wpt>\n";
+      }
+
+      ofs << "</gpx>\n";
+      if (!userID.empty())
+        break;
+
+      ++i;
+    }
+  }
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_match.cpp

@@ -0,0 +1,225 @@
+#include "track_analyzing/track_analyzer/utils.hpp"
+
+#include "track_analyzing/serialization.hpp"
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/track_analyzer/utils.hpp"
+#include "track_analyzing/track_matcher.hpp"
+#include "track_analyzing/utils.hpp"
+
+#include "routing_common/num_mwm_id.hpp"
+
+#include "storage/routing_helpers.hpp"
+#include "storage/storage.hpp"
+
+#include "coding/file_reader.hpp"
+#include "coding/file_writer.hpp"
+#include "coding/zlib.hpp"
+
+#include "platform/platform.hpp"
+
+#include "base/assert.hpp"
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+#include "base/timer.hpp"
+
+#include <algorithm>
+#include <exception>
+#include <memory>
+#include <string>
+#include <thread>
+
+using namespace routing;
+using namespace std;
+using namespace storage;
+using namespace track_analyzing;
+
+namespace
+{
+using Iter = typename vector<string>::iterator;
+
+void MatchTracks(MwmToTracks const & mwmToTracks, storage::Storage const & storage, NumMwmIds const & numMwmIds,
+                 MwmToMatchedTracks & mwmToMatchedTracks)
+{
+  base::Timer timer;
+
+  uint64_t tracksCount = 0;
+  uint64_t pointsCount = 0;
+  uint64_t nonMatchedPointsCount = 0;
+
+  auto processMwm = [&](string const & mwmName, UserToTrack const & userToTrack)
+  {
+    auto const countryFile = platform::CountryFile(mwmName);
+    auto const mwmId = numMwmIds.GetId(countryFile);
+    TrackMatcher matcher(storage, mwmId, countryFile);
+
+    auto & userToMatchedTracks = mwmToMatchedTracks[mwmId];
+
+    for (auto const & it : userToTrack)
+    {
+      string const & user = it.first;
+      auto & matchedTracks = userToMatchedTracks[user];
+      try
+      {
+        matcher.MatchTrack(it.second, matchedTracks);
+      }
+      catch (RootException const & e)
+      {
+        LOG(LERROR, ("Can't match track for mwm:", mwmName, ", user:", user));
+        LOG(LERROR, ("  ", e.what()));
+      }
+
+      if (matchedTracks.empty())
+        userToMatchedTracks.erase(user);
+    }
+
+    if (userToMatchedTracks.empty())
+      mwmToMatchedTracks.erase(mwmId);
+
+    tracksCount += matcher.GetTracksCount();
+    pointsCount += matcher.GetPointsCount();
+    nonMatchedPointsCount += matcher.GetNonMatchedPointsCount();
+
+    LOG(LINFO,
+        (numMwmIds.GetFile(mwmId).GetName(), ", users:", userToTrack.size(), ", tracks:", matcher.GetTracksCount(),
+         ", points:", matcher.GetPointsCount(), ", non matched points:", matcher.GetNonMatchedPointsCount()));
+  };
+
+  ForTracksSortedByMwmName(mwmToTracks, numMwmIds, processMwm);
+
+  LOG(LINFO, ("Matching finished, elapsed:", timer.ElapsedSeconds(), "seconds, tracks:", tracksCount,
+              ", points:", pointsCount, ", non matched points:", nonMatchedPointsCount));
+}
+}  // namespace
+
+namespace track_analyzing
+{
+void CmdMatch(string const & logFile, string const & trackFile, shared_ptr<NumMwmIds> const & numMwmIds,
+              Storage const & storage, Stats & stats)
+{
+  MwmToTracks mwmToTracks;
+  ParseTracks(logFile, numMwmIds, mwmToTracks);
+  stats.AddTracksStats(mwmToTracks, *numMwmIds, storage);
+
+  MwmToMatchedTracks mwmToMatchedTracks;
+  MatchTracks(mwmToTracks, storage, *numMwmIds, mwmToMatchedTracks);
+
+  FileWriter writer(trackFile, FileWriter::OP_WRITE_TRUNCATE);
+  MwmToMatchedTracksSerializer serializer(numMwmIds);
+  serializer.Serialize(mwmToMatchedTracks, writer);
+  LOG(LINFO, ("Matched tracks were saved to", trackFile));
+}
+
+void CmdMatch(string const & logFile, string const & trackFile, string const & inputDistribution)
+{
+  LOG(LINFO, ("Matching", logFile));
+  Storage storage;
+  storage.RegisterAllLocalMaps();
+  shared_ptr<NumMwmIds> numMwmIds = CreateNumMwmIds(storage);
+
+  Stats stats;
+  CmdMatch(logFile, trackFile, numMwmIds, storage, stats);
+  stats.SaveMwmDistributionToCsv(inputDistribution);
+  stats.Log();
+}
+
+void UnzipAndMatch(Iter begin, Iter end, string const & trackExt, Stats & stats)
+{
+  Storage storage;
+  storage.RegisterAllLocalMaps();
+  shared_ptr<NumMwmIds> numMwmIds = CreateNumMwmIds(storage);
+  for (auto it = begin; it != end; ++it)
+  {
+    auto & file = *it;
+    string data;
+    try
+    {
+      auto const r = GetPlatform().GetReader(file);
+      r->ReadAsString(data);
+    }
+    catch (FileReader::ReadException const & e)
+    {
+      LOG(LWARNING, (e.what()));
+      continue;
+    }
+
+    using Inflate = coding::ZLib::Inflate;
+    Inflate inflate(Inflate::Format::GZip);
+    string track;
+    inflate(data.data(), data.size(), back_inserter(track));
+    base::GetNameWithoutExt(file);
+    try
+    {
+      FileWriter w(file);
+      w.Write(track.data(), track.size());
+    }
+    catch (std::exception const & e)
+    {
+      LOG(LWARNING, (e.what()));
+      continue;
+    }
+
+    CmdMatch(file, file + trackExt, numMwmIds, storage, stats);
+    FileWriter::DeleteFileX(file);
+  }
+}
+
+void CmdMatchDir(string const & logDir, string const & trackExt, string const & inputDistribution)
+{
+  LOG(LINFO, ("Matching dir:", logDir, ". Input distribution will be saved to:", inputDistribution));
+  Platform::EFileType fileType = Platform::EFileType::Unknown;
+  Platform::EError const result = Platform::GetFileType(logDir, fileType);
+
+  if (result == Platform::ERR_FILE_DOES_NOT_EXIST)
+  {
+    LOG(LINFO, ("Directory doesn't exist", logDir));
+    return;
+  }
+
+  if (result != Platform::ERR_OK)
+  {
+    LOG(LINFO, ("Can't get file type for", logDir));
+    return;
+  }
+
+  if (fileType != Platform::EFileType::Directory)
+  {
+    LOG(LINFO, (logDir, "is not a directory."));
+    return;
+  }
+
+  Platform::FilesList filesList;
+  Platform::GetFilesRecursively(logDir, filesList);
+  if (filesList.empty())
+  {
+    LOG(LINFO, (logDir, "is empty."));
+    return;
+  }
+
+  auto const size = filesList.size();
+  auto const hardwareConcurrency = static_cast<size_t>(thread::hardware_concurrency());
+  CHECK_GREATER(hardwareConcurrency, 0, ("No available threads."));
+  LOG(LINFO, ("Number of available threads =", hardwareConcurrency));
+  auto const threadsCount = min(size, hardwareConcurrency);
+  auto const blockSize = size / threadsCount;
+  vector<thread> threads(threadsCount - 1);
+  vector<Stats> stats(threadsCount);
+  auto begin = filesList.begin();
+  for (size_t i = 0; i < threadsCount - 1; ++i)
+  {
+    auto end = begin + blockSize;
+    threads[i] = thread(UnzipAndMatch, begin, end, trackExt, ref(stats[i]));
+    begin = end;
+  }
+
+  UnzipAndMatch(begin, filesList.end(), trackExt, ref(stats[threadsCount - 1]));
+  for (auto & t : threads)
+    t.join();
+
+  Stats statSum;
+  for (auto const & s : stats)
+    statSum.Add(s);
+
+  statSum.SaveMwmDistributionToCsv(inputDistribution);
+  statSum.Log();
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_table.cpp

@@ -0,0 +1,450 @@
+#include "track_analyzing/track_analyzer/crossroad_checker.hpp"
+
+#include "track_analyzing/track_analyzer/utils.hpp"
+
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/utils.hpp"
+
+#include "routing/city_roads.hpp"
+#include "routing/data_source.hpp"
+#include "routing/geometry.hpp"
+#include "routing/index_graph_loader.hpp"
+#include "routing/maxspeeds.hpp"
+
+#include "routing_common/car_model.hpp"
+#include "routing_common/maxspeed_conversion.hpp"
+#include "routing_common/vehicle_model.hpp"
+
+#include "traffic/speed_groups.hpp"
+
+#include "indexer/classificator.hpp"
+#include "indexer/feature_data.hpp"
+#include "indexer/features_vector.hpp"
+
+#include "storage/routing_helpers.hpp"
+#include "storage/storage.hpp"
+
+#include "coding/file_reader.hpp"
+
+#include "geometry/latlon.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+#include "base/stl_helpers.hpp"
+#include "base/sunrise_sunset.hpp"
+
+#include <algorithm>
+#include <iostream>
+#include <iterator>
+#include <limits>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <vector>
+
+#include "defines.hpp"
+
+namespace track_analyzing
+{
+using namespace routing;
+using namespace std;
+
+namespace
+{
+MaxspeedType constexpr kMaxspeedTopBound = 200;
+auto constexpr kValidTrafficValue = traffic::SpeedGroup::G5;
+
+string TypeToString(uint32_t type)
+{
+  if (type == 0)
+    return "unknown-type";
+
+  return classif().GetReadableObjectName(type);
+}
+
+bool DayTimeToBool(DayTimeType type)
+{
+  switch (type)
+  {
+  case DayTimeType::Day:
+  case DayTimeType::PolarDay: return true;
+  case DayTimeType::Night:
+  case DayTimeType::PolarNight: return false;
+  }
+
+  UNREACHABLE();
+}
+
+class CarModelTypes final
+{
+public:
+  CarModelTypes()
+  {
+    auto const & cl = classif();
+
+    for (auto const & speed : CarModel::GetOptions())
+      m_hwtags.push_back(cl.GetTypeForIndex(static_cast<uint32_t>(speed.m_type)));
+
+    for (auto const & surface : CarModel::GetSurfaces())
+      m_surfaceTags.push_back(cl.GetTypeByPath(surface.m_type));
+  }
+
+  struct Type
+  {
+    bool operator<(Type const & rhs) const
+    {
+      return tie(m_hwType, m_surfaceType) < tie(rhs.m_hwType, rhs.m_surfaceType);
+    }
+
+    bool operator==(Type const & rhs) const { return tie(m_hwType, m_surfaceType) == tie(rhs.m_hwType, m_surfaceType); }
+
+    bool operator!=(Type const & rhs) const { return !(*this == rhs); }
+
+    string GetSummary() const
+    {
+      ostringstream out;
+      out << TypeToString(m_hwType) << "," << TypeToString(m_surfaceType);
+      return out.str();
+    }
+
+    uint32_t m_hwType = 0;
+    uint32_t m_surfaceType = 0;
+  };
+
+  Type GetType(feature::TypesHolder const & types) const
+  {
+    Type ret;
+    for (uint32_t type : m_hwtags)
+    {
+      if (types.Has(type))
+      {
+        ret.m_hwType = type;
+        break;
+      }
+    }
+
+    for (uint32_t type : m_surfaceTags)
+    {
+      if (types.Has(type))
+      {
+        ret.m_surfaceType = type;
+        break;
+      }
+    }
+
+    return ret;
+  }
+
+private:
+  vector<uint32_t> m_hwtags;
+  vector<uint32_t> m_surfaceTags;
+};
+
+struct RoadInfo
+{
+  bool operator==(RoadInfo const & rhs) const
+  {
+    return tie(m_type, m_maxspeedKMpH, m_isCityRoad, m_isOneWay) ==
+           tie(rhs.m_type, rhs.m_maxspeedKMpH, rhs.m_isCityRoad, rhs.m_isOneWay);
+  }
+
+  bool operator!=(RoadInfo const & rhs) const { return !(*this == rhs); }
+
+  bool operator<(RoadInfo const & rhs) const
+  {
+    return tie(m_type, m_maxspeedKMpH, m_isCityRoad, m_isOneWay) <
+           tie(rhs.m_type, rhs.m_maxspeedKMpH, rhs.m_isCityRoad, rhs.m_isOneWay);
+  }
+
+  string GetSummary() const
+  {
+    ostringstream out;
+    out << TypeToString(m_type.m_hwType) << "," << TypeToString(m_type.m_surfaceType) << "," << m_maxspeedKMpH << ","
+        << m_isCityRoad << "," << m_isOneWay;
+
+    return out.str();
+  }
+
+  CarModelTypes::Type m_type;
+  MaxspeedType m_maxspeedKMpH = kInvalidSpeed;
+  bool m_isCityRoad = false;
+  bool m_isOneWay = false;
+};
+
+class MoveType final
+{
+public:
+  MoveType() = default;
+
+  MoveType(RoadInfo const & roadType, traffic::SpeedGroup speedGroup, DataPoint const & dataPoint)
+    : m_roadInfo(roadType)
+    , m_speedGroup(speedGroup)
+    , m_latLon(dataPoint.m_latLon)
+  {
+    m_isDayTime = DayTimeToBool(GetDayTime(dataPoint.m_timestamp, m_latLon.m_lat, m_latLon.m_lon));
+  }
+
+  bool operator==(MoveType const & rhs) const
+  {
+    return tie(m_roadInfo, m_speedGroup) == tie(rhs.m_roadInfo, rhs.m_speedGroup);
+  }
+
+  bool operator<(MoveType const & rhs) const
+  {
+    auto const lhsGroup = base::Underlying(m_speedGroup);
+    auto const rhsGroup = base::Underlying(rhs.m_speedGroup);
+    return tie(m_roadInfo, lhsGroup) < tie(rhs.m_roadInfo, rhsGroup);
+  }
+
+  bool IsValid() const
+  {
+    // In order to collect cleaner data we don't use speed group lower than G5.
+    return m_roadInfo.m_type.m_hwType != 0 && m_roadInfo.m_type.m_surfaceType != 0 &&
+           m_speedGroup == kValidTrafficValue;
+  }
+
+  string GetSummary() const
+  {
+    ostringstream out;
+    out << m_roadInfo.GetSummary() << "," << m_isDayTime << "," << m_latLon.m_lat << " " << m_latLon.m_lon;
+
+    return out.str();
+  }
+
+private:
+  RoadInfo m_roadInfo;
+  traffic::SpeedGroup m_speedGroup = traffic::SpeedGroup::Unknown;
+  ms::LatLon m_latLon;
+  bool m_isDayTime = false;
+};
+
+class SpeedInfo final
+{
+public:
+  void Add(double distance, uint64_t time, IsCrossroadChecker::CrossroadInfo const & crossroads,
+           uint32_t dataPointsNumber)
+  {
+    m_totalDistance += distance;
+    m_totalTime += time;
+    IsCrossroadChecker::MergeCrossroads(crossroads, m_crossroads);
+    m_dataPointsNumber += dataPointsNumber;
+  }
+
+  string GetSummary() const
+  {
+    ostringstream out;
+    out << m_totalDistance << "," << m_totalTime << "," << CalcSpeedKMpH(m_totalDistance, m_totalTime) << ",";
+
+    for (size_t i = 0; i < m_crossroads.size(); ++i)
+    {
+      out << m_crossroads[i];
+      if (i != m_crossroads.size() - 1)
+        out << ",";
+    }
+
+    return out.str();
+  }
+
+  uint32_t GetDataPointsNumber() const { return m_dataPointsNumber; }
+
+private:
+  double m_totalDistance = 0.0;
+  uint64_t m_totalTime = 0;
+  IsCrossroadChecker::CrossroadInfo m_crossroads{};
+  uint32_t m_dataPointsNumber = 0;
+};
+
+class MoveTypeAggregator final
+{
+public:
+  void Add(MoveType && moveType, IsCrossroadChecker::CrossroadInfo const & crossroads,
+           MatchedTrack::const_iterator begin, MatchedTrack::const_iterator end, Geometry & geometry)
+  {
+    if (begin + 1 >= end)
+      return;
+
+    auto const & beginDataPoint = begin->GetDataPoint();
+    auto const & endDataPoint = (end - 1)->GetDataPoint();
+    uint64_t const time = endDataPoint.m_timestamp - beginDataPoint.m_timestamp;
+
+    if (time == 0)
+    {
+      LOG(LWARNING, ("Track with the same time at the beginning and at the end. Beginning:", beginDataPoint.m_latLon,
+                     " End:", endDataPoint.m_latLon, " Timestamp:", beginDataPoint.m_timestamp,
+                     " Segment:", begin->GetSegment()));
+      return;
+    }
+
+    double const length = CalcSubtrackLength(begin, end, geometry);
+    m_moveInfos[moveType].Add(length, time, crossroads, static_cast<uint32_t>(distance(begin, end)));
+  }
+
+  string GetSummary(string const & user, string const & mwmName, string const & countryName, Stats & stats) const
+  {
+    ostringstream out;
+    for (auto const & it : m_moveInfos)
+    {
+      if (!it.first.IsValid())
+        continue;
+
+      out << user << "," << mwmName << "," << it.first.GetSummary() << "," << it.second.GetSummary() << '\n';
+
+      stats.AddDataPoints(mwmName, countryName, it.second.GetDataPointsNumber());
+    }
+
+    return out.str();
+  }
+
+private:
+  map<MoveType, SpeedInfo> m_moveInfos;
+};
+
+class MatchedTrackPointToMoveType final
+{
+public:
+  MatchedTrackPointToMoveType(FilesContainerR const & container, VehicleModelInterface & vehicleModel)
+    : m_featuresVector(container)
+    , m_vehicleModel(vehicleModel)
+  {
+    if (container.IsExist(CITY_ROADS_FILE_TAG))
+      m_cityRoads.Load(container.GetReader(CITY_ROADS_FILE_TAG));
+
+    if (container.IsExist(MAXSPEEDS_FILE_TAG))
+      m_maxspeeds.Load(container.GetReader(MAXSPEEDS_FILE_TAG));
+  }
+
+  MoveType GetMoveType(MatchedTrackPoint const & point)
+  {
+    auto const & dataPoint = point.GetDataPoint();
+    return MoveType(GetRoadInfo(point.GetSegment()), static_cast<traffic::SpeedGroup>(dataPoint.m_traffic), dataPoint);
+  }
+
+private:
+  RoadInfo GetRoadInfo(Segment const & segment)
+  {
+    auto const featureId = segment.GetFeatureId();
+    if (featureId == m_prevFeatureId)
+      return m_prevRoadInfo;
+
+    auto feature = m_featuresVector.GetVector().GetByIndex(featureId);
+    CHECK(feature, ());
+
+    auto const maxspeed = m_maxspeeds.GetMaxspeed(featureId);
+    auto const maxspeedValueKMpH =
+        maxspeed.IsValid() ? min(maxspeed.GetSpeedKmPH(segment.IsForward()), kMaxspeedTopBound) : kInvalidSpeed;
+
+    m_prevFeatureId = featureId;
+
+    feature::TypesHolder const types(*feature);
+    m_prevRoadInfo = {m_carModelTypes.GetType(types), maxspeedValueKMpH, m_cityRoads.IsCityRoad(featureId),
+                      m_vehicleModel.IsOneWay(types)};
+    return m_prevRoadInfo;
+  }
+
+  FeaturesVectorTest m_featuresVector;
+  VehicleModelInterface & m_vehicleModel;
+  CarModelTypes const m_carModelTypes;
+  CityRoads m_cityRoads;
+  Maxspeeds m_maxspeeds;
+  uint32_t m_prevFeatureId = numeric_limits<uint32_t>::max();
+  RoadInfo m_prevRoadInfo;
+};
+}  // namespace
+
+void CmdTagsTable(string const & filepath, string const & trackExtension, StringFilter mwmFilter,
+                  StringFilter userFilter)
+{
+  WriteCsvTableHeader(cout);
+
+  storage::Storage storage;
+  storage.RegisterAllLocalMaps();
+  FrozenDataSource dataSource;
+  auto numMwmIds = CreateNumMwmIds(storage);
+
+  Stats stats;
+  auto processMwm = [&](string const & mwmName, UserToMatchedTracks const & userToMatchedTracks)
+  {
+    if (mwmFilter(mwmName))
+      return;
+
+    auto const countryName = storage.GetTopmostParentFor(mwmName);
+    auto const carModelFactory = make_shared<CarModelFactory>(VehicleModelFactory::CountryParentNameGetterFn{});
+    shared_ptr<VehicleModelInterface> vehicleModel = carModelFactory->GetVehicleModelForCountry(mwmName);
+    string const mwmFile = GetCurrentVersionMwmFile(storage, mwmName);
+    MatchedTrackPointToMoveType pointToMoveType(FilesContainerR(make_unique<FileReader>(mwmFile)), *vehicleModel);
+    Geometry geometry(GeometryLoader::CreateFromFile(mwmFile, vehicleModel));
+    auto const vehicleType = VehicleType::Car;
+    auto const edgeEstimator =
+        EdgeEstimator::Create(vehicleType, *vehicleModel, nullptr /* trafficStash */, &dataSource, numMwmIds);
+
+    MwmDataSource routingSource(dataSource, numMwmIds);
+    auto indexGraphLoader =
+        IndexGraphLoader::Create(vehicleType, false /* loadAltitudes */, carModelFactory, edgeEstimator, routingSource);
+
+    platform::CountryFile const countryFile(mwmName);
+    auto localCountryFile = storage.GetLatestLocalFile(countryFile);
+    CHECK(localCountryFile, ("Can't find latest country file for", countryFile.GetName()));
+    if (!dataSource.IsLoaded(countryFile))
+    {
+      auto registerResult = dataSource.Register(*localCountryFile);
+      CHECK_EQUAL(registerResult.second, MwmSet::RegResult::Success, ("Can't register mwm", countryFile.GetName()));
+    }
+
+    auto const mwmId = numMwmIds->GetId(countryFile);
+    IsCrossroadChecker checker(indexGraphLoader->GetIndexGraph(mwmId), geometry);
+
+    for (auto const & kv : userToMatchedTracks)
+    {
+      string const & user = kv.first;
+      if (userFilter(user))
+        continue;
+
+      for (auto const & track : kv.second)
+      {
+        if (track.size() <= 1)
+          continue;
+
+        MoveTypeAggregator aggregator;
+        IsCrossroadChecker::CrossroadInfo info = {};
+        for (auto subtrackBegin = track.begin(); subtrackBegin != track.end();)
+        {
+          auto moveType = pointToMoveType.GetMoveType(*subtrackBegin);
+          auto prev = subtrackBegin;
+          auto end = subtrackBegin + 1;
+          // Splitting track with points where MoveType is changed.
+          while (end != track.end() && pointToMoveType.GetMoveType(*end) == moveType)
+          {
+            IsCrossroadChecker::MergeCrossroads(checker(prev->GetSegment(), end->GetSegment()), info);
+            prev = end;
+            ++end;
+          }
+
+          // If it's not the end of the track than it could be a crossroad.
+          if (end != track.end())
+            IsCrossroadChecker::MergeCrossroads(checker(prev->GetSegment(), end->GetSegment()), info);
+
+          aggregator.Add(std::move(moveType), info, subtrackBegin, end, geometry);
+          subtrackBegin = end;
+          info.fill(0);
+        }
+
+        auto const summary = aggregator.GetSummary(user, mwmName, countryName, stats);
+        if (!summary.empty())
+          cout << summary;
+      }
+    }
+  };
+
+  auto processTrack = [&](string const & filename, MwmToMatchedTracks const & mwmToMatchedTracks)
+  {
+    LOG(LINFO, ("Processing", filename));
+    ForTracksSortedByMwmName(mwmToMatchedTracks, *numMwmIds, processMwm);
+  };
+
+  ForEachTrackFile(filepath, trackExtension, numMwmIds, processTrack);
+
+  stats.Log();
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_track.cpp

@@ -0,0 +1,68 @@
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/utils.hpp"
+
+#include "routing/geometry.hpp"
+
+#include "routing_common/car_model.hpp"
+#include "routing_common/vehicle_model.hpp"
+
+#include "indexer/feature.hpp"
+
+#include "storage/routing_helpers.hpp"
+#include "storage/storage.hpp"
+
+#include "coding/file_reader.hpp"
+
+#include "geometry/mercator.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/timer.hpp"
+
+using namespace routing;
+using namespace std;
+
+namespace track_analyzing
+{
+void CmdTrack(string const & trackFile, string const & mwmName, string const & user, size_t trackIdx)
+{
+  storage::Storage storage;
+  auto const numMwmIds = CreateNumMwmIds(storage);
+  MwmToMatchedTracks mwmToMatchedTracks;
+  ReadTracks(numMwmIds, trackFile, mwmToMatchedTracks);
+
+  MatchedTrack const & track = GetMatchedTrack(mwmToMatchedTracks, *numMwmIds, mwmName, user, trackIdx);
+
+  string const mwmFile = GetCurrentVersionMwmFile(storage, mwmName);
+  shared_ptr<VehicleModelInterface> vehicleModel = CarModelFactory({}).GetVehicleModelForCountry(mwmName);
+  Geometry geometry(GeometryLoader::CreateFromFile(mwmFile, vehicleModel));
+
+  uint64_t const duration = track.back().GetDataPoint().m_timestamp - track.front().GetDataPoint().m_timestamp;
+  double const length = CalcTrackLength(track, geometry);
+  double const averageSpeed = CalcSpeedKMpH(length, duration);
+  LOG(LINFO, ("Mwm:", mwmName, ", user:", user, ", points:", track.size(), "duration:", duration, "length:", length,
+              ", speed:", averageSpeed, "km/h"));
+
+  for (size_t i = 0; i < track.size(); ++i)
+  {
+    MatchedTrackPoint const & point = track[i];
+
+    double speed = 0.0;
+    uint64_t elapsed = 0;
+    double distance = 0.0;
+    if (i > 0)
+    {
+      MatchedTrackPoint const & prevPoint = track[i - 1];
+      elapsed = point.GetDataPoint().m_timestamp - prevPoint.GetDataPoint().m_timestamp;
+      distance = mercator::DistanceOnEarth(mercator::FromLatLon(prevPoint.GetDataPoint().m_latLon),
+                                           mercator::FromLatLon(point.GetDataPoint().m_latLon));
+    }
+
+    if (elapsed != 0)
+      speed = CalcSpeedKMpH(distance, elapsed);
+
+    LOG(LINFO, (base::SecondsSinceEpochToString(point.GetDataPoint().m_timestamp), point.GetDataPoint().m_latLon,
+                point.GetSegment(), ", traffic:", point.GetDataPoint().m_traffic, ", distance:", distance,
+                ", elapsed:", elapsed, ", speed:", speed));
+  }
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_tracks.cpp

@@ -0,0 +1,229 @@
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/utils.hpp"
+
+#include "traffic/speed_groups.hpp"
+
+#include "routing_common/car_model.hpp"
+#include "routing_common/vehicle_model.hpp"
+
+#include "routing/edge_estimator.hpp"
+#include "routing/geometry.hpp"
+
+#include "storage/routing_helpers.hpp"
+#include "storage/storage.hpp"
+
+#include "platform/platform.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+#include "base/timer.hpp"
+
+#include <algorithm>
+#include <iostream>
+#include <sstream>
+
+namespace track_analyzing
+{
+using namespace routing;
+using namespace std;
+
+class TrackStats final
+{
+public:
+  void AddUsers(uint64_t numUsers) { m_numUsers += numUsers; }
+  void AddTracks(uint64_t numTracks) { m_numTracks += numTracks; }
+  void AddPoints(uint64_t numPoints) { m_numPoints += numPoints; }
+
+  void Add(TrackStats const & rhs)
+  {
+    m_numUsers += rhs.m_numUsers;
+    m_numTracks += rhs.m_numTracks;
+    m_numPoints += rhs.m_numPoints;
+  }
+
+  string GetSummary() const
+  {
+    ostringstream out;
+    out << "users: " << m_numUsers << ", tracks: " << m_numTracks << ", points: " << m_numPoints;
+    return out.str();
+  }
+
+  bool IsEmpty() const { return m_numPoints == 0; }
+
+private:
+  uint64_t m_numUsers = 0;
+  uint64_t m_numTracks = 0;
+  uint64_t m_numPoints = 0;
+};
+
+class ErrorStat final
+{
+public:
+  void Add(double value)
+  {
+    m_count += 1.0;
+    m_squares += value * value;
+    m_min = min(m_min, value);
+    m_max = max(m_max, value);
+  }
+
+  double GetStdDev() const
+  {
+    CHECK_GREATER(m_count, 1.0, ());
+    return std::sqrt(m_squares / (m_count - 1.0));
+  }
+
+  string GetStdDevString() const
+  {
+    if (m_count <= 1.0)
+      return "N/A";
+
+    return to_string(GetStdDev());
+  }
+
+  double GetMin() const { return m_min; }
+  double GetMax() const { return m_max; }
+
+private:
+  double m_count = 0.0;
+  double m_squares = 0.0;
+  double m_min = numeric_limits<double>::max();
+  double m_max = numeric_limits<double>::min();
+};
+
+bool TrackHasTrafficPoints(MatchedTrack const & track)
+{
+  for (MatchedTrackPoint const & point : track)
+  {
+    size_t const index = static_cast<size_t>(point.GetDataPoint().m_traffic);
+    CHECK_LESS(index, static_cast<size_t>(traffic::SpeedGroup::Count), ());
+    if (traffic::kSpeedGroupThresholdPercentage[index] != 100)
+      return true;
+  }
+
+  return false;
+}
+
+double EstimateDuration(MatchedTrack const & track, shared_ptr<EdgeEstimator> estimator, Geometry & geometry)
+{
+  double result = 0.0;
+  Segment segment;
+
+  for (MatchedTrackPoint const & point : track)
+  {
+    if (point.GetSegment() == segment)
+      continue;
+
+    segment = point.GetSegment();
+    result +=
+        estimator->CalcSegmentWeight(segment, geometry.GetRoad(segment.GetFeatureId()), EdgeEstimator::Purpose::ETA);
+  }
+
+  return result;
+}
+
+void CmdTracks(string const & filepath, string const & trackExtension, StringFilter mwmFilter, StringFilter userFilter,
+               TrackFilter const & filter, bool noTrackLogs, bool noMwmLogs, bool noWorldLogs)
+{
+  storage::Storage storage;
+  auto numMwmIds = CreateNumMwmIds(storage);
+
+  map<string, TrackStats> mwmToStats;
+  ErrorStat absoluteError;
+  ErrorStat relativeError;
+
+  auto processMwm = [&](string const & mwmName, UserToMatchedTracks const & userToMatchedTracks)
+  {
+    if (mwmFilter(mwmName))
+      return;
+
+    TrackStats & mwmStats = mwmToStats[mwmName];
+
+    shared_ptr<VehicleModelInterface> vehicleModel = CarModelFactory({}).GetVehicleModelForCountry(mwmName);
+
+    Geometry geometry(GeometryLoader::CreateFromFile(GetCurrentVersionMwmFile(storage, mwmName), vehicleModel));
+
+    shared_ptr<EdgeEstimator> estimator = EdgeEstimator::Create(
+        VehicleType::Car, *vehicleModel, nullptr /* trafficStash */, nullptr /* dataSource */, nullptr /* numMwmIds */);
+
+    for (auto const & it : userToMatchedTracks)
+    {
+      string const & user = it.first;
+      if (userFilter(user))
+        continue;
+
+      vector<MatchedTrack> const & tracks = it.second;
+      if (!noTrackLogs)
+        cout << mwmName << ", user: " << user << endl;
+
+      bool thereAreUnfilteredTracks = false;
+      for (MatchedTrack const & track : tracks)
+      {
+        DataPoint const & start = track.front().GetDataPoint();
+        DataPoint const & finish = track.back().GetDataPoint();
+
+        double const length = CalcTrackLength(track, geometry);
+        uint64_t const duration = finish.m_timestamp - start.m_timestamp;
+        double const speed = CalcSpeedKMpH(length, duration);
+        bool const hasTrafficPoints = TrackHasTrafficPoints(track);
+
+        if (!filter.Passes(duration, length, speed, hasTrafficPoints))
+          continue;
+
+        double const estimatedDuration = EstimateDuration(track, estimator, geometry);
+        double const timeError = estimatedDuration - static_cast<double>(duration);
+
+        if (!noTrackLogs)
+        {
+          cout << fixed << setprecision(1) << "  points: " << track.size() << ", length: " << length
+               << ", duration: " << duration << ", estimated duration: " << estimatedDuration << ", speed: " << speed
+               << ", traffic: " << hasTrafficPoints
+               << ", departure: " << base::SecondsSinceEpochToString(start.m_timestamp)
+               << ", arrival: " << base::SecondsSinceEpochToString(finish.m_timestamp)
+               << setprecision(numeric_limits<double>::max_digits10) << ", start: " << start.m_latLon.m_lat << ", "
+               << start.m_latLon.m_lon << ", finish: " << finish.m_latLon.m_lat << ", " << finish.m_latLon.m_lon
+               << endl;
+        }
+
+        mwmStats.AddTracks(1);
+        mwmStats.AddPoints(track.size());
+        absoluteError.Add(timeError);
+        relativeError.Add(timeError / static_cast<double>(duration));
+        thereAreUnfilteredTracks = true;
+      }
+
+      if (thereAreUnfilteredTracks)
+        mwmStats.AddUsers(1);
+    }
+  };
+
+  auto processFile = [&](string const & filename, MwmToMatchedTracks const & mwmToMatchedTracks)
+  {
+    LOG(LINFO, ("Processing", filename));
+    ForTracksSortedByMwmName(mwmToMatchedTracks, *numMwmIds, processMwm);
+  };
+
+  ForEachTrackFile(filepath, trackExtension, numMwmIds, processFile);
+
+  if (!noMwmLogs)
+  {
+    cout << endl;
+    for (auto const & it : mwmToStats)
+      if (!it.second.IsEmpty())
+        cout << it.first << ": " << it.second.GetSummary() << endl;
+  }
+
+  if (!noWorldLogs)
+  {
+    TrackStats worldStats;
+    for (auto const & it : mwmToStats)
+      worldStats.Add(it.second);
+
+    cout << endl << "World: " << worldStats.GetSummary() << endl;
+    cout << fixed << setprecision(1) << "Absolute error: deviation: " << absoluteError.GetStdDevString()
+         << ", min: " << absoluteError.GetMin() << ", max: " << absoluteError.GetMax() << endl;
+    cout << fixed << setprecision(3) << "Relative error: deviation: " << relativeError.GetStdDevString()
+         << ", min: " << relativeError.GetMin() << ", max: " << relativeError.GetMax() << endl;
+  }
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/cmd_unmatched_tracks.cpp

@@ -0,0 +1,41 @@
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/track_analyzer/utils.hpp"
+
+#include "routing_common/num_mwm_id.hpp"
+
+#include "storage/routing_helpers.hpp"
+#include "storage/storage.hpp"
+
+#include "base/logging.hpp"
+
+#include <fstream>
+#include <memory>
+#include <string>
+
+namespace track_analyzing
+{
+using namespace routing;
+using namespace std;
+
+void CmdUnmatchedTracks(string const & logFile, string const & trackFileCsv)
+{
+  LOG(LINFO, ("Saving unmatched tracks", logFile));
+  storage::Storage storage;
+  shared_ptr<NumMwmIds> numMwmIds = CreateNumMwmIds(storage);
+  MwmToTracks mwmToTracks;
+  ParseTracks(logFile, numMwmIds, mwmToTracks);
+
+  string const sep = ",";
+  ofstream ofs(trackFileCsv, std::ofstream::out);
+  for (auto const & kv : mwmToTracks)
+  {
+    for (auto const & idTrack : kv.second)
+    {
+      ofs << numMwmIds->GetFile(kv.first).GetName() << sep << idTrack.first;
+      for (auto const & pnt : idTrack.second)
+        ofs << sep << pnt.m_timestamp << sep << pnt.m_latLon.m_lat << sep << pnt.m_latLon.m_lon;
+      ofs << "\n";
+    }
+  }
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/crossroad_checker.cpp

@@ -0,0 +1,171 @@
+#include "track_analyzing/track_analyzer/crossroad_checker.hpp"
+
+#include "routing/joint.hpp"
+#include "routing/segment.hpp"
+
+#include "routing_common/vehicle_model.hpp"
+
+#include "base/assert.hpp"
+
+#include <algorithm>
+
+using namespace std;
+
+namespace
+{
+using namespace routing;
+
+bool IsHighwayLink(HighwayType type)
+{
+  switch (type)
+  {
+  case HighwayType::HighwayMotorwayLink:
+  case HighwayType::HighwayTrunkLink:
+  case HighwayType::HighwayPrimaryLink:
+  case HighwayType::HighwaySecondaryLink:
+  case HighwayType::HighwayTertiaryLink: return true;
+  default: return false;
+  }
+
+  UNREACHABLE();
+}
+
+bool IsBigHighway(HighwayType type)
+{
+  switch (type)
+  {
+  case HighwayType::HighwayMotorway:
+  case HighwayType::HighwayTrunk:
+  case HighwayType::HighwayPrimary:
+  case HighwayType::HighwaySecondary:
+  case HighwayType::HighwayTertiary: return true;
+  default: return false;
+  }
+
+  UNREACHABLE();
+}
+
+bool FromSmallerToBigger(HighwayType lhs, HighwayType rhs)
+{
+  CHECK_NOT_EQUAL(lhs, rhs, ());
+
+  static std::array<HighwayType, 5> constexpr kHighwayTypes = {
+      HighwayType::HighwayTertiary, HighwayType::HighwaySecondary, HighwayType::HighwayPrimary,
+      HighwayType::HighwayTrunk, HighwayType::HighwayMotorway};
+
+  auto const lhsIt = find(kHighwayTypes.begin(), kHighwayTypes.end(), lhs);
+  auto const rhsIt = find(kHighwayTypes.begin(), kHighwayTypes.end(), rhs);
+  if (lhsIt == kHighwayTypes.end() && rhsIt != kHighwayTypes.end())
+    return true;
+
+  if (lhsIt != kHighwayTypes.end() && rhsIt == kHighwayTypes.end())
+    return false;
+
+  return lhsIt < rhsIt;
+}
+}  // namespace
+
+namespace routing
+{
+IsCrossroadChecker::Type IsCrossroadChecker::operator()(Segment const & current, Segment const & next) const
+{
+  auto const currentSegmentFeatureId = current.GetFeatureId();
+  auto const currentSegmentHwType = m_geometry.GetRoad(currentSegmentFeatureId).GetHighwayType();
+  auto const nextSegmentFeatureId = next.GetFeatureId();
+  auto const nextSegmentHwType = m_geometry.GetRoad(nextSegmentFeatureId).GetHighwayType();
+  auto const currentRoadPoint = current.GetRoadPoint(true /* isFront */);
+  auto const jointId = m_indexGraph.GetJointId(currentRoadPoint);
+  if (jointId == Joint::kInvalidId)
+    return Type::Count;
+
+  bool const isCurrentLink = IsHighwayLink(*currentSegmentHwType);
+  bool const isNextLink = IsHighwayLink(*nextSegmentHwType);
+  bool const isCurrentBig = IsBigHighway(*currentSegmentHwType);
+  bool const isNextBig = IsBigHighway(*nextSegmentHwType);
+
+  if (currentSegmentFeatureId != nextSegmentFeatureId && currentSegmentHwType != nextSegmentHwType)
+  {
+    // Changing highway type.
+    if (isCurrentLink && !isNextLink && isNextBig)
+      return Type::TurnFromSmallerToBigger;
+
+    if (!isCurrentLink && isNextLink && isCurrentBig)
+      return Type::TurnFromBiggerToSmaller;
+
+    // It's move without links.
+    if (!isCurrentLink && !isNextLink)
+    {
+      if (isCurrentBig && !isNextBig)
+        return Type::TurnFromBiggerToSmaller;
+      if (!isCurrentBig && isNextBig)
+        return Type::TurnFromSmallerToBigger;
+    }
+  }
+
+  Type retType = Type::Count;
+  auto const nextRoadPoint = next.GetRoadPoint(false /* isFront */);
+  m_indexGraph.ForEachPoint(jointId, [&](RoadPoint const & point)
+  {
+    if (retType != IsCrossroadChecker::Type::Count)
+      return;
+
+    // Check for already included roads.
+    auto const pointFeatureId = point.GetFeatureId();
+    if (pointFeatureId == currentSegmentFeatureId || pointFeatureId == nextSegmentFeatureId)
+      return;
+
+    auto const & roadGeometry = m_geometry.GetRoad(pointFeatureId);
+    auto const pointHwType = roadGeometry.GetHighwayType();
+    if (currentSegmentHwType == pointHwType)
+      return;
+
+    if (pointHwType == nextSegmentHwType)
+    {
+      // Is the same road but parted on different features.
+      if (roadGeometry.IsEndPointId(point.GetPointId()) && roadGeometry.IsEndPointId(nextRoadPoint.GetPointId()))
+        return;
+    }
+
+    if (isCurrentLink && IsBigHighway(*pointHwType))
+    {
+      retType = Type::IntersectionWithBig;
+      return;
+    }
+
+    if (FromSmallerToBigger(*currentSegmentHwType, *pointHwType))
+    {
+      retType = Type::IntersectionWithBig;
+      return;
+    }
+  });
+
+  return retType;
+}
+
+// static
+void IsCrossroadChecker::MergeCrossroads(Type from, CrossroadInfo & to)
+{
+  if (from != Type::Count)
+    ++to[base::Underlying(from)];
+}
+
+// static
+void IsCrossroadChecker::MergeCrossroads(IsCrossroadChecker::CrossroadInfo const & from,
+                                         IsCrossroadChecker::CrossroadInfo & to)
+{
+  for (size_t i = 0; i < from.size(); ++i)
+    to[i] += from[i];
+}
+
+std::string DebugPrint(IsCrossroadChecker::Type type)
+{
+  switch (type)
+  {
+  case IsCrossroadChecker::Type::TurnFromSmallerToBigger: return "TurnFromSmallerToBigger";
+  case IsCrossroadChecker::Type::TurnFromBiggerToSmaller: return "TurnFromBiggerToSmaller";
+  case IsCrossroadChecker::Type::IntersectionWithBig: return "IntersectionWithBig";
+  case IsCrossroadChecker::Type::Count: return "Count";
+  }
+  UNREACHABLE();
+}
+}  // namespace routing

tools/track_analyzing/track_analyzer/crossroad_checker.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "routing/geometry.hpp"
+#include "routing/index_graph.hpp"
+#include "routing/segment.hpp"
+
+#include "base/stl_helpers.hpp"
+
+#include <array>
+#include <string>
+
+namespace routing
+{
+class IsCrossroadChecker
+{
+public:
+  enum class Type
+  {
+    TurnFromSmallerToBigger,
+    TurnFromBiggerToSmaller,
+    IntersectionWithBig,
+    Count
+  };
+
+  using CrossroadInfo = std::array<size_t, base::Underlying(Type::Count)>;
+
+  IsCrossroadChecker(IndexGraph & indexGraph, Geometry & geometry) : m_indexGraph(indexGraph), m_geometry(geometry) {}
+  /// \brief Compares two segments by their highway type to find if there was a crossroad between them.
+  /// Check if current segment is a joint to find and find all intersections with other roads.
+  Type operator()(Segment const & current, Segment const & next) const;
+
+  static void MergeCrossroads(Type from, CrossroadInfo & to);
+  static void MergeCrossroads(IsCrossroadChecker::CrossroadInfo const & from, IsCrossroadChecker::CrossroadInfo & to);
+
+private:
+  IndexGraph & m_indexGraph;
+  Geometry & m_geometry;
+};
+
+std::string DebugPrint(IsCrossroadChecker::Type type);
+}  // namespace routing

tools/track_analyzing/track_analyzer/track_analyzer.cpp

@@ -0,0 +1,193 @@
+#include "track_analyzing/track_analyzer/cmd_balance_csv.hpp"
+
+#include "track_analyzing/exceptions.hpp"
+#include "track_analyzing/track.hpp"
+#include "track_analyzing/utils.hpp"
+
+#include "indexer/classificator.hpp"
+#include "indexer/classificator_loader.hpp"
+
+#include "base/checked_cast.hpp"
+
+#include <gflags/gflags.h>
+
+#include <string>
+
+using namespace std;
+using namespace track_analyzing;
+
+namespace
+{
+#define DEFINE_string_ext(name, value, description)                           \
+  DEFINE_string(name, value, description);                                    \
+                                                                              \
+  string const & Checked_##name()                                             \
+  {                                                                           \
+    if (FLAGS_##name.empty())                                                 \
+      MYTHROW(MessageException, (string("Specify the argument --") + #name)); \
+                                                                              \
+    return FLAGS_##name;                                                      \
+  }
+
+DEFINE_string_ext(cmd, "",
+                  "command:\n"
+                  "match - based on raw logs gathers points to tracks and matches them to "
+                  "features. To use the tool raw logs should be taken from \"trafin\" production "
+                  "project in gz files and extracted.\n"
+                  "match_dir - the same as match but applies to the directory with raw logs in gz format."
+                  "Process files in several threads.\n"
+                  "unmatched_tracks - based on raw logs gathers points to tracks\n"
+                  "and save tracks to csv. Track points save as lat, log, timestamp in seconds\n"
+                  "tracks - prints track statistics\n"
+                  "track - prints info about single track\n"
+                  "cpptrack - prints track coords to insert them to cpp code\n"
+                  "table - prints csv table based on matched tracks to stdout\n"
+                  "balance_csv - prints csv table based on csv table set in \"in\" param "
+                  "with a distribution set according to input_distribution param.\n"
+                  "gpx - convert raw logs into gpx files\n");
+DEFINE_string_ext(in, "", "input log file name");
+DEFINE_string(out, "", "output track file name");
+DEFINE_string_ext(output_dir, "", "output dir for gpx files");
+DEFINE_string_ext(mwm, "", "short mwm name");
+DEFINE_string_ext(user, "", "user id");
+DEFINE_int32(track, -1, "track index");
+DEFINE_string(input_distribution, "",
+              "path to input data point distribution file. It's a csv file with data point count for "
+              "some mwms. Usage:\n"
+              "- It may be used with match and match_dir command. If so it's a path to save file with "
+              "data point distribution by mwm. If it's empty no file is saved.\n"
+              "- It may be used with balance_csv command. If so it's a path of a file with distribution which "
+              "will be used for normalization (balancing) the result of the command. It should not be empty.");
+DEFINE_uint64(ignore_datapoints_number, 100,
+              "The number of datapoints in an mwm to exclude the mwm from balancing process. If this number "
+              "of datapoints or less number is in an mwm after matching and tabling, the mwm will not used "
+              "for balancing. This param should be used with balance_csv command.");
+
+DEFINE_string(track_extension, ".track", "track files extension");
+DEFINE_bool(no_world_logs, false, "don't print world summary logs");
+DEFINE_bool(no_mwm_logs, false, "don't print logs per mwm");
+DEFINE_bool(no_track_logs, false, "don't print logs per track");
+
+DEFINE_uint64(min_duration, 5 * 60, "minimum track duration in seconds");
+DEFINE_double(min_length, 1000.0, "minimum track length in meters");
+DEFINE_double(min_speed, 15.0, "minimum track average speed in km/hour");
+DEFINE_double(max_speed, 110.0, "maximum track average speed in km/hour");
+DEFINE_bool(ignore_traffic, true, "ignore tracks with traffic data");
+
+size_t Checked_track()
+{
+  if (FLAGS_track < 0)
+    MYTHROW(MessageException, ("Specify the --track key"));
+
+  return static_cast<size_t>(FLAGS_track);
+}
+
+StringFilter MakeFilter(string const & filter)
+{
+  return [&](string const & value)
+  {
+    if (filter.empty())
+      return false;
+    return value != filter;
+  };
+}
+}  // namespace
+
+namespace track_analyzing
+{
+// Print the specified track in C++ form that you can copy paste to C++ source for debugging.
+void CmdCppTrack(string const & trackFile, string const & mwmName, string const & user, size_t trackIdx);
+// Match raw gps logs to tracks.
+void CmdMatch(string const & logFile, string const & trackFile, string const & inputDistribution);
+// The same as match but applies for the directory with raw logs.
+void CmdMatchDir(string const & logDir, string const & trackExt, string const & inputDistribution);
+// Parse |logFile| and save tracks (mwm name, aloha id, lats, lons, timestamps in seconds in csv).
+void CmdUnmatchedTracks(string const & logFile, string const & trackFileCsv);
+// Print aggregated tracks to csv table.
+void CmdTagsTable(string const & filepath, string const & trackExtension, StringFilter mwmIsFiltered,
+                  StringFilter userFilter);
+// Print track information.
+void CmdTrack(string const & trackFile, string const & mwmName, string const & user, size_t trackIdx);
+// Print tracks statistics.
+void CmdTracks(string const & filepath, string const & trackExtension, StringFilter mwmFilter, StringFilter userFilter,
+               TrackFilter const & filter, bool noTrackLogs, bool noMwmLogs, bool noWorldLogs);
+
+void CmdGPX(string const & logFile, string const & outputFilesDir, string const & userID);
+}  // namespace track_analyzing
+
+int main(int argc, char ** argv)
+{
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  string const & cmd = Checked_cmd();
+
+  classificator::Load();
+
+  try
+  {
+    if (cmd == "match")
+    {
+      string const & logFile = Checked_in();
+      CmdMatch(logFile, FLAGS_out.empty() ? logFile + ".track" : FLAGS_out, FLAGS_input_distribution);
+    }
+    else if (cmd == "match_dir")
+    {
+      string const & logDir = Checked_in();
+      CmdMatchDir(logDir, FLAGS_track_extension, FLAGS_input_distribution);
+    }
+    else if (cmd == "unmatched_tracks")
+    {
+      string const & logFile = Checked_in();
+      CmdUnmatchedTracks(logFile, FLAGS_out.empty() ? logFile + ".track.csv" : FLAGS_out);
+    }
+    else if (cmd == "tracks")
+    {
+      TrackFilter const filter(FLAGS_min_duration, FLAGS_min_length, FLAGS_min_speed, FLAGS_max_speed,
+                               FLAGS_ignore_traffic);
+      CmdTracks(Checked_in(), FLAGS_track_extension, MakeFilter(FLAGS_mwm), MakeFilter(FLAGS_user), filter,
+                FLAGS_no_track_logs, FLAGS_no_mwm_logs, FLAGS_no_world_logs);
+    }
+    else if (cmd == "track")
+    {
+      CmdTrack(Checked_in(), Checked_mwm(), Checked_user(), Checked_track());
+    }
+    else if (cmd == "cpptrack")
+    {
+      CmdCppTrack(Checked_in(), Checked_mwm(), Checked_user(), Checked_track());
+    }
+    else if (cmd == "table")
+    {
+      CmdTagsTable(Checked_in(), FLAGS_track_extension, MakeFilter(FLAGS_mwm), MakeFilter(FLAGS_user));
+    }
+    else if (cmd == "balance_csv")
+    {
+      if (FLAGS_input_distribution.empty())
+      {
+        LOG(LERROR, ("input_distribution param should be set to a path to csv file with required "
+                     "distribution. Hint: this file may be saved by match or match_dir command."));
+        return 1;
+      }
+      CmdBalanceCsv(FLAGS_in, FLAGS_input_distribution, base::checked_cast<uint64_t>(FLAGS_ignore_datapoints_number));
+    }
+    else if (cmd == "gpx")
+    {
+      CmdGPX(Checked_in(), Checked_output_dir(), FLAGS_user);
+    }
+    else
+    {
+      LOG(LWARNING, ("Unknown command", FLAGS_cmd));
+      return 1;
+    }
+  }
+  catch (MessageException const & e)
+  {
+    LOG(LWARNING, (e.Msg()));
+    return 1;
+  }
+  catch (RootException const & e)
+  {
+    LOG(LERROR, (e.what()));
+    return 1;
+  }
+
+  return 0;
+}

tools/track_analyzing/track_analyzer/utils.cpp

@@ -0,0 +1,252 @@
+#include "track_analyzing/track_analyzer/utils.hpp"
+
+#include "track_analyzing/log_parser.hpp"
+
+#include "storage/country_info_getter.hpp"
+#include "storage/routing_helpers.hpp"
+
+#include "geometry/tree4d.hpp"
+
+#include "platform/platform.hpp"
+
+#include "coding/csv_reader.hpp"
+
+#include "base/checked_cast.hpp"
+#include "base/logging.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+#include <fstream>
+#include <set>
+#include <sstream>
+#include <utility>
+
+using namespace routing;
+using namespace std;
+using namespace storage;
+using namespace track_analyzing;
+
+namespace
+{
+set<string> GetKeys(Stats::NameToCountMapping const & mapping)
+{
+  set<string> keys;
+  transform(mapping.begin(), mapping.end(), inserter(keys, keys.end()), [](auto const & kv) { return kv.first; });
+  return keys;
+}
+
+void Add(Stats::NameToCountMapping const & addition, Stats::NameToCountMapping & base)
+{
+  set<storage::CountryId> userKeys = GetKeys(addition);
+  set<storage::CountryId> const userKeys2 = GetKeys(base);
+  userKeys.insert(userKeys2.cbegin(), userKeys2.cend());
+
+  for (auto const & c : userKeys)
+  {
+    if (addition.count(c) == 0)
+      continue;
+
+    if (base.count(c) == 0)
+    {
+      base[c] = addition.at(c);
+      continue;
+    }
+
+    base[c] += addition.at(c);
+  }
+}
+
+void PrintMap(string const & keyName, string const & descr, Stats::NameToCountMapping const & mapping,
+              ostringstream & ss)
+{
+  ss << descr << '\n';
+  if (mapping.empty())
+  {
+    ss << "Map is empty." << endl;
+    return;
+  }
+
+  MappingToCsv(keyName, mapping, true /* printPercentage */, ss);
+}
+}  // namespace
+
+namespace track_analyzing
+{
+// Stats ===========================================================================================
+Stats::Stats(NameToCountMapping const & mwmToTotalDataPoints, NameToCountMapping const & countryToTotalDataPoint)
+  : m_mwmToTotalDataPoints(mwmToTotalDataPoints)
+  , m_countryToTotalDataPoints(countryToTotalDataPoint)
+{}
+
+bool Stats::operator==(Stats const & stats) const
+{
+  return m_mwmToTotalDataPoints == stats.m_mwmToTotalDataPoints &&
+         m_countryToTotalDataPoints == stats.m_countryToTotalDataPoints;
+}
+
+void Stats::Add(Stats const & stats)
+{
+  ::Add(stats.m_mwmToTotalDataPoints, m_mwmToTotalDataPoints);
+  ::Add(stats.m_countryToTotalDataPoints, m_countryToTotalDataPoints);
+}
+
+void Stats::AddTracksStats(MwmToTracks const & mwmToTracks, NumMwmIds const & numMwmIds, Storage const & storage)
+{
+  for (auto const & kv : mwmToTracks)
+  {
+    auto const & userToTrack = kv.second;
+    uint64_t dataPointNum = 0;
+    for (auto const & userTrack : userToTrack)
+      dataPointNum += userTrack.second.size();
+
+    NumMwmId const numMwmId = kv.first;
+    auto const mwmName = numMwmIds.GetFile(numMwmId).GetName();
+    AddDataPoints(mwmName, storage, dataPointNum);
+  }
+}
+
+void Stats::AddDataPoints(string const & mwmName, string const & countryName, uint64_t dataPointNum)
+{
+  m_mwmToTotalDataPoints[mwmName] += dataPointNum;
+  m_countryToTotalDataPoints[countryName] += dataPointNum;
+}
+
+void Stats::AddDataPoints(string const & mwmName, storage::Storage const & storage, uint64_t dataPointNum)
+{
+  auto const countryName = storage.GetTopmostParentFor(mwmName);
+  // Note. In case of disputed mwms |countryName| will be empty.
+  AddDataPoints(mwmName, countryName, dataPointNum);
+}
+
+void Stats::SaveMwmDistributionToCsv(string const & csvPath) const
+{
+  LOG(LINFO, ("Saving mwm distribution to", csvPath, "m_mwmToTotalDataPoints size is", m_mwmToTotalDataPoints.size()));
+  if (csvPath.empty())
+    return;
+
+  ofstream ofs(csvPath);
+  CHECK(ofs.is_open(), ("Cannot open file", csvPath));
+  MappingToCsv("mwm", m_mwmToTotalDataPoints, false /* printPercentage */, ofs);
+}
+
+void Stats::Log() const
+{
+  LogMwms();
+  LogCountries();
+}
+
+Stats::NameToCountMapping const & Stats::GetMwmToTotalDataPointsForTesting() const
+{
+  return m_mwmToTotalDataPoints;
+}
+
+Stats::NameToCountMapping const & Stats::GetCountryToTotalDataPointsForTesting() const
+{
+  return m_countryToTotalDataPoints;
+}
+
+void Stats::LogMwms() const
+{
+  LogNameToCountMapping("mwm", "Mwm to total data points number:", m_mwmToTotalDataPoints);
+}
+
+void Stats::LogCountries() const
+{
+  LogNameToCountMapping("country", "Country name to data points number:", m_countryToTotalDataPoints);
+}
+
+void MappingToCsv(string const & keyName, Stats::NameToCountMapping const & mapping, bool printPercentage,
+                  basic_ostream<char> & ss)
+{
+  struct KeyValue
+  {
+    string m_key;
+    uint64_t m_value = 0;
+  };
+
+  if (mapping.empty())
+    return;
+
+  ss << keyName;
+  if (printPercentage)
+    ss << ",number,percent\n";
+  else
+    ss << ",number\n";
+
+  // Sorting from bigger to smaller values.
+  vector<KeyValue> keyValues;
+  keyValues.reserve(mapping.size());
+  for (auto const & kv : mapping)
+    keyValues.push_back({kv.first, kv.second});
+
+  sort(keyValues.begin(), keyValues.end(),
+       [](KeyValue const & a, KeyValue const & b) { return a.m_value > b.m_value; });
+
+  uint64_t allValues = 0;
+  for (auto const & kv : keyValues)
+    allValues += kv.m_value;
+
+  for (auto const & kv : keyValues)
+  {
+    if (kv.m_value == 0)
+      continue;
+
+    ss << kv.m_key << "," << kv.m_value;
+    if (printPercentage)
+      ss << "," << 100.0 * static_cast<double>(kv.m_value) / allValues;
+    ss << '\n';
+  }
+  ss.flush();
+}
+
+void MappingFromCsv(basic_istream<char> & ss, Stats::NameToCountMapping & mapping)
+{
+  for (auto const & row : coding::CSVRunner(coding::CSVReader(ss, true /* hasHeader */, ',' /* kCsvDelimiter */)))
+  {
+    CHECK_EQUAL(row.size(), 2, (row));
+    auto const & key = row[0];
+    uint64_t value = 0;
+    CHECK(strings::to_uint64(row[1], value), ());
+    auto const it = mapping.insert(make_pair(key, value));
+    CHECK(it.second, ());
+  }
+}
+
+void ParseTracks(string const & logFile, shared_ptr<NumMwmIds> const & numMwmIds, MwmToTracks & mwmToTracks)
+{
+  Platform const & platform = GetPlatform();
+  string const dataDir = platform.WritableDir();
+  auto countryInfoGetter = CountryInfoReader::CreateCountryInfoGetter(platform);
+  unique_ptr<m4::Tree<NumMwmId>> mwmTree = MakeNumMwmTree(*numMwmIds, *countryInfoGetter);
+
+  LOG(LINFO, ("Parsing", logFile));
+  LogParser parser(numMwmIds, std::move(mwmTree), dataDir);
+  parser.Parse(logFile, mwmToTracks);
+}
+
+void WriteCsvTableHeader(basic_ostream<char> & stream)
+{
+  stream << "user,mwm,hw type,surface type,maxspeed km/h,is city road,is one way,is day,lat lon,"
+            "distance,time,mean speed km/h,turn from smaller to bigger,turn from bigger to smaller,"
+            "intersection with big\n";
+}
+
+void LogNameToCountMapping(string const & keyName, string const & descr, Stats::NameToCountMapping const & mapping)
+{
+  ostringstream ss;
+  LOG(LINFO, ("\n"));
+  PrintMap(keyName, descr, mapping, ss);
+  LOG(LINFO, (ss.str()));
+}
+
+string DebugPrint(Stats const & s)
+{
+  ostringstream ss;
+  ss << "Stats [\n";
+  PrintMap("mwm", "Mwm to total data points number:", s.m_mwmToTotalDataPoints, ss);
+  PrintMap("country", "Country name to data points number:", s.m_countryToTotalDataPoints, ss);
+  ss << "]" << endl;
+
+  return ss.str();
+}
+}  // namespace track_analyzing

tools/track_analyzing/track_analyzer/utils.hpp

@@ -0,0 +1,75 @@
+#pragma once
+
+#include "storage/storage.hpp"
+
+#include "routing_common/num_mwm_id.hpp"
+
+#include "track_analyzing/track.hpp"
+
+#include <cstdint>
+#include <istream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <string>
+
+namespace track_analyzing
+{
+class Stats
+{
+public:
+  friend std::string DebugPrint(Stats const & s);
+
+  using NameToCountMapping = std::map<std::string, uint64_t>;
+
+  Stats() = default;
+  Stats(NameToCountMapping const & mwmToTotalDataPoints, NameToCountMapping const & countryToTotalDataPoint);
+
+  bool operator==(Stats const & stats) const;
+  void Add(Stats const & stats);
+
+  /// \brief Adds some stats according to |mwmToTracks|.
+  void AddTracksStats(MwmToTracks const & mwmToTracks, routing::NumMwmIds const & numMwmIds,
+                      storage::Storage const & storage);
+
+  /// \brief Adds |dataPointNum| to |m_mwmToTotalDataPoints| and |m_countryToTotalDataPoints|.
+  void AddDataPoints(std::string const & mwmName, std::string const & countryName, uint64_t dataPointNum);
+
+  void AddDataPoints(std::string const & mwmName, storage::Storage const & storage, uint64_t dataPointNum);
+
+  /// \brief Saves csv file with numbers of DataPoints for each mwm to |csvPath|.
+  /// If |csvPath| is empty it does nothing.
+  void SaveMwmDistributionToCsv(std::string const & csvPath) const;
+
+  void Log() const;
+
+  NameToCountMapping const & GetMwmToTotalDataPointsForTesting() const;
+  NameToCountMapping const & GetCountryToTotalDataPointsForTesting() const;
+
+private:
+  void LogMwms() const;
+  void LogCountries() const;
+
+  /// \note These fields may present mapping from territory name to either DataPoints
+  /// or MatchedTrackPoint count.
+  NameToCountMapping m_mwmToTotalDataPoints;
+  NameToCountMapping m_countryToTotalDataPoints;
+};
+
+/// \brief Saves |mapping| as csv to |ss|.
+void MappingToCsv(std::string const & keyName, Stats::NameToCountMapping const & mapping, bool printPercentage,
+                  std::basic_ostream<char> & ss);
+/// \breif Fills |mapping| according to csv in |ss|. Csv header is skipped.
+void MappingFromCsv(std::basic_istream<char> & ss, Stats::NameToCountMapping & mapping);
+
+/// \brief Parses tracks from |logFile| and fills |mwmToTracks|.
+void ParseTracks(std::string const & logFile, std::shared_ptr<routing::NumMwmIds> const & numMwmIds,
+                 MwmToTracks & mwmToTracks);
+
+void WriteCsvTableHeader(std::basic_ostream<char> & stream);
+
+void LogNameToCountMapping(std::string const & keyName, std::string const & descr,
+                           Stats::NameToCountMapping const & mapping);
+
+std::string DebugPrint(Stats const & s);
+}  // namespace track_analyzing