Repo created

libs/traffic/CMakeLists.txt

@@ -0,0 +1,19 @@
+project(traffic)
+
+set(SRC
+  speed_groups.cpp
+  speed_groups.hpp
+  traffic_cache.cpp
+  traffic_cache.hpp
+  traffic_info.cpp
+  traffic_info.hpp
+)
+
+omim_add_library(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  routing_common
+  indexer
+)
+
+omim_add_test_subdirectory(traffic_tests)

libs/traffic/pytraffic/CMakeLists.txt

@@ -0,0 +1,49 @@
+project(pytraffic)
+
+set(
+  SRC
+  bindings.cpp
+)
+
+include_directories(${CMAKE_BINARY_DIR})
+
+omim_add_library(${PROJECT_NAME} MODULE ${SRC})
+
+if (PLATFORM_MAC)
+  omim_link_libraries(
+    ${PROJECT_NAME}
+    Qt5::Widgets
+    "-framework QuartzCore"
+    "-Wl,-undefined,dynamic_lookup"
+  )
+endif()
+
+if (PLATFORM_WIN OR PLATFORM_LINUX)
+  omim_link_libraries(
+    ${PROJECT_NAME}
+    Qt5::Widgets
+  )
+endif()
+
+omim_link_libraries(
+  ${PROJECT_NAME}
+  routing
+  traffic
+  routing_common
+  indexer
+  editor
+  platform
+  geometry
+  coding
+  base
+  cppjansson
+  protobuf
+  pugixml
+  opening_hours
+  succinct
+  ICU::i18n
+  ${Boost_LIBRARIES}
+  ZLIB::ZLIB
+)
+
+set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "")

libs/traffic/pytraffic/bindings.cpp

@@ -0,0 +1,207 @@
+#include "traffic/speed_groups.hpp"
+#include "traffic/traffic_info.hpp"
+
+#include "indexer/classificator_loader.hpp"
+
+#include "platform/platform.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+#include "base/math.hpp"
+
+#include <cstdint>
+#include <map>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "pyhelpers/module_version.hpp"
+#include "pyhelpers/vector_list_conversion.hpp"
+#include "pyhelpers/vector_uint8.hpp"
+
+#include <boost/python.hpp>
+#include <boost/python/suite/indexing/map_indexing_suite.hpp>
+#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
+
+namespace
+{
+using namespace boost::python;
+
+struct SegmentSpeeds
+{
+  SegmentSpeeds() = default;
+  SegmentSpeeds(double weightedSpeed, double weightedRefSpeed, double weight)
+    : m_weightedSpeed(weightedSpeed)
+    , m_weightedRefSpeed(weightedRefSpeed)
+    , m_weight(weight)
+  {}
+
+  double m_weightedSpeed = 0;
+  double m_weightedRefSpeed = 0;
+  double m_weight = 0;
+};
+
+using SegmentMapping = std::map<traffic::TrafficInfo::RoadSegmentId, SegmentSpeeds>;
+
+std::string SegmentSpeedsRepr(SegmentSpeeds const & v)
+{
+  std::ostringstream ss;
+  ss << "SegmentSpeeds("
+     << " weighted_speed=" << v.m_weightedSpeed << " weighted_ref_speed=" << v.m_weightedRefSpeed
+     << " weight=" << v.m_weight << " )";
+  return ss.str();
+}
+
+traffic::TrafficInfo::Coloring TransformToSpeedGroups(SegmentMapping const & segmentMapping)
+{
+  double const kEps = 1e-9;
+  traffic::TrafficInfo::Coloring result;
+  for (auto const & kv : segmentMapping)
+  {
+    double const ws = kv.second.m_weightedSpeed;
+    double const wrs = kv.second.m_weightedRefSpeed;
+    double const w = kv.second.m_weight;
+    if (AlmostEqualAbs(w, 0.0, kEps))
+    {
+      LOG(LWARNING, ("A traffic segment has zero weight."));
+      continue;
+    }
+    double const u = ws / w;
+    double const v = wrs / w;
+    bool const uz = AlmostEqualAbs(u, 0.0, kEps);
+    bool const vz = AlmostEqualAbs(v, 0.0, kEps);
+    if (uz && vz)
+    {
+      result[kv.first] = traffic::SpeedGroup::TempBlock;
+    }
+    else if (vz)
+    {
+      LOG(LWARNING, ("A traffic segment has zero reference speed."));
+      continue;
+    }
+    else
+    {
+      double p = 100.0 * u / v;
+      p = math::Clamp(p, 0.0, 100.0);
+      result[kv.first] = traffic::GetSpeedGroupByPercentage(p);
+    }
+  }
+  return result;
+}
+
+std::string RoadSegmentIdRepr(traffic::TrafficInfo::RoadSegmentId const & v)
+{
+  std::ostringstream ss;
+  ss << "RoadSegmentId(" << v.m_fid << ", " << v.m_idx << ", " << int(v.m_dir) << ")";
+  return ss.str();
+}
+
+boost::python::list GenerateTrafficKeys(std::string const & mwmPath)
+{
+  std::vector<traffic::TrafficInfo::RoadSegmentId> result;
+  traffic::TrafficInfo::ExtractTrafficKeys(mwmPath, result);
+  return pyhelpers::StdVectorToPythonList(result);
+}
+
+std::vector<uint8_t> GenerateTrafficValues(std::vector<traffic::TrafficInfo::RoadSegmentId> const & keys,
+                                           boost::python::dict const & segmentMappingDict, uint8_t useTempBlock)
+{
+  SegmentMapping segmentMapping;
+  boost::python::list mappingKeys = segmentMappingDict.keys();
+  for (size_t i = 0; i < len(mappingKeys); ++i)
+  {
+    object curArg = segmentMappingDict[mappingKeys[i]];
+    if (curArg)
+      segmentMapping[extract<traffic::TrafficInfo::RoadSegmentId>(mappingKeys[i])] =
+          extract<SegmentSpeeds>(segmentMappingDict[mappingKeys[i]]);
+  }
+
+  traffic::TrafficInfo::Coloring const knownColors = TransformToSpeedGroups(segmentMapping);
+  traffic::TrafficInfo::Coloring coloring;
+  traffic::TrafficInfo::CombineColorings(keys, knownColors, coloring);
+
+  std::vector<traffic::SpeedGroup> values(coloring.size());
+
+  size_t i = 0;
+  for (auto const & kv : coloring)
+  {
+    ASSERT_EQUAL(kv.first, keys[i], ());
+    if (useTempBlock == 0 && kv.second == traffic::SpeedGroup::TempBlock)
+      continue;
+
+    values[i] = kv.second;
+    ++i;
+  }
+  ASSERT_EQUAL(i, values.size(), ());
+
+  std::vector<uint8_t> buf;
+  traffic::TrafficInfo::SerializeTrafficValues(values, buf);
+  return buf;
+}
+
+std::vector<uint8_t> GenerateTrafficValuesFromList(boost::python::list const & keys,
+                                                   boost::python::dict const & segmentMappingDict)
+{
+  std::vector<traffic::TrafficInfo::RoadSegmentId> keysVec =
+      pyhelpers::PythonListToStdVector<traffic::TrafficInfo::RoadSegmentId>(keys);
+
+  return GenerateTrafficValues(keysVec, segmentMappingDict, 1 /* useTempBlock */);
+}
+
+std::vector<uint8_t> GenerateTrafficValuesFromBinary(std::vector<uint8_t> const & keysBlob,
+                                                     boost::python::dict const & segmentMappingDict,
+                                                     uint8_t useTempBlock = 1)
+{
+  std::vector<traffic::TrafficInfo::RoadSegmentId> keys;
+  traffic::TrafficInfo::DeserializeTrafficKeys(keysBlob, keys);
+
+  return GenerateTrafficValues(keys, segmentMappingDict, useTempBlock);
+}
+
+void LoadClassificator(std::string const & classifPath)
+{
+  GetPlatform().SetResourceDir(classifPath);
+  classificator::Load();
+}
+}  // namespace
+
+BOOST_PYTHON_MODULE(pytraffic)
+{
+  using namespace boost::python;
+  scope().attr("__version__") = PYBINDINGS_VERSION;
+
+  // Register the to-python converters.
+  to_python_converter<std::vector<uint8_t>, vector_uint8t_to_str>();
+  vector_uint8t_from_python_str();
+
+  class_<SegmentSpeeds>("SegmentSpeeds", init<double, double, double>())
+      .def("__repr__", &SegmentSpeedsRepr)
+      .def_readwrite("weighted_speed", &SegmentSpeeds::m_weightedSpeed)
+      .def_readwrite("weighted_ref_speed", &SegmentSpeeds::m_weightedRefSpeed)
+      .def_readwrite("weight", &SegmentSpeeds::m_weight);
+
+  class_<traffic::TrafficInfo::RoadSegmentId>("RoadSegmentId", init<uint32_t, uint16_t, uint8_t>())
+      .def("__repr__", &RoadSegmentIdRepr)
+      .add_property("fid", &traffic::TrafficInfo::RoadSegmentId::GetFid)
+      .add_property("idx", &traffic::TrafficInfo::RoadSegmentId::GetIdx)
+      .add_property("dir", &traffic::TrafficInfo::RoadSegmentId::GetDir);
+
+  class_<std::vector<traffic::TrafficInfo::RoadSegmentId>>("RoadSegmentIdVec")
+      .def(vector_indexing_suite<std::vector<traffic::TrafficInfo::RoadSegmentId>>());
+
+  enum_<traffic::SpeedGroup>("SpeedGroup")
+      .value("G0", traffic::SpeedGroup::G0)
+      .value("G1", traffic::SpeedGroup::G1)
+      .value("G2", traffic::SpeedGroup::G2)
+      .value("G3", traffic::SpeedGroup::G3)
+      .value("G4", traffic::SpeedGroup::G4)
+      .value("G5", traffic::SpeedGroup::G5)
+      .value("TempBlock", traffic::SpeedGroup::TempBlock)
+      .value("Unknown", traffic::SpeedGroup::Unknown);
+
+  def("load_classificator", LoadClassificator);
+  def("generate_traffic_keys", GenerateTrafficKeys);
+  def("generate_traffic_values_from_list", GenerateTrafficValuesFromList);
+  def("generate_traffic_values_from_binary", GenerateTrafficValuesFromBinary,
+      (arg("keysBlob"), arg("segmentMappingDict"), arg("useTempBlock") = 1));
+}

libs/traffic/pytraffic/example.py

@@ -0,0 +1,47 @@
+from __future__ import print_function
+from pytraffic import (RoadSegmentId,
+                       SegmentSpeeds,
+                       load_classificator,
+                       generate_traffic_keys,
+                       generate_traffic_values_from_binary,
+                       generate_traffic_values_from_list)
+import argparse
+
+parser = argparse.ArgumentParser(description='Example usage of pytraffic.')
+parser.add_argument("--path_to_classificator", dest="path_to_classificator",
+                    help="Path to the directory that contains classificator.txt.")
+parser.add_argument("--path_to_mwm", dest="path_to_mwm", help="Path to the target mwm file.")
+parser.add_argument("--path_to_keys", dest="path_to_keys",
+                    help="Path to serialized key data (i.e. the \"traffic\" section).")
+
+options = parser.parse_args()
+if not options.path_to_classificator or not options.path_to_mwm or not options.path_to_keys:
+  parser.print_help()
+  exit()
+
+load_classificator(options.path_to_classificator)
+
+keys = [
+  RoadSegmentId(0, 0, 0),
+  RoadSegmentId(1, 0, 0),
+  RoadSegmentId(1, 0, 1),
+]
+
+fid, idx, dir = keys[2].fid, keys[2].idx, keys[2].dir
+print(fid, idx, dir)
+
+keys_from_mwm = generate_traffic_keys(options.path_to_mwm)
+
+seg_speeds = SegmentSpeeds(1.0, 2.0, 3.0)
+ws, wrs, w = seg_speeds.weighted_speed, seg_speeds.weighted_ref_speed, seg_speeds.weight
+print(ws, wrs, w)
+
+mapping = {
+  RoadSegmentId(0, 0, 0): SegmentSpeeds(1.0, 2.0, 3.0),
+  RoadSegmentId(1, 0, 1): SegmentSpeeds(4.0, 5.0, 6.0),
+}
+
+buf1 = generate_traffic_values_from_list(keys, mapping)
+
+with open(options.path_to_keys, "rb") as bin_data:
+  buf2 = generate_traffic_values_from_binary(bin_data.read(), {})

libs/traffic/pytraffic/setup.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import sys
+module_dir = os.path.abspath(os.path.dirname(__file__))
+sys.path.insert(0, os.path.join(module_dir, '..', '..'))
+
+from pyhelpers.setup import setup_omim_pybinding
+
+
+NAME = "pytraffic"
+
+setup_omim_pybinding(name=NAME)

libs/traffic/speed_groups.cpp

@@ -0,0 +1,35 @@
+#include "traffic/speed_groups.hpp"
+
+#include "base/math.hpp"
+
+namespace traffic
+{
+uint32_t constexpr kSpeedGroupThresholdPercentage[] = {8, 16, 33, 58, 83, 100, 100, 100};
+
+SpeedGroup GetSpeedGroupByPercentage(double p)
+{
+  p = math::Clamp(p, 0.0, 100.0);
+  SpeedGroup res = SpeedGroup::Unknown;
+  for (int i = static_cast<int>(SpeedGroup::Count) - 1; i >= 0; --i)
+    if (p <= kSpeedGroupThresholdPercentage[i])
+      res = static_cast<SpeedGroup>(i);
+  return res;
+}
+
+std::string DebugPrint(SpeedGroup const & group)
+{
+  switch (group)
+  {
+  case SpeedGroup::G0: return "G0";
+  case SpeedGroup::G1: return "G1";
+  case SpeedGroup::G2: return "G2";
+  case SpeedGroup::G3: return "G3";
+  case SpeedGroup::G4: return "G4";
+  case SpeedGroup::G5: return "G5";
+  case SpeedGroup::TempBlock: return "TempBlock";
+  case SpeedGroup::Unknown: return "Unknown";
+  case SpeedGroup::Count: return "Count";
+  }
+  UNREACHABLE();
+}
+}  // namespace traffic

libs/traffic/speed_groups.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+namespace traffic
+{
+enum class SpeedGroup : uint8_t
+{
+  G0 = 0,
+  G1,
+  G2,
+  G3,
+  G4,
+  G5,
+  TempBlock,
+  Unknown,
+  Count
+};
+
+static_assert(static_cast<uint8_t>(SpeedGroup::Count) <= 8, "");
+
+// Let M be the maximal speed that is possible on a free road
+// and let V be the maximal speed that is possible on this road when
+// taking the traffic data into account.
+// We group all possible ratios (V/M) into a small number of
+// buckets and only use the number of a bucket everywhere.
+// That is, we forget the specific values of V when transmitting and
+// displaying traffic information. The value M of a road is known at the
+// stage of building the mwm containing this road.
+//
+// kSpeedGroupThresholdPercentage[g] denotes the maximal value of (V/M)
+// that is possible for group |g|. Values falling on a border of two groups
+// may belong to either group.
+//
+// The threshold percentage is defined to be 100 for the
+// special groups where V is unknown or not defined.
+extern uint32_t const kSpeedGroupThresholdPercentage[static_cast<size_t>(SpeedGroup::Count)];
+
+/// \note This method is used in traffic jam generation.
+SpeedGroup GetSpeedGroupByPercentage(double p);
+
+std::string DebugPrint(SpeedGroup const & group);
+}  // namespace traffic

libs/traffic/traffic_cache.cpp

@@ -0,0 +1,29 @@
+#include "traffic/traffic_cache.hpp"
+
+namespace traffic
+{
+
+void TrafficCache::Set(MwmSet::MwmId const & mwmId, std::shared_ptr<TrafficInfo::Coloring const> coloring)
+{
+  auto guard = std::lock_guard(m_mutex);
+  m_trafficColoring[mwmId] = std::move(coloring);
+}
+
+void TrafficCache::Remove(MwmSet::MwmId const & mwmId)
+{
+  auto guard = std::lock_guard(m_mutex);
+  m_trafficColoring.erase(mwmId);
+}
+
+void TrafficCache::CopyTraffic(AllMwmTrafficInfo & trafficColoring) const
+{
+  auto guard = std::lock_guard(m_mutex);
+  trafficColoring = m_trafficColoring;
+}
+
+void TrafficCache::Clear()
+{
+  auto guard = std::lock_guard(m_mutex);
+  m_trafficColoring.clear();
+}
+}  // namespace traffic

libs/traffic/traffic_cache.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "traffic/traffic_info.hpp"
+
+#include "indexer/mwm_set.hpp"
+
+#include <map>
+#include <memory>
+#include <mutex>
+
+namespace traffic
+{
+using AllMwmTrafficInfo = std::map<MwmSet::MwmId, std::shared_ptr<traffic::TrafficInfo::Coloring const>>;
+
+class TrafficCache
+{
+public:
+  TrafficCache() = default;
+  virtual ~TrafficCache() = default;
+
+  virtual void CopyTraffic(AllMwmTrafficInfo & trafficColoring) const;
+
+protected:
+  void Set(MwmSet::MwmId const & mwmId, std::shared_ptr<TrafficInfo::Coloring const> coloring);
+  void Remove(MwmSet::MwmId const & mwmId);
+  void Clear();
+
+private:
+  mutable std::mutex m_mutex;
+  AllMwmTrafficInfo m_trafficColoring;
+};
+}  // namespace traffic

libs/traffic/traffic_info.cpp

@@ -0,0 +1,523 @@
+#include "traffic/traffic_info.hpp"
+
+#include "platform/http_client.hpp"
+#include "platform/platform.hpp"
+
+#include "routing_common/car_model.hpp"
+
+#include "indexer/feature_algo.hpp"
+#include "indexer/feature_processor.hpp"
+
+#include "coding/bit_streams.hpp"
+#include "coding/elias_coder.hpp"
+#include "coding/files_container.hpp"
+#include "coding/reader.hpp"
+#include "coding/url.hpp"
+#include "coding/varint.hpp"
+#include "coding/write_to_sink.hpp"
+#include "coding/writer.hpp"
+#include "coding/zlib.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+#include <sstream>
+#include <string>
+
+#include "defines.hpp"
+#include "private.h"
+
+namespace traffic
+{
+using namespace std;
+
+namespace
+{
+bool ReadRemoteFile(string const & url, vector<uint8_t> & contents, int & errorCode)
+{
+  platform::HttpClient request(url);
+  if (!request.RunHttpRequest())
+  {
+    errorCode = request.ErrorCode();
+    LOG(LINFO, ("Couldn't run traffic request", url, ". Error:", errorCode));
+    return false;
+  }
+
+  errorCode = request.ErrorCode();
+
+  string const & result = request.ServerResponse();
+  contents.resize(result.size());
+  memcpy(contents.data(), result.data(), result.size());
+
+  if (errorCode != 200)
+  {
+    LOG(LINFO, ("Traffic request", url, "failed. HTTP Error:", errorCode));
+    return false;
+  }
+
+  return true;
+}
+
+string MakeRemoteURL(string const & name, uint64_t version)
+{
+  if (string(TRAFFIC_DATA_BASE_URL).empty())
+    return {};
+
+  stringstream ss;
+  ss << TRAFFIC_DATA_BASE_URL;
+  if (version != 0)
+    ss << version << "/";
+  ss << url::UrlEncode(name) << TRAFFIC_FILE_EXTENSION;
+  return ss.str();
+}
+
+char constexpr kETag[] = "etag";
+}  // namespace
+
+// TrafficInfo::RoadSegmentId -----------------------------------------------------------------
+TrafficInfo::RoadSegmentId::RoadSegmentId() : m_fid(0), m_idx(0), m_dir(0) {}
+
+TrafficInfo::RoadSegmentId::RoadSegmentId(uint32_t fid, uint16_t idx, uint8_t dir) : m_fid(fid), m_idx(idx), m_dir(dir)
+{}
+
+// TrafficInfo --------------------------------------------------------------------------------
+
+// static
+uint8_t const TrafficInfo::kLatestKeysVersion = 0;
+uint8_t const TrafficInfo::kLatestValuesVersion = 0;
+
+TrafficInfo::TrafficInfo(MwmSet::MwmId const & mwmId, int64_t currentDataVersion)
+  : m_mwmId(mwmId)
+  , m_currentDataVersion(currentDataVersion)
+{
+  if (!mwmId.IsAlive())
+  {
+    LOG(LWARNING, ("Attempt to create a traffic info for dead mwm."));
+    return;
+  }
+  string const mwmPath = mwmId.GetInfo()->GetLocalFile().GetPath(MapFileType::Map);
+  try
+  {
+    FilesContainerR rcont(mwmPath);
+    if (rcont.IsExist(TRAFFIC_KEYS_FILE_TAG))
+    {
+      auto reader = rcont.GetReader(TRAFFIC_KEYS_FILE_TAG);
+      vector<uint8_t> buf(static_cast<size_t>(reader.Size()));
+      reader.Read(0, buf.data(), buf.size());
+      LOG(LINFO, ("Reading keys for", mwmId, "from section"));
+      try
+      {
+        DeserializeTrafficKeys(buf, m_keys);
+      }
+      catch (Reader::Exception const & e)
+      {
+        auto const info = mwmId.GetInfo();
+        LOG(LINFO,
+            ("Could not read traffic keys from section. MWM:", info->GetCountryName(), "Version:", info->GetVersion()));
+      }
+    }
+    else
+    {
+      LOG(LINFO, ("Reading traffic keys for", mwmId, "from the web"));
+      ReceiveTrafficKeys();
+    }
+  }
+  catch (RootException const & e)
+  {
+    LOG(LWARNING, ("Could not initialize traffic keys"));
+  }
+}
+
+// static
+TrafficInfo TrafficInfo::BuildForTesting(Coloring && coloring)
+{
+  TrafficInfo info;
+  info.m_coloring = std::move(coloring);
+  return info;
+}
+
+void TrafficInfo::SetTrafficKeysForTesting(vector<RoadSegmentId> const & keys)
+{
+  m_keys = keys;
+  m_availability = Availability::IsAvailable;
+}
+
+bool TrafficInfo::ReceiveTrafficData(string & etag)
+{
+  vector<SpeedGroup> values;
+  switch (ReceiveTrafficValues(etag, values))
+  {
+  case ServerDataStatus::New: return UpdateTrafficData(values);
+  case ServerDataStatus::NotChanged: return true;
+  case ServerDataStatus::NotFound:
+  case ServerDataStatus::Error: return false;
+  }
+  return false;
+}
+
+SpeedGroup TrafficInfo::GetSpeedGroup(RoadSegmentId const & id) const
+{
+  auto const it = m_coloring.find(id);
+  if (it == m_coloring.cend())
+    return SpeedGroup::Unknown;
+  return it->second;
+}
+
+// static
+void TrafficInfo::ExtractTrafficKeys(string const & mwmPath, vector<RoadSegmentId> & result)
+{
+  result.clear();
+  feature::ForEachFeature(mwmPath, [&](FeatureType & ft, uint32_t const fid)
+  {
+    feature::TypesHolder const types(ft);
+    if (!routing::CarModel::AllLimitsInstance().IsRoad(types))
+      return;
+
+    ft.ParseGeometry(FeatureType::BEST_GEOMETRY);
+    auto const numPoints = static_cast<uint16_t>(ft.GetPointsCount());
+    uint8_t const numDirs = routing::CarModel::AllLimitsInstance().IsOneWay(types) ? 1 : 2;
+    for (uint16_t i = 0; i + 1 < numPoints; ++i)
+      for (uint8_t dir = 0; dir < numDirs; ++dir)
+        result.emplace_back(fid, i, dir);
+  });
+
+  ASSERT(is_sorted(result.begin(), result.end()), ());
+}
+
+// static
+void TrafficInfo::CombineColorings(vector<TrafficInfo::RoadSegmentId> const & keys,
+                                   TrafficInfo::Coloring const & knownColors, TrafficInfo::Coloring & result)
+{
+  result.clear();
+  size_t numKnown = 0;
+  size_t numUnknown = 0;
+#ifdef DEBUG
+  size_t numUnexpectedKeys = knownColors.size();
+#endif
+  for (auto const & key : keys)
+  {
+    auto it = knownColors.find(key);
+    if (it == knownColors.end())
+    {
+      result[key] = SpeedGroup::Unknown;
+      ++numUnknown;
+    }
+    else
+    {
+      result[key] = it->second;
+      ASSERT_GREATER(numUnexpectedKeys--, 0, ());
+      ++numKnown;
+    }
+  }
+
+  LOG(LINFO, ("Road segments: known/unknown/total =", numKnown, numUnknown, numKnown + numUnknown));
+  ASSERT_EQUAL(numUnexpectedKeys, 0, ());
+}
+
+// static
+void TrafficInfo::SerializeTrafficKeys(vector<RoadSegmentId> const & keys, vector<uint8_t> & result)
+{
+  vector<uint32_t> fids;
+  vector<size_t> numSegs;
+  vector<bool> oneWay;
+  for (size_t i = 0; i < keys.size();)
+  {
+    size_t j = i;
+    while (j < keys.size() && keys[i].m_fid == keys[j].m_fid)
+      ++j;
+
+    bool ow = true;
+    for (size_t k = i; k < j; ++k)
+    {
+      if (keys[k].m_dir == RoadSegmentId::kReverseDirection)
+      {
+        ow = false;
+        break;
+      }
+    }
+
+    auto const numDirs = ow ? 1 : 2;
+    size_t numSegsForThisFid = j - i;
+    CHECK_GREATER(numDirs, 0, ());
+    CHECK_EQUAL(numSegsForThisFid % numDirs, 0, ());
+    numSegsForThisFid /= numDirs;
+
+    fids.push_back(keys[i].m_fid);
+    numSegs.push_back(numSegsForThisFid);
+    oneWay.push_back(ow);
+
+    i = j;
+  }
+
+  MemWriter<vector<uint8_t>> memWriter(result);
+  WriteToSink(memWriter, kLatestKeysVersion);
+  WriteVarUint(memWriter, fids.size());
+
+  {
+    BitWriter<decltype(memWriter)> bitWriter(memWriter);
+
+    uint32_t prevFid = 0;
+    for (auto const & fid : fids)
+    {
+      uint64_t const fidDiff = static_cast<uint64_t>(fid - prevFid);
+      bool ok = coding::GammaCoder::Encode(bitWriter, fidDiff + 1);
+      ASSERT(ok, ());
+      UNUSED_VALUE(ok);
+      prevFid = fid;
+    }
+
+    for (auto const & s : numSegs)
+    {
+      bool ok = coding::GammaCoder::Encode(bitWriter, s + 1);
+      ASSERT(ok, ());
+      UNUSED_VALUE(ok);
+    }
+
+    for (auto const val : oneWay)
+      bitWriter.Write(val ? 1 : 0, 1 /* numBits */);
+  }
+}
+
+// static
+void TrafficInfo::DeserializeTrafficKeys(vector<uint8_t> const & data, vector<TrafficInfo::RoadSegmentId> & result)
+{
+  MemReaderWithExceptions memReader(data.data(), data.size());
+  ReaderSource<decltype(memReader)> src(memReader);
+  auto const version = ReadPrimitiveFromSource<uint8_t>(src);
+  CHECK_EQUAL(version, kLatestKeysVersion, ("Unsupported version of traffic values."));
+  auto const n = static_cast<size_t>(ReadVarUint<uint64_t>(src));
+
+  vector<uint32_t> fids(n);
+  vector<size_t> numSegs(n);
+  vector<bool> oneWay(n);
+
+  {
+    BitReader<decltype(src)> bitReader(src);
+    uint32_t prevFid = 0;
+    for (size_t i = 0; i < n; ++i)
+    {
+      prevFid += coding::GammaCoder::Decode(bitReader) - 1;
+      fids[i] = prevFid;
+    }
+
+    for (size_t i = 0; i < n; ++i)
+      numSegs[i] = static_cast<size_t>(coding::GammaCoder::Decode(bitReader) - 1);
+
+    for (size_t i = 0; i < n; ++i)
+      oneWay[i] = bitReader.Read(1) > 0;
+  }
+
+  ASSERT_EQUAL(src.Size(), 0, ());
+
+  result.clear();
+  for (size_t i = 0; i < n; ++i)
+  {
+    auto const fid = fids[i];
+    uint8_t numDirs = oneWay[i] ? 1 : 2;
+    for (size_t j = 0; j < numSegs[i]; ++j)
+    {
+      for (uint8_t dir = 0; dir < numDirs; ++dir)
+      {
+        RoadSegmentId key(fid, j, dir);
+        result.push_back(key);
+      }
+    }
+  }
+}
+
+// static
+void TrafficInfo::SerializeTrafficValues(vector<SpeedGroup> const & values, vector<uint8_t> & result)
+{
+  vector<uint8_t> buf;
+  MemWriter<vector<uint8_t>> memWriter(buf);
+  WriteToSink(memWriter, kLatestValuesVersion);
+  WriteVarUint(memWriter, values.size());
+  {
+    BitWriter<decltype(memWriter)> bitWriter(memWriter);
+    auto const numSpeedGroups = static_cast<uint8_t>(SpeedGroup::Count);
+    static_assert(numSpeedGroups <= 8, "A speed group's value may not fit into 3 bits");
+    for (auto const & v : values)
+    {
+      uint8_t const u = static_cast<uint8_t>(v);
+      CHECK_LESS(u, numSpeedGroups, ());
+      bitWriter.Write(u, 3);
+    }
+  }
+
+  using Deflate = coding::ZLib::Deflate;
+  Deflate deflate(Deflate::Format::ZLib, Deflate::Level::BestCompression);
+
+  deflate(buf.data(), buf.size(), back_inserter(result));
+}
+
+// static
+void TrafficInfo::DeserializeTrafficValues(vector<uint8_t> const & data, vector<SpeedGroup> & result)
+{
+  using Inflate = coding::ZLib::Inflate;
+
+  vector<uint8_t> decompressedData;
+
+  Inflate inflate(Inflate::Format::ZLib);
+  inflate(data.data(), data.size(), back_inserter(decompressedData));
+
+  MemReaderWithExceptions memReader(decompressedData.data(), decompressedData.size());
+  ReaderSource<decltype(memReader)> src(memReader);
+
+  auto const version = ReadPrimitiveFromSource<uint8_t>(src);
+  CHECK_EQUAL(version, kLatestValuesVersion, ("Unsupported version of traffic keys."));
+
+  auto const n = ReadVarUint<uint32_t>(src);
+  result.resize(n);
+  BitReader<decltype(src)> bitReader(src);
+  for (size_t i = 0; i < static_cast<size_t>(n); ++i)
+  {
+    // SpeedGroup's values fit into 3 bits.
+    result[i] = static_cast<SpeedGroup>(bitReader.Read(3));
+  }
+
+  ASSERT_EQUAL(src.Size(), 0, ());
+}
+
+// todo(@m) This is a temporary method. Do not refactor it.
+bool TrafficInfo::ReceiveTrafficKeys()
+{
+  if (!m_mwmId.IsAlive())
+    return false;
+  auto const & info = m_mwmId.GetInfo();
+  if (!info)
+    return false;
+
+  string const url = MakeRemoteURL(info->GetCountryName(), info->GetVersion());
+
+  if (url.empty())
+    return false;
+
+  vector<uint8_t> contents;
+  int errorCode;
+  if (!ReadRemoteFile(url + ".keys", contents, errorCode))
+    return false;
+  if (errorCode != 200)
+  {
+    LOG(LWARNING, ("Network error when reading keys"));
+    return false;
+  }
+
+  vector<RoadSegmentId> keys;
+  try
+  {
+    DeserializeTrafficKeys(contents, keys);
+  }
+  catch (Reader::Exception const & e)
+  {
+    LOG(LINFO, ("Could not read traffic keys received from server. MWM:", info->GetCountryName(),
+                "Version:", info->GetVersion()));
+    return false;
+  }
+  m_keys.swap(keys);
+  return true;
+}
+
+TrafficInfo::ServerDataStatus TrafficInfo::ReceiveTrafficValues(string & etag, vector<SpeedGroup> & values)
+{
+  if (!m_mwmId.IsAlive())
+    return ServerDataStatus::Error;
+
+  auto const & info = m_mwmId.GetInfo();
+  if (!info)
+    return ServerDataStatus::Error;
+
+  auto const version = info->GetVersion();
+  string const url = MakeRemoteURL(info->GetCountryName(), version);
+
+  if (url.empty())
+    return ServerDataStatus::Error;
+
+  platform::HttpClient request(url);
+  request.LoadHeaders(true);
+  request.SetRawHeader("If-None-Match", etag);
+
+  if (!request.RunHttpRequest() || request.ErrorCode() != 200)
+    return ProcessFailure(request, version);
+  try
+  {
+    string const & response = request.ServerResponse();
+    vector<uint8_t> contents(response.cbegin(), response.cend());
+    DeserializeTrafficValues(contents, values);
+  }
+  catch (Reader::Exception const & e)
+  {
+    m_availability = Availability::NoData;
+    LOG(LWARNING, ("Could not read traffic values received from server. MWM:", info->GetCountryName(),
+                   "Version:", info->GetVersion()));
+    return ServerDataStatus::Error;
+  }
+  // Update ETag for this MWM.
+  auto const & headers = request.GetHeaders();
+  auto const it = headers.find(kETag);
+  if (it != headers.end())
+    etag = it->second;
+
+  m_availability = Availability::IsAvailable;
+  return ServerDataStatus::New;
+}
+
+bool TrafficInfo::UpdateTrafficData(vector<SpeedGroup> const & values)
+{
+  m_coloring.clear();
+
+  if (m_keys.size() != values.size())
+  {
+    LOG(LWARNING, ("The number of received traffic values does not correspond to the number of keys:", m_keys.size(),
+                   "keys", values.size(), "values."));
+    m_availability = Availability::NoData;
+    return false;
+  }
+
+  for (size_t i = 0; i < m_keys.size(); ++i)
+    if (values[i] != SpeedGroup::Unknown)
+      m_coloring.emplace(m_keys[i], values[i]);
+
+  return true;
+}
+
+TrafficInfo::ServerDataStatus TrafficInfo::ProcessFailure(platform::HttpClient const & request,
+                                                          int64_t const mwmVersion)
+{
+  switch (request.ErrorCode())
+  {
+  case 404: /* Not Found */
+  {
+    int64_t version = 0;
+    VERIFY(strings::to_int64(request.ServerResponse().c_str(), version), ());
+
+    if (version > mwmVersion && version <= m_currentDataVersion)
+      m_availability = Availability::ExpiredData;
+    else if (version > m_currentDataVersion)
+      m_availability = Availability::ExpiredApp;
+    else
+      m_availability = Availability::NoData;
+    return ServerDataStatus::NotFound;
+  }
+  case 304: /* Not Modified */
+  {
+    m_availability = Availability::IsAvailable;
+    return ServerDataStatus::NotChanged;
+  }
+  }
+
+  m_availability = Availability::Unknown;
+
+  return ServerDataStatus::Error;
+}
+
+string DebugPrint(TrafficInfo::RoadSegmentId const & id)
+{
+  string const dir = id.m_dir == TrafficInfo::RoadSegmentId::kForwardDirection ? "Forward" : "Backward";
+  ostringstream oss;
+  oss << "RoadSegmentId ["
+      << " fid = " << id.m_fid << " idx = " << id.m_idx << " dir = " << dir << " ]";
+  return oss.str();
+}
+}  // namespace traffic

libs/traffic/traffic_info.hpp

@@ -0,0 +1,167 @@
+#pragma once
+
+#include "traffic/speed_groups.hpp"
+
+#include "indexer/mwm_set.hpp"
+
+#include <cstdint>
+#include <map>
+#include <vector>
+
+namespace platform
+{
+class HttpClient;
+}
+
+namespace traffic
+{
+// This class is responsible for providing the real-time
+// information about road traffic for one mwm file.
+class TrafficInfo
+{
+public:
+  static uint8_t const kLatestKeysVersion;
+  static uint8_t const kLatestValuesVersion;
+
+  enum class Availability
+  {
+    IsAvailable,
+    NoData,
+    ExpiredData,
+    ExpiredApp,
+    Unknown
+  };
+
+  struct RoadSegmentId
+  {
+    // m_dir can be kForwardDirection or kReverseDirection.
+    static uint8_t constexpr kForwardDirection = 0;
+    static uint8_t constexpr kReverseDirection = 1;
+
+    RoadSegmentId();
+
+    RoadSegmentId(uint32_t fid, uint16_t idx, uint8_t dir);
+
+    bool operator==(RoadSegmentId const & o) const { return m_fid == o.m_fid && m_idx == o.m_idx && m_dir == o.m_dir; }
+
+    bool operator<(RoadSegmentId const & o) const
+    {
+      if (m_fid != o.m_fid)
+        return m_fid < o.m_fid;
+      if (m_idx != o.m_idx)
+        return m_idx < o.m_idx;
+      return m_dir < o.m_dir;
+    }
+
+    uint32_t GetFid() const { return m_fid; }
+    uint16_t GetIdx() const { return m_idx; }
+    uint8_t GetDir() const { return m_dir; }
+
+    // The ordinal number of feature this segment belongs to.
+    uint32_t m_fid;
+
+    // The ordinal number of this segment in the list of
+    // its feature's segments.
+    uint16_t m_idx : 15;
+
+    // The direction of the segment.
+    uint8_t m_dir : 1;
+  };
+
+  // todo(@m) unordered_map?
+  using Coloring = std::map<RoadSegmentId, SpeedGroup>;
+
+  TrafficInfo() = default;
+
+  TrafficInfo(MwmSet::MwmId const & mwmId, int64_t currentDataVersion);
+
+  static TrafficInfo BuildForTesting(Coloring && coloring);
+  void SetTrafficKeysForTesting(std::vector<RoadSegmentId> const & keys);
+
+  // Fetches the latest traffic data from the server and updates the coloring and ETag.
+  // Construct the url by passing an MwmId.
+  // The ETag or entity tag is part of HTTP, the protocol for the World Wide Web.
+  // It is one of several mechanisms that HTTP provides for web cache validation,
+  // which allows a client to make conditional requests.
+  // *NOTE* This method must not be called on the UI thread.
+  bool ReceiveTrafficData(std::string & etag);
+
+  // Returns the latest known speed group by a feature segment's id
+  // or SpeedGroup::Unknown if there is no information about the segment.
+  SpeedGroup GetSpeedGroup(RoadSegmentId const & id) const;
+
+  MwmSet::MwmId const & GetMwmId() const { return m_mwmId; }
+  Coloring const & GetColoring() const { return m_coloring; }
+  Availability GetAvailability() const { return m_availability; }
+
+  // Extracts RoadSegmentIds from mwm and stores them in a sorted order.
+  static void ExtractTrafficKeys(std::string const & mwmPath, std::vector<RoadSegmentId> & result);
+
+  // Adds the unknown values to the partially known coloring map |knownColors|
+  // so that the keys of the resulting map are exactly |keys|.
+  static void CombineColorings(std::vector<TrafficInfo::RoadSegmentId> const & keys,
+                               TrafficInfo::Coloring const & knownColors, TrafficInfo::Coloring & result);
+
+  // Serializes the keys of the coloring map to |result|.
+  // The keys are road segments ids which do not change during
+  // an mwm's lifetime so there's no point in downloading them every time.
+  // todo(@m) Document the format.
+  static void SerializeTrafficKeys(std::vector<RoadSegmentId> const & keys, std::vector<uint8_t> & result);
+
+  static void DeserializeTrafficKeys(std::vector<uint8_t> const & data, std::vector<RoadSegmentId> & result);
+
+  static void SerializeTrafficValues(std::vector<SpeedGroup> const & values, std::vector<uint8_t> & result);
+
+  static void DeserializeTrafficValues(std::vector<uint8_t> const & data, std::vector<SpeedGroup> & result);
+
+private:
+  enum class ServerDataStatus
+  {
+    New,
+    NotChanged,
+    NotFound,
+    Error,
+  };
+
+  friend void UnitTest_TrafficInfo_UpdateTrafficData();
+
+  // todo(@m) A temporary method. Remove it once the keys are added
+  // to the generator and the data is regenerated.
+  bool ReceiveTrafficKeys();
+
+  // Tries to read the values of the Coloring map from server into |values|.
+  // Returns result of communicating with server as ServerDataStatus.
+  // Otherwise, returns false and does not change m_coloring.
+  ServerDataStatus ReceiveTrafficValues(std::string & etag, std::vector<SpeedGroup> & values);
+
+  // Updates the coloring and changes the availability status if needed.
+  bool UpdateTrafficData(std::vector<SpeedGroup> const & values);
+
+  ServerDataStatus ProcessFailure(platform::HttpClient const & request, int64_t const mwmVersion);
+
+  // The mapping from feature segments to speed groups (see speed_groups.hpp).
+  Coloring m_coloring;
+
+  // The keys of the coloring map. The values are downloaded periodically
+  // and combined with the keys to form m_coloring.
+  // *NOTE* The values must be received in the exact same order that the
+  // keys are saved in.
+  std::vector<RoadSegmentId> m_keys;
+
+  MwmSet::MwmId m_mwmId;
+  Availability m_availability = Availability::Unknown;
+  int64_t m_currentDataVersion = 0;
+};
+
+class TrafficObserver
+{
+public:
+  virtual ~TrafficObserver() = default;
+
+  virtual void OnTrafficInfoClear() = 0;
+  virtual void OnTrafficInfoAdded(traffic::TrafficInfo && info) = 0;
+  virtual void OnTrafficInfoRemoved(MwmSet::MwmId const & mwmId) = 0;
+};
+
+std::string DebugPrint(TrafficInfo::RoadSegmentId const & id);
+}  // namespace traffic

libs/traffic/traffic_tests/CMakeLists.txt

@@ -0,0 +1,7 @@
+project(traffic_tests)
+
+set(SRC traffic_info_test.cpp)
+
+omim_add_test(${PROJECT_NAME} ${SRC} REQUIRE_QT)
+
+target_link_libraries(${PROJECT_NAME} traffic)

libs/traffic/traffic_tests/traffic_info_test.cpp

@@ -0,0 +1,150 @@
+#include "testing/testing.hpp"
+
+#include "traffic/speed_groups.hpp"
+#include "traffic/traffic_info.hpp"
+
+#include "platform/local_country_file.hpp"
+#include "platform/platform_tests_support/writable_dir_changer.hpp"
+
+#include "indexer/mwm_set.hpp"
+
+#include <algorithm>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+using namespace std;
+
+namespace traffic
+{
+namespace
+{
+string const & kMapTestDir = "traffic-test";
+
+class TestMwmSet : public MwmSet
+{
+protected:
+  // MwmSet overrides:
+  unique_ptr<MwmInfo> CreateInfo(platform::LocalCountryFile const &) const override
+  {
+    unique_ptr<MwmInfo> info(new MwmInfo());
+    info->m_version.SetFormat(version::Format::lastFormat);
+    return info;
+  }
+
+  unique_ptr<MwmValue> CreateValue(MwmInfo & info) const override { return make_unique<MwmValue>(info.GetLocalFile()); }
+};
+}  // namespace
+
+/// @todo Need TRAFFIC_DATA_BASE_URL for this test.
+/*
+UNIT_TEST(TrafficInfo_RemoteFile)
+{
+  WritableDirChanger writableDirChanger(kMapTestDir);
+  {
+    TestMwmSet mwmSet;
+    auto const & r =
+        mwmSet.Register(platform::LocalCountryFile::MakeForTesting("traffic_data_test"));
+    TrafficInfo trafficInfo(r.first, r.first.GetInfo()->GetVersion());
+    string etag;
+    TEST(trafficInfo.ReceiveTrafficData(etag), ());
+  }
+  {
+    TestMwmSet mwmSet;
+    auto const & r =
+        mwmSet.Register(platform::LocalCountryFile::MakeForTesting("traffic_data_test2"));
+    TrafficInfo trafficInfo(r.first, r.first.GetInfo()->GetVersion());
+    string etag;
+    TEST(!trafficInfo.ReceiveTrafficData(etag), ());
+  }
+  {
+    TestMwmSet mwmSet;
+    auto const & r =
+        mwmSet.Register(platform::LocalCountryFile::MakeForTesting("traffic_data_test", 101010));
+    TrafficInfo trafficInfo(r.first, r.first.GetInfo()->GetVersion());
+    string etag;
+    TEST(trafficInfo.ReceiveTrafficData(etag), ());
+  }
+}
+*/
+
+UNIT_TEST(TrafficInfo_Serialization)
+{
+  TrafficInfo::Coloring coloring = {
+      {TrafficInfo::RoadSegmentId(0, 0, 0), SpeedGroup::G0},
+
+      {TrafficInfo::RoadSegmentId(1, 0, 0), SpeedGroup::G1},
+      {TrafficInfo::RoadSegmentId(1, 0, 1), SpeedGroup::G3},
+
+      {TrafficInfo::RoadSegmentId(5, 0, 0), SpeedGroup::G2},
+      {TrafficInfo::RoadSegmentId(5, 0, 1), SpeedGroup::G2},
+      {TrafficInfo::RoadSegmentId(5, 1, 0), SpeedGroup::G2},
+      {TrafficInfo::RoadSegmentId(5, 1, 1), SpeedGroup::G5},
+
+      {TrafficInfo::RoadSegmentId(4294967295, 0, 0), SpeedGroup::TempBlock},
+  };
+
+  vector<TrafficInfo::RoadSegmentId> keys;
+  vector<SpeedGroup> values;
+  for (auto const & kv : coloring)
+  {
+    keys.push_back(kv.first);
+    values.push_back(kv.second);
+  }
+
+  {
+    vector<uint8_t> buf;
+    TrafficInfo::SerializeTrafficKeys(keys, buf);
+
+    vector<TrafficInfo::RoadSegmentId> deserializedKeys;
+    TrafficInfo::DeserializeTrafficKeys(buf, deserializedKeys);
+
+    TEST(is_sorted(keys.begin(), keys.end()), ());
+    TEST(is_sorted(deserializedKeys.begin(), deserializedKeys.end()), ());
+    TEST_EQUAL(keys, deserializedKeys, ());
+  }
+
+  {
+    vector<uint8_t> buf;
+    TrafficInfo::SerializeTrafficValues(values, buf);
+
+    vector<SpeedGroup> deserializedValues;
+    TrafficInfo::DeserializeTrafficValues(buf, deserializedValues);
+    TEST_EQUAL(values, deserializedValues, ());
+  }
+}
+
+UNIT_TEST(TrafficInfo_UpdateTrafficData)
+{
+  vector<TrafficInfo::RoadSegmentId> const keys = {
+      TrafficInfo::RoadSegmentId(0, 0, 0),
+
+      TrafficInfo::RoadSegmentId(1, 0, 0),
+      TrafficInfo::RoadSegmentId(1, 0, 1),
+  };
+
+  vector<SpeedGroup> const values1 = {
+      SpeedGroup::G1,
+      SpeedGroup::G2,
+      SpeedGroup::G3,
+  };
+
+  vector<SpeedGroup> const values2 = {
+      SpeedGroup::G4,
+      SpeedGroup::G5,
+      SpeedGroup::Unknown,
+  };
+
+  TrafficInfo info;
+  info.SetTrafficKeysForTesting(keys);
+
+  TEST(info.UpdateTrafficData(values1), ());
+  for (size_t i = 0; i < keys.size(); ++i)
+    TEST_EQUAL(info.GetSpeedGroup(keys[i]), values1[i], ());
+
+  TEST(info.UpdateTrafficData(values2), ());
+  for (size_t i = 0; i < keys.size(); ++i)
+    TEST_EQUAL(info.GetSpeedGroup(keys[i]), values2[i], ());
+}
+}  // namespace traffic