Repo created

libs/search/search_quality/CMakeLists.txt

@@ -0,0 +1,30 @@
+project(search_quality)
+
+set(SRC
+  helpers.cpp
+  helpers.hpp
+  helpers_json.cpp
+  helpers_json.hpp
+  matcher.cpp
+  matcher.hpp
+  sample.cpp
+  sample.hpp
+)
+
+omim_add_library(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  search_tests_support  # TestSearchEngine in helpers.cpp
+  search
+  indexer
+)
+
+if (NOT SKIP_QT_GUI)
+  omim_add_tool_subdirectory(assessment_tool)
+endif()
+
+omim_add_tool_subdirectory(features_collector_tool)
+omim_add_tool_subdirectory(samples_generation_tool)
+omim_add_tool_subdirectory(search_quality_tool)
+
+omim_add_test_subdirectory(search_quality_tests)

libs/search/search_quality/README.txt

@@ -0,0 +1,56 @@
+This document describes how to use the tools for search quality analysis.
+
+
+1. Prerequisites.
+
+   * Get the latest version of the project (https://codeberg.org/comaps/comaps) and build it.
+
+   * Get the latest samples.lisp file with search queries. If you don't know
+     how to get it, please, contact the search team.
+
+   * Install Common Lisp. Note that there are many implementations,
+     but we recommend to use SBCL (http://www.sbcl.org/).
+
+   * Install Python 3.x and packages for data analysis (sklearn, scipy, numpy, pandas, matplotlib).
+
+   * Download maps necessary for search quality tests.
+     For example:
+
+     ./download-maps.sh -v 160524
+
+     will download all necessary maps of version 160524 to the current directory.
+
+
+2. This section describes how to run search engine on a set of search
+   queries and how to get a CSV file with search engine output.
+
+    i) Run gen-samples.lisp script to get search queries with lists of
+       vital or relevant responses in JSON format. For example:
+
+       ./gen-samples.lisp < samples.lisp > samples.jsonl
+
+   ii) Run features_collector_tool from the build directory.
+       For example:
+
+       features_collector_tool --mwm_path path-to-downloaded-maps \
+         --json_in samples.jsonl \
+         --stats_path /tmp/stats.txt \
+         2>/dev/null >samples.csv
+
+       runs search engine on all queries from samples.jsonl, prints
+       useful info to /tmp/stats.txt and generates a CSV file with
+       search engine output on each query.
+
+       The resulting CSV file is ready for analysis, i.e. for search
+       quality evaluation, ranking models learning etc. For details,
+       take a look at scoring_model.py script.
+
+  iii) To take a quick look at what the search returns without
+       launching the application, consider using search_quality_tool:
+
+       search_quality_tool --viewport=moscow \
+         --queries_path=path-to-omim/search/search_quality/search_quality_tool/queries.txt
+         --top 1 \
+         2>/dev/null
+
+       By default, map files in path-to-omim/data are used.

libs/search/search_quality/assessment_tool/CMakeLists.txt

@@ -0,0 +1,49 @@
+project(assessment_tool)
+
+set(SRC
+  assessment_tool.cpp
+  context.cpp
+  context.hpp
+  edits.cpp
+  edits.hpp
+  feature_info_dialog.cpp
+  feature_info_dialog.hpp
+  helpers.cpp
+  helpers.hpp
+  main_model.cpp
+  main_model.hpp
+  main_view.cpp
+  main_view.hpp
+  model.hpp
+  result_view.cpp
+  result_view.hpp
+  results_view.cpp
+  results_view.hpp
+  sample_view.cpp
+  sample_view.hpp
+  samples_view.cpp
+  samples_view.hpp
+  search_request_runner.cpp
+  search_request_runner.hpp
+  view.hpp
+)
+
+#omim_add_executable(${PROJECT_NAME} MACOSX_BUNDLE ${SRC})
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+set_target_properties(${PROJECT_NAME} PROPERTIES AUTOMOC ON)
+
+target_link_libraries(${PROJECT_NAME}
+  search_quality
+  qt_common
+  map
+  gflags::gflags
+)
+
+# if (PLATFORM_MAC)
+#   set_target_properties(
+#     ${PROJECT_NAME}
+#     PROPERTIES
+#     MACOSX_BUNDLE_INFO_PLIST ${PROJECT_SOURCE_DIR}/Info.plist
+#   )
+# endif()

libs/search/search_quality/assessment_tool/Info.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<plist>
+  <dict>
+    <key>NSPrincipalClass</key>
+    <string>NSApplication</string>
+
+    <key>NSHighResolutionCapable</key>
+    <string>True</string>
+  </dict>
+</plist>

libs/search/search_quality/assessment_tool/assessment_tool.cpp

@@ -0,0 +1,58 @@
+#include "main_model.hpp"
+#include "main_view.hpp"
+
+#include "qt/qt_common/helpers.hpp"
+
+#include "map/framework.hpp"
+
+#include "search/search_quality/helpers.hpp"
+
+#include "platform/platform_tests_support/helpers.hpp"
+
+#include "platform/platform.hpp"
+
+#include <QtWidgets/QApplication>
+
+#include <gflags/gflags.h>
+
+DEFINE_string(resources_path, "", "Path to resources directory");
+DEFINE_string(data_path, "", "Path to data directory");
+DEFINE_string(samples_path, "", "Path to the file with samples to open on startup");
+DEFINE_uint64(num_threads, 4, "Number of search engine threads");
+
+int main(int argc, char ** argv)
+{
+  platform::tests_support::ChangeMaxNumberOfOpenFiles(search::search_quality::kMaxOpenFiles);
+
+  gflags::SetUsageMessage("Assessment tool.");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  Platform & platform = GetPlatform();
+  if (!FLAGS_resources_path.empty())
+    platform.SetResourceDir(FLAGS_resources_path);
+  if (!FLAGS_data_path.empty())
+    platform.SetWritableDirForTests(FLAGS_data_path);
+
+  Q_INIT_RESOURCE(resources_common);
+  QApplication app(argc, argv);
+
+  qt::common::SetDefaultSurfaceFormat(app.platformName());
+
+  FrameworkParams params;
+  CHECK_GREATER(FLAGS_num_threads, 0, ());
+  params.m_numSearchAPIThreads = FLAGS_num_threads;
+
+  Framework framework(params);
+  MainView view(framework, app.primaryScreen()->geometry());
+  MainModel model(framework);
+
+  model.SetView(view);
+  view.SetModel(model);
+
+  view.showMaximized();
+
+  if (!FLAGS_samples_path.empty())
+    model.Open(FLAGS_samples_path);
+
+  return app.exec();
+}

libs/search/search_quality/assessment_tool/context.cpp

@@ -0,0 +1,181 @@
+#include "search/search_quality/assessment_tool/context.hpp"
+
+#include "search/feature_loader.hpp"
+#include "search/search_quality/matcher.hpp"
+
+#include "base/assert.hpp"
+
+#include <utility>
+
+using namespace std;
+
+// Context -----------------------------------------------------------------------------------------
+void Context::Clear()
+{
+  m_goldenMatching.clear();
+  m_actualMatching.clear();
+
+  m_foundResults.Clear();
+  m_foundResultsEdits.Clear();
+
+  m_nonFoundResults.clear();
+  m_nonFoundResultsEdits.Clear();
+
+  m_sampleEdits.Clear();
+
+  m_initialized = false;
+}
+
+void Context::LoadFromSample(search::Sample const & sample)
+{
+  Clear();
+  m_sample = sample;
+  m_sampleEdits.Reset(sample.m_useless);
+}
+
+search::Sample Context::MakeSample(search::FeatureLoader & loader) const
+{
+  search::Sample outSample = m_sample;
+
+  if (!m_initialized)
+    return outSample;
+
+  auto const & foundEntries = m_foundResultsEdits.GetEntries();
+  auto const & nonFoundEntries = m_nonFoundResultsEdits.GetEntries();
+
+  auto & outResults = outSample.m_results;
+  outResults.clear();
+
+  CHECK_EQUAL(m_goldenMatching.size(), m_sample.m_results.size(), ());
+  CHECK_EQUAL(m_actualMatching.size(), foundEntries.size(), ());
+  CHECK_EQUAL(m_actualMatching.size(), m_foundResults.GetCount(), ());
+
+  // Iterates over original (loaded from the file with search samples)
+  // results first.
+  size_t k = 0;
+  for (size_t i = 0; i < m_sample.m_results.size(); ++i)
+  {
+    auto const j = m_goldenMatching[i];
+
+    if (j == search::Matcher::kInvalidId)
+    {
+      auto const & entry = nonFoundEntries[k++];
+      auto const deleted = entry.m_deleted;
+      auto const & curr = entry.m_currRelevance;
+      if (!deleted && curr)
+      {
+        auto result = m_sample.m_results[i];
+        result.m_relevance = *curr;
+        outResults.push_back(result);
+      }
+      continue;
+    }
+
+    if (!foundEntries[j].m_currRelevance)
+      continue;
+
+    auto result = m_sample.m_results[i];
+    result.m_relevance = *foundEntries[j].m_currRelevance;
+    outResults.push_back(std::move(result));
+  }
+
+  // Iterates over results retrieved during assessment.
+  for (size_t i = 0; i < m_foundResults.GetCount(); ++i)
+  {
+    auto const j = m_actualMatching[i];
+    if (j != search::Matcher::kInvalidId)
+    {
+      // This result was processed by the loop above.
+      continue;
+    }
+
+    if (!foundEntries[i].m_currRelevance)
+      continue;
+
+    auto const & result = m_foundResults[i];
+    // No need in non-feature results.
+    if (result.GetResultType() != search::Result::Type::Feature)
+      continue;
+
+    auto ft = loader.Load(result.GetFeatureID());
+    CHECK(ft, ());
+    outResults.push_back(search::Sample::Result::Build(*ft, *foundEntries[i].m_currRelevance));
+  }
+
+  outSample.m_useless = m_sampleEdits.m_currUseless;
+
+  return outSample;
+}
+
+void Context::ApplyEdits()
+{
+  if (!m_initialized)
+    return;
+  m_foundResultsEdits.Apply();
+  m_nonFoundResultsEdits.Apply();
+  m_sampleEdits.Apply();
+}
+
+// ContextList -------------------------------------------------------------------------------------
+ContextList::ContextList(OnResultsUpdate onResultsUpdate, OnResultsUpdate onNonFoundResultsUpdate,
+                         OnSampleUpdate onSampleUpdate)
+  : m_onResultsUpdate(onResultsUpdate)
+  , m_onNonFoundResultsUpdate(onNonFoundResultsUpdate)
+  , m_onSampleUpdate(onSampleUpdate)
+{}
+
+void ContextList::Resize(size_t size)
+{
+  size_t const oldSize = m_contexts.size();
+
+  for (size_t i = size; i < oldSize; ++i)
+    m_contexts[i].Clear();
+  if (size < m_contexts.size())
+    m_contexts.erase(m_contexts.begin() + size, m_contexts.end());
+
+  m_hasChanges.resize(size);
+  for (size_t i = oldSize; i < size; ++i)
+  {
+    m_contexts.emplace_back(
+        [this, i](ResultsEdits::Update const & update)
+    {
+      OnContextUpdated(i);
+      if (m_onResultsUpdate)
+        m_onResultsUpdate(i, update);
+    },
+        [this, i](ResultsEdits::Update const & update)
+    {
+      OnContextUpdated(i);
+      if (m_onNonFoundResultsUpdate)
+        m_onNonFoundResultsUpdate(i, update);
+    }, [this, i]()
+    {
+      OnContextUpdated(i);
+      if (m_onSampleUpdate)
+        m_onSampleUpdate(i);
+    });
+  }
+}
+
+vector<search::Sample> ContextList::MakeSamples(search::FeatureLoader & loader) const
+{
+  vector<search::Sample> samples;
+  for (auto const & context : m_contexts)
+    samples.push_back(context.MakeSample(loader));
+  return samples;
+}
+
+void ContextList::ApplyEdits()
+{
+  for (auto & context : m_contexts)
+    context.ApplyEdits();
+}
+
+void ContextList::OnContextUpdated(size_t index)
+{
+  if (!m_hasChanges[index] && m_contexts[index].HasChanges())
+    ++m_numChanges;
+  if (m_hasChanges[index] && !m_contexts[index].HasChanges())
+    --m_numChanges;
+  m_hasChanges[index] = m_contexts[index].HasChanges();
+}

libs/search/search_quality/assessment_tool/context.hpp

@@ -0,0 +1,139 @@
+#pragma once
+
+#include "search/result.hpp"
+#include "search/search_quality/assessment_tool/edits.hpp"
+#include "search/search_quality/matcher.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "base/string_utils.hpp"
+
+#include <cstddef>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace search
+{
+class FeatureLoader;
+}
+
+struct Context
+{
+  enum class SearchState
+  {
+    Untouched,
+    InQueue,
+    Completed
+  };
+
+  Context(ResultsEdits::OnUpdate onFoundResultsUpdate, ResultsEdits::OnUpdate onNonFoundResultsUpdate,
+          SampleEdits::OnUpdate onSampleUpdate)
+    : m_foundResultsEdits(onFoundResultsUpdate)
+    , m_nonFoundResultsEdits(onNonFoundResultsUpdate)
+    , m_sampleEdits(onSampleUpdate)
+  {}
+
+  void AddNonFoundResult(search::Sample::Result const & result)
+  {
+    CHECK_EQUAL(m_goldenMatching.size(), m_sample.m_results.size(), ());
+
+    m_sample.m_results.push_back(result);
+    m_goldenMatching.push_back(search::Matcher::kInvalidId);
+
+    m_nonFoundResults.push_back(result);
+    m_nonFoundResultsEdits.Add(result.m_relevance);
+  }
+
+  bool IsUseless() const { return m_sampleEdits.m_currUseless; }
+
+  bool HasChanges() const
+  {
+    if (m_sampleEdits.HasChanges())
+      return true;
+    if (!m_initialized)
+      return false;
+    return m_foundResultsEdits.HasChanges() || m_nonFoundResultsEdits.HasChanges();
+  }
+
+  void Clear();
+
+  void LoadFromSample(search::Sample const & sample);
+
+  // Makes sample in accordance with uncommited edits.
+  search::Sample MakeSample(search::FeatureLoader & loader) const;
+
+  // Commits all edits.
+  void ApplyEdits();
+
+  search::Sample m_sample;
+  search::Results m_foundResults;
+  ResultsEdits m_foundResultsEdits;
+
+  std::vector<size_t> m_goldenMatching;
+  std::vector<size_t> m_actualMatching;
+
+  std::vector<search::Sample::Result> m_nonFoundResults;
+  ResultsEdits m_nonFoundResultsEdits;
+
+  SampleEdits m_sampleEdits;
+
+  SearchState m_searchState = SearchState::Untouched;
+
+  bool m_initialized = false;
+};
+
+class ContextList
+{
+public:
+  class SamplesSlice
+  {
+  public:
+    SamplesSlice() = default;
+    explicit SamplesSlice(ContextList const & contexts) : m_contexts(&contexts) {}
+
+    bool IsValid() const { return m_contexts != nullptr; }
+
+    std::string GetLabel(size_t index) const { return strings::ToUtf8((*m_contexts)[index].m_sample.m_query); }
+
+    bool IsChanged(size_t index) const { return (*m_contexts)[index].HasChanges(); }
+
+    Context::SearchState GetSearchState(size_t index) const { return (*m_contexts)[index].m_searchState; }
+
+    bool IsUseless(size_t index) const { return (*m_contexts)[index].m_sampleEdits.m_currUseless; }
+
+    size_t Size() const { return m_contexts->Size(); }
+
+  private:
+    ContextList const * m_contexts = nullptr;
+  };
+
+  using OnResultsUpdate = std::function<void(size_t index, ResultsEdits::Update const & update)>;
+  using OnSampleUpdate = std::function<void(size_t index)>;
+
+  ContextList(OnResultsUpdate onResultsUpdate, OnResultsUpdate onNonFoundResultsUpdate, OnSampleUpdate onSampleUpdate);
+
+  void Resize(size_t size);
+  size_t Size() const { return m_contexts.size(); }
+
+  Context & operator[](size_t i) { return m_contexts[i]; }
+  Context const & operator[](size_t i) const { return m_contexts[i]; }
+
+  bool HasChanges() const { return m_numChanges != 0; }
+
+  // Generates search samples in accordance with uncommited edits.
+  std::vector<search::Sample> MakeSamples(search::FeatureLoader & loader) const;
+
+  // Commits all edits.
+  void ApplyEdits();
+
+private:
+  void OnContextUpdated(size_t index);
+
+  std::vector<Context> m_contexts;
+  std::vector<bool> m_hasChanges;
+  size_t m_numChanges = 0;
+
+  OnResultsUpdate m_onResultsUpdate;
+  OnResultsUpdate m_onNonFoundResultsUpdate;
+  OnSampleUpdate m_onSampleUpdate;
+};

libs/search/search_quality/assessment_tool/edits.cpp

@@ -0,0 +1,213 @@
+#include "search/search_quality/assessment_tool/edits.hpp"
+
+#include "base/assert.hpp"
+
+using namespace std;
+
+namespace
+{
+void UpdateNumEdits(ResultsEdits::Entry const & entry, ResultsEdits::Relevance const & r, size_t & numEdits)
+{
+  if (entry.m_currRelevance != entry.m_origRelevance && r == entry.m_origRelevance)
+  {
+    CHECK_GREATER(numEdits, 0, ());
+    --numEdits;
+  }
+  if (entry.m_currRelevance == entry.m_origRelevance && r != entry.m_origRelevance)
+    ++numEdits;
+}
+}  // namespace
+
+// SampleEdits -------------------------------------------------------------------------------------
+void SampleEdits::Reset(bool origUseless)
+{
+  m_origUseless = origUseless;
+  m_currUseless = origUseless;
+}
+
+void SampleEdits::FlipUsefulness()
+{
+  m_currUseless ^= true;
+  if (m_onUpdate)
+    m_onUpdate();
+}
+
+void SampleEdits::Apply()
+{
+  m_origUseless = m_currUseless;
+  if (m_onUpdate)
+    m_onUpdate();
+}
+
+// ResultsEdits::Editor ----------------------------------------------------------------------------
+ResultsEdits::Editor::Editor(ResultsEdits & parent, size_t index) : m_parent(parent), m_index(index) {}
+
+bool ResultsEdits::Editor::Set(Relevance relevance)
+{
+  return m_parent.SetRelevance(m_index, relevance);
+}
+
+optional<ResultsEdits::Relevance> const & ResultsEdits::Editor::Get() const
+{
+  return m_parent.Get(m_index).m_currRelevance;
+}
+
+bool ResultsEdits::Editor::HasChanges() const
+{
+  return m_parent.HasChanges(m_index);
+}
+
+ResultsEdits::Entry::Type ResultsEdits::Editor::GetType() const
+{
+  return m_parent.Get(m_index).m_type;
+}
+
+// ResultsEdits ------------------------------------------------------------------------------------
+void ResultsEdits::Apply()
+{
+  WithObserver(Update::MakeAll(), [this]()
+  {
+    for (auto & entry : m_entries)
+    {
+      entry.m_origRelevance = entry.m_currRelevance;
+      entry.m_type = Entry::Type::Loaded;
+    }
+    m_numEdits = 0;
+  });
+}
+
+void ResultsEdits::Reset(vector<optional<ResultsEdits::Relevance>> const & relevances)
+{
+  WithObserver(Update::MakeAll(), [this, &relevances]()
+  {
+    m_entries.resize(relevances.size());
+    for (size_t i = 0; i < m_entries.size(); ++i)
+    {
+      auto & entry = m_entries[i];
+      entry.m_origRelevance = relevances[i];
+      entry.m_currRelevance = relevances[i];
+      entry.m_deleted = false;
+      entry.m_type = Entry::Type::Loaded;
+    }
+    m_numEdits = 0;
+  });
+}
+
+bool ResultsEdits::SetRelevance(size_t index, Relevance relevance)
+{
+  return WithObserver(Update::MakeSingle(index), [this, index, relevance]()
+  {
+    CHECK_LESS(index, m_entries.size(), ());
+
+    auto & entry = m_entries[index];
+
+    UpdateNumEdits(entry, relevance, m_numEdits);
+
+    entry.m_currRelevance = relevance;
+    return entry.m_currRelevance != entry.m_origRelevance;
+  });
+}
+
+void ResultsEdits::SetAllRelevances(Relevance relevance)
+{
+  WithObserver(Update::MakeAll(), [this, relevance]()
+  {
+    for (auto & entry : m_entries)
+    {
+      UpdateNumEdits(entry, relevance, m_numEdits);
+      entry.m_currRelevance = relevance;
+    }
+  });
+}
+
+void ResultsEdits::Add(Relevance relevance)
+{
+  auto const index = m_entries.size();
+  WithObserver(Update::MakeAdd(index), [&]()
+  {
+    m_entries.emplace_back(relevance, Entry::Type::Created);
+    ++m_numEdits;
+  });
+}
+
+void ResultsEdits::Delete(size_t index)
+{
+  return WithObserver(Update::MakeDelete(index), [this, index]()
+  {
+    CHECK_LESS(index, m_entries.size(), ());
+
+    auto & entry = m_entries[index];
+    CHECK(!entry.m_deleted, ());
+    entry.m_deleted = true;
+    switch (entry.m_type)
+    {
+    case Entry::Type::Loaded: ++m_numEdits; break;
+    case Entry::Type::Created: --m_numEdits; break;
+    }
+  });
+}
+
+void ResultsEdits::Resurrect(size_t index)
+{
+  return WithObserver(Update::MakeResurrect(index), [this, index]()
+  {
+    CHECK_LESS(index, m_entries.size(), ());
+
+    auto & entry = m_entries[index];
+    CHECK(entry.m_deleted, ());
+    CHECK_GREATER(m_numEdits, 0, ());
+    entry.m_deleted = false;
+    switch (entry.m_type)
+    {
+    case Entry::Type::Loaded: --m_numEdits; break;
+    case Entry::Type::Created: ++m_numEdits; break;
+    }
+  });
+}
+
+ResultsEdits::Entry & ResultsEdits::GetEntry(size_t index)
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  return m_entries[index];
+}
+
+ResultsEdits::Entry const & ResultsEdits::GetEntry(size_t index) const
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  return m_entries[index];
+}
+
+vector<optional<ResultsEdits::Relevance>> ResultsEdits::GetRelevances() const
+{
+  vector<optional<ResultsEdits::Relevance>> relevances(m_entries.size());
+  for (size_t i = 0; i < m_entries.size(); ++i)
+    relevances[i] = m_entries[i].m_currRelevance;
+  return relevances;
+}
+
+ResultsEdits::Entry const & ResultsEdits::Get(size_t index) const
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  return m_entries[index];
+}
+
+void ResultsEdits::Clear()
+{
+  WithObserver(Update::MakeAll(), [this]()
+  {
+    m_entries.clear();
+    m_numEdits = 0;
+  });
+}
+
+bool ResultsEdits::HasChanges() const
+{
+  return m_numEdits != 0;
+}
+
+bool ResultsEdits::HasChanges(size_t index) const
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  auto const & entry = m_entries[index];
+  return entry.m_currRelevance != entry.m_origRelevance;
+}

libs/search/search_quality/assessment_tool/edits.hpp

@@ -0,0 +1,153 @@
+#pragma once
+
+#include "search/search_quality/sample.hpp"
+
+#include "base/scope_guard.hpp"
+
+#include <cstddef>
+#include <functional>
+#include <limits>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+struct SampleEdits
+{
+  using OnUpdate = std::function<void()>;
+
+  explicit SampleEdits(OnUpdate onUpdate) : m_onUpdate(onUpdate) {}
+
+  void Reset(bool origUseless);
+  void FlipUsefulness();
+  void Apply();
+  bool HasChanges() const { return m_origUseless != m_currUseless; }
+  void Clear() {}
+
+  bool m_origUseless = false;
+  bool m_currUseless = false;
+
+  OnUpdate m_onUpdate;
+};
+
+class ResultsEdits
+{
+public:
+  using Relevance = search::Sample::Result::Relevance;
+
+  struct Entry
+  {
+    enum class Type
+    {
+      Loaded,
+      Created
+    };
+
+    Entry() = default;
+    Entry(std::optional<Relevance> relevance, Type type)
+      : m_currRelevance(relevance)
+      , m_origRelevance(relevance)
+      , m_type(type)
+    {}
+
+    std::optional<Relevance> m_currRelevance = {};
+    std::optional<Relevance> m_origRelevance = {};
+    bool m_deleted = false;
+    Type m_type = Type::Loaded;
+  };
+
+  struct Update
+  {
+    static auto constexpr kInvalidIndex = std::numeric_limits<size_t>::max();
+
+    enum class Type
+    {
+      Single,
+      All,
+      Add,
+      Delete,
+      Resurrect
+    };
+
+    Update() = default;
+    Update(Type type, size_t index) : m_type(type), m_index(index) {}
+
+    static Update MakeAll() { return {}; }
+    static Update MakeSingle(size_t index) { return {Type::Single, index}; }
+    static Update MakeAdd(size_t index) { return {Type::Add, index}; }
+    static Update MakeDelete(size_t index) { return {Type::Delete, index}; }
+    static Update MakeResurrect(size_t index) { return {Type::Resurrect, index}; }
+
+    Type m_type = Type::All;
+    size_t m_index = kInvalidIndex;
+  };
+
+  using OnUpdate = std::function<void(Update const & update)>;
+
+  class Editor
+  {
+  public:
+    Editor(ResultsEdits & parent, size_t index);
+
+    // Sets relevance to |relevance|. Returns true iff |relevance|
+    // differs from the original one.
+    bool Set(Relevance relevance);
+    std::optional<Relevance> const & Get() const;
+    bool HasChanges() const;
+    Entry::Type GetType() const;
+
+  private:
+    ResultsEdits & m_parent;
+    size_t m_index = 0;
+  };
+
+  explicit ResultsEdits(OnUpdate onUpdate) : m_onUpdate(onUpdate) {}
+
+  void Apply();
+  void Reset(std::vector<std::optional<Relevance>> const & relevances);
+
+  // Sets relevance at |index| to |relevance|. Returns true iff
+  // |relevance| differs from the original one.
+  bool SetRelevance(size_t index, Relevance relevance);
+
+  // Sets relevances of all entries to |relevance|.
+  void SetAllRelevances(Relevance relevance);
+
+  // Adds a new entry.
+  void Add(Relevance relevance);
+
+  // Marks entry at |index| as deleted.
+  void Delete(size_t index);
+
+  // Resurrects previously deleted entry at |index|.
+  void Resurrect(size_t index);
+
+  std::vector<Entry> const & GetEntries() const { return m_entries; }
+  Entry & GetEntry(size_t index);
+  Entry const & GetEntry(size_t index) const;
+  size_t NumEntries() const { return m_entries.size(); }
+  std::vector<std::optional<Relevance>> GetRelevances() const;
+
+  Entry const & Get(size_t index) const;
+
+  void Clear();
+  bool HasChanges() const;
+  bool HasChanges(size_t index) const;
+
+private:
+  template <typename Fn>
+  std::invoke_result_t<Fn> WithObserver(Update const & update, Fn && fn)
+  {
+    SCOPE_GUARD(obsCall, ([this, &update]()
+    {
+      if (m_onUpdate)
+        m_onUpdate(update);
+    }));
+    return fn();
+  }
+
+  std::vector<Entry> m_entries;
+
+  size_t m_numEdits = 0;
+
+  OnUpdate m_onUpdate;
+};

libs/search/search_quality/assessment_tool/feature_info_dialog.cpp

@@ -0,0 +1,112 @@
+#include "search/search_quality/assessment_tool/feature_info_dialog.hpp"
+
+#include "indexer/classificator.hpp"
+#include "indexer/map_object.hpp"
+
+#include "coding/string_utf8_multilang.hpp"
+
+#include "geometry/latlon.hpp"
+
+#include "base/assert.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <vector>
+
+#include <QtCore/QString>
+#include <QtGui/QAction>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QPushButton>
+
+using namespace std;
+
+namespace
+{
+QLabel * MakeSelectableLabel(string const & s)
+{
+  auto * result = new QLabel(QString::fromStdString(s));
+  result->setTextInteractionFlags(Qt::TextSelectableByMouse);
+  return result;
+}
+}  // namespace
+
+FeatureInfoDialog::FeatureInfoDialog(QWidget * parent, osm::MapObject const & mapObject,
+                                     search::ReverseGeocoder::Address const & address, string const & locale)
+  : QDialog(parent)
+{
+  auto * layout = new QGridLayout();
+
+  {
+    auto const & id = mapObject.GetID();
+    CHECK(id.IsValid(), ());
+
+    auto * label = new QLabel("id:");
+    auto * content = MakeSelectableLabel(id.GetMwmName() + ", " + strings::to_string(id.m_index));
+
+    AddRow(*layout, label, content);
+  }
+
+  {
+    auto * label = new QLabel("lat lon:");
+    auto const ll = mapObject.GetLatLon();
+    auto const ss = strings::to_string_dac(ll.m_lat, 5) + " " + strings::to_string_dac(ll.m_lon, 5);
+    auto * content = MakeSelectableLabel(ss);
+
+    AddRow(*layout, label, content);
+  }
+
+  {
+    int8_t const localeCode = StringUtf8Multilang::GetLangIndex(locale);
+    vector<int8_t> codes = {{StringUtf8Multilang::kDefaultCode, StringUtf8Multilang::kEnglishCode}};
+    if (localeCode != StringUtf8Multilang::kUnsupportedLanguageCode &&
+        ::find(codes.begin(), codes.end(), localeCode) == codes.end())
+    {
+      codes.push_back(localeCode);
+    }
+
+    for (auto const & code : codes)
+    {
+      string_view name;
+      if (!mapObject.GetNameMultilang().GetString(code, name))
+        continue;
+
+      auto const lang = StringUtf8Multilang::GetLangByCode(code);
+      CHECK(!lang.empty(), ("Can't find lang by code:", code));
+      auto * label = new QLabel(QString::fromStdString(std::string{lang} + ":"));
+      auto * content = MakeSelectableLabel(std::string{name});
+
+      AddRow(*layout, label, content);
+    }
+  }
+
+  {
+    auto const & c = classif();
+
+    vector<string> types;
+    for (auto type : mapObject.GetTypes())
+      types.push_back(c.GetReadableObjectName(type));
+
+    if (!types.empty())
+    {
+      auto * label = new QLabel("types:");
+      auto * content = MakeSelectableLabel(strings::JoinStrings(types, " " /* delimiter */));
+
+      AddRow(*layout, label, content);
+    }
+  }
+
+  {
+    auto * label = new QLabel("address:");
+    auto * content = MakeSelectableLabel(address.FormatAddress());
+
+    AddRow(*layout, label, content);
+  }
+
+  auto * ok = new QPushButton("OK");
+  ok->setDefault(true);
+  connect(ok, &QPushButton::clicked, [&]() { accept(); });
+  AddRow(*layout, ok);
+
+  setLayout(layout);
+}

libs/search/search_quality/assessment_tool/feature_info_dialog.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "search/reverse_geocoder.hpp"
+
+#include <string>
+
+#include <QtWidgets/QDialog>
+#include <QtWidgets/QGridLayout>
+
+namespace osm
+{
+class MapObject;
+}  // namespace osm
+
+class FeatureInfoDialog : public QDialog
+{
+  Q_OBJECT
+
+public:
+  FeatureInfoDialog(QWidget * parent, osm::MapObject const & mapObject,
+                    search::ReverseGeocoder::Address const & address, std::string const & locale);
+
+private:
+  void AddElems(QGridLayout & layout, int row, int col, QWidget * widget) { layout.addWidget(widget, row, col); }
+
+  template <typename... Args>
+  void AddElems(QGridLayout & layout, int row, int col, QWidget * widget, Args *... args)
+  {
+    AddElems(layout, row, col, widget);
+    AddElems(layout, row, col + 1, args...);
+  }
+
+  template <typename... Args>
+  void AddRow(QGridLayout & layout, Args *... args)
+  {
+    AddElems(layout, m_numRows /* row */, 0 /* col */, args...);
+    ++m_numRows;
+  }
+
+  int m_numRows = 0;
+};

libs/search/search_quality/assessment_tool/helpers.cpp

@@ -0,0 +1,13 @@
+#include "search/search_quality/assessment_tool/helpers.hpp"
+
+#include "base/string_utils.hpp"
+
+QString ToQString(strings::UniString const & s)
+{
+  return QString::fromUtf8(strings::ToUtf8(s).c_str());
+}
+
+QString ToQString(std::string const & s)
+{
+  return QString::fromUtf8(s.c_str());
+}

libs/search/search_quality/assessment_tool/helpers.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <string>
+#include <utility>
+
+#include <QtCore/QString>
+
+namespace strings
+{
+class UniString;
+}
+
+QString ToQString(strings::UniString const & s);
+QString ToQString(std::string const & s);
+
+template <typename Layout, typename... Args>
+Layout * BuildLayout(Args &&... args)
+{
+  return new Layout(std::forward<Args>(args)...);
+}
+
+template <typename Layout, typename... Args>
+Layout * BuildLayoutWithoutMargins(Args &&... args)
+{
+  auto * layout = BuildLayout<Layout>(std::forward<Args>(args)...);
+  layout->setContentsMargins(0 /* left */, 0 /* top */, 0 /* right */, 0 /* bottom */);
+  return layout;
+}

libs/search/search_quality/assessment_tool/main_model.cpp

@@ -0,0 +1,387 @@
+#include "main_model.hpp"
+#include "view.hpp"
+
+#include "search/feature_loader.hpp"
+#include "search/search_params.hpp"
+#include "search/search_quality/helpers.hpp"
+#include "search/search_quality/matcher.hpp"
+
+#include "map/framework.hpp"
+
+#include "geometry/algorithm.hpp"
+#include "geometry/mercator.hpp"
+
+#include "platform/platform.hpp"
+
+#include "base/assert.hpp"
+
+#include <algorithm>
+#include <fstream>
+#include <iterator>
+
+using namespace std;
+
+// MainModel ---------------------------------------------------------------------------------------
+MainModel::MainModel(Framework & framework)
+  : m_framework(framework)
+  , m_dataSource(m_framework.GetDataSource())
+  , m_loader(m_dataSource)
+  , m_contexts([this](size_t sampleIndex, ResultsEdits::Update const & update)
+{ OnUpdate(View::ResultType::Found, sampleIndex, update); },
+               [this](size_t sampleIndex, ResultsEdits::Update const & update)
+{ OnUpdate(View::ResultType::NonFound, sampleIndex, update); },
+               [this](size_t sampleIndex) { OnSampleUpdate(sampleIndex); })
+  , m_runner(m_framework, m_dataSource, m_contexts,
+             [this](search::Results const & results) { UpdateViewOnResults(results); }, [this](size_t index)
+{
+  // Only the first parameter matters because we only change SearchStatus.
+  m_view->OnSampleChanged(index, false /* isUseless */, false /* hasEdits */);
+})
+{
+  search::search_quality::CheckLocale();
+}
+
+void MainModel::Open(string const & path)
+{
+  CHECK(m_view, ());
+
+  string contents;
+
+  {
+    ifstream ifs(path);
+    if (!ifs)
+    {
+      m_view->ShowError("Can't open file: " + path);
+      return;
+    }
+    contents.assign(istreambuf_iterator<char>(ifs), istreambuf_iterator<char>());
+  }
+
+  vector<search::Sample> samples;
+  if (!search::Sample::DeserializeFromJSONLines(contents, samples))
+  {
+    m_view->ShowError("Can't parse samples: " + path);
+    return;
+  }
+
+  m_runner.ResetForegroundSearch();
+  m_runner.ResetBackgroundSearch();
+
+  m_view->Clear();
+
+  m_contexts.Resize(samples.size());
+  for (size_t i = 0; i < samples.size(); ++i)
+    m_contexts[i].LoadFromSample(samples[i]);
+
+  m_path = path;
+
+  m_view->SetSamples(ContextList::SamplesSlice(m_contexts));
+  m_selectedSample = kInvalidIndex;
+}
+
+void MainModel::Save()
+{
+  CHECK(HasChanges(), ());
+  SaveAs(m_path);
+}
+
+void MainModel::SaveAs(string const & path)
+{
+  CHECK(HasChanges(), ());
+  CHECK(!path.empty(), ());
+
+  string contents;
+  search::Sample::SerializeToJSONLines(m_contexts.MakeSamples(m_loader), contents);
+
+  {
+    ofstream ofs(path);
+    if (!ofs)
+    {
+      m_view->ShowError("Can't open file: " + path);
+      return;
+    }
+    copy(contents.begin(), contents.end(), ostreambuf_iterator<char>(ofs));
+  }
+
+  m_contexts.ApplyEdits();
+  m_path = path;
+}
+
+void MainModel::InitiateBackgroundSearch(size_t from, size_t to)
+{
+  m_runner.InitiateBackgroundSearch(from, to);
+}
+
+void MainModel::OnSampleSelected(int index)
+{
+  CHECK(m_threadChecker.CalledOnOriginalThread(), ());
+
+  CHECK_GREATER_OR_EQUAL(index, 0, ());
+  CHECK_LESS(static_cast<size_t>(index), m_contexts.Size(), ());
+  CHECK(m_view, ());
+
+  m_selectedSample = index;
+
+  auto & context = m_contexts[index];
+  auto const & sample = context.m_sample;
+
+  m_view->ShowSample(index, sample, sample.m_pos, context.IsUseless(), context.HasChanges());
+
+  m_runner.ResetForegroundSearch();
+
+  if (context.m_initialized)
+  {
+    UpdateViewOnResults(context.m_foundResults);
+    return;
+  }
+
+  InitiateForegroundSearch(index);
+}
+
+void MainModel::OnResultSelected(int index)
+{
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+  auto const & context = m_contexts[m_selectedSample];
+  auto const & foundResults = context.m_foundResults;
+
+  CHECK_GREATER_OR_EQUAL(index, 0, ());
+  CHECK_LESS(static_cast<size_t>(index), foundResults.GetCount(), ());
+  m_view->MoveViewportToResult(foundResults[index]);
+}
+
+void MainModel::OnNonFoundResultSelected(int index)
+{
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+  auto const & context = m_contexts[m_selectedSample];
+  auto const & results = context.m_nonFoundResults;
+
+  CHECK_GREATER_OR_EQUAL(index, 0, ());
+  CHECK_LESS(static_cast<size_t>(index), results.size(), ());
+  m_view->MoveViewportToResult(results[index]);
+}
+
+void MainModel::OnShowViewportClicked()
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  auto const & context = m_contexts[m_selectedSample];
+  m_view->MoveViewportToRect(context.m_sample.m_viewport);
+}
+
+void MainModel::OnShowPositionClicked()
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  static int constexpr kViewportAroundTopResultsSizeM = 100;
+  static double constexpr kViewportAroundTopResultsScale = 1.2;
+  static size_t constexpr kMaxTopResults = 3;
+
+  auto const & context = m_contexts[m_selectedSample];
+
+  vector<m2::PointD> points;
+  if (context.m_sample.m_pos)
+    points.push_back(*context.m_sample.m_pos);
+
+  size_t resultsAdded = 0;
+  for (auto const & result : context.m_foundResults)
+  {
+    if (!result.HasPoint())
+      continue;
+
+    if (resultsAdded == kMaxTopResults)
+      break;
+
+    points.push_back(result.GetFeatureCenter());
+    ++resultsAdded;
+  }
+
+  CHECK(!points.empty(), ());
+  auto boundingBox = m2::ApplyCalculator(points, m2::CalculateBoundingBox());
+  boundingBox.Scale(kViewportAroundTopResultsScale);
+
+  auto const minRect = mercator::RectByCenterXYAndSizeInMeters(boundingBox.Center(), kViewportAroundTopResultsSizeM);
+  m_view->MoveViewportToRect(m2::Add(boundingBox, minRect));
+}
+
+void MainModel::OnMarkAllAsRelevantClicked()
+{
+  OnChangeAllRelevancesClicked(ResultsEdits::Relevance::Relevant);
+}
+
+void MainModel::OnMarkAllAsIrrelevantClicked()
+{
+  OnChangeAllRelevancesClicked(ResultsEdits::Relevance::Irrelevant);
+}
+
+bool MainModel::HasChanges()
+{
+  return m_contexts.HasChanges();
+}
+
+bool MainModel::AlreadyInSamples(FeatureID const & id)
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  bool found = false;
+  ForAnyMatchingEntry(m_contexts[m_selectedSample], id, [&](ResultsEdits & edits, size_t index)
+  {
+    auto const & entry = edits.GetEntry(index);
+    if (!entry.m_deleted)
+      found = true;
+  });
+  return found;
+}
+
+void MainModel::AddNonFoundResult(FeatureID const & id)
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  auto & context = m_contexts[m_selectedSample];
+
+  bool resurrected = false;
+  ForAnyMatchingEntry(context, id, [&](ResultsEdits & edits, size_t index)
+  {
+    auto const & entry = edits.GetEntry(index);
+    CHECK(entry.m_deleted, ());
+    edits.Resurrect(index);
+    resurrected = true;
+  });
+  if (resurrected)
+    return;
+
+  auto ft = m_loader.Load(id);
+  CHECK(ft, ("Can't load feature:", id));
+  auto const result = search::Sample::Result::Build(*ft, search::Sample::Result::Relevance::Vital);
+  context.AddNonFoundResult(result);
+}
+
+void MainModel::FlipSampleUsefulness(int index)
+{
+  CHECK_EQUAL(m_selectedSample, index, ());
+
+  m_contexts[index].m_sampleEdits.FlipUsefulness();
+
+  // Don't bother with resetting search: we cannot tell whether
+  // the sample is useless without its results anyway.
+}
+
+void MainModel::InitiateForegroundSearch(size_t index)
+{
+  auto & context = m_contexts[index];
+  auto const & sample = context.m_sample;
+
+  m_view->ShowSample(index, sample, sample.m_pos, context.IsUseless(), context.HasChanges());
+  m_runner.InitiateForegroundSearch(index);
+  m_view->OnSearchStarted();
+}
+
+void MainModel::OnUpdate(View::ResultType type, size_t sampleIndex, ResultsEdits::Update const & update)
+{
+  using Type = ResultsEdits::Update::Type;
+
+  CHECK_LESS(sampleIndex, m_contexts.Size(), ());
+  auto & context = m_contexts[sampleIndex];
+
+  if (update.m_type == Type::Add)
+  {
+    CHECK_EQUAL(type, View::ResultType::NonFound, ());
+    m_view->ShowNonFoundResults(context.m_nonFoundResults, context.m_nonFoundResultsEdits.GetEntries());
+    m_view->SetResultsEdits(m_selectedSample, context.m_foundResultsEdits, context.m_nonFoundResultsEdits);
+  }
+
+  m_view->OnResultChanged(sampleIndex, type, update);
+  m_view->OnSampleChanged(sampleIndex, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+
+  if (update.m_type == Type::Add || update.m_type == Type::Resurrect || update.m_type == Type::Delete)
+  {
+    CHECK(context.m_initialized, ());
+    CHECK_EQUAL(type, View::ResultType::NonFound, ());
+    m_view->ShowMarks(context);
+  }
+}
+
+void MainModel::OnSampleUpdate(size_t sampleIndex)
+{
+  auto & context = m_contexts[sampleIndex];
+
+  m_view->OnSampleChanged(sampleIndex, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+}
+
+void MainModel::UpdateViewOnResults(search::Results const & results)
+{
+  CHECK(m_threadChecker.CalledOnOriginalThread(), ());
+
+  m_view->AddFoundResults(results);
+
+  if (!results.IsEndedNormal())
+    return;
+
+  auto & context = m_contexts[m_selectedSample];
+
+  m_view->ShowNonFoundResults(context.m_nonFoundResults, context.m_nonFoundResultsEdits.GetEntries());
+  m_view->ShowMarks(context);
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::Found, ResultsEdits::Update::MakeAll());
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::NonFound, ResultsEdits::Update::MakeAll());
+  m_view->OnSampleChanged(m_selectedSample, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+
+  m_view->SetResultsEdits(m_selectedSample, context.m_foundResultsEdits, context.m_nonFoundResultsEdits);
+  m_view->OnSearchCompleted();
+}
+
+void MainModel::OnChangeAllRelevancesClicked(ResultsEdits::Relevance relevance)
+{
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+  auto & context = m_contexts[m_selectedSample];
+
+  context.m_foundResultsEdits.SetAllRelevances(relevance);
+  context.m_nonFoundResultsEdits.SetAllRelevances(relevance);
+
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::Found, ResultsEdits::Update::MakeAll());
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::NonFound, ResultsEdits::Update::MakeAll());
+  m_view->OnSampleChanged(m_selectedSample, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+}
+
+template <typename Fn>
+void MainModel::ForAnyMatchingEntry(Context & context, FeatureID const & id, Fn && fn)
+{
+  CHECK(context.m_initialized, ());
+
+  auto const & foundResults = context.m_foundResults;
+  CHECK_EQUAL(foundResults.GetCount(), context.m_foundResultsEdits.NumEntries(), ());
+  for (size_t i = 0; i < foundResults.GetCount(); ++i)
+  {
+    auto const & result = foundResults[i];
+    if (result.GetResultType() != search::Result::Type::Feature)
+      continue;
+    if (result.GetFeatureID() == id)
+      return fn(context.m_foundResultsEdits, i);
+  }
+
+  auto ft = m_loader.Load(id);
+  CHECK(ft, ("Can't load feature:", id));
+  search::Matcher matcher(m_loader);
+
+  auto const & nonFoundResults = context.m_nonFoundResults;
+  CHECK_EQUAL(nonFoundResults.size(), context.m_nonFoundResultsEdits.NumEntries(), ());
+  for (size_t i = 0; i < nonFoundResults.size(); ++i)
+  {
+    auto const & result = context.m_nonFoundResults[i];
+    if (matcher.Matches(context.m_sample.m_query, result, *ft))
+      return fn(context.m_nonFoundResultsEdits, i);
+  }
+}

libs/search/search_quality/assessment_tool/main_model.hpp

@@ -0,0 +1,73 @@
+#pragma once
+#include "context.hpp"
+#include "edits.hpp"
+#include "model.hpp"
+#include "search_request_runner.hpp"
+#include "view.hpp"
+
+#include "search/feature_loader.hpp"
+
+#include "base/thread_checker.hpp"
+
+class Framework;
+class DataSource;
+
+namespace search
+{
+class Results;
+}
+
+class MainModel : public Model
+{
+public:
+  explicit MainModel(Framework & framework);
+
+  // Model overrides:
+  void Open(std::string const & path) override;
+  void Save() override;
+  void SaveAs(std::string const & path) override;
+  void InitiateBackgroundSearch(size_t const from, size_t const to) override;
+
+  void OnSampleSelected(int index) override;
+  void OnResultSelected(int index) override;
+  void OnNonFoundResultSelected(int index) override;
+  void OnShowViewportClicked() override;
+  void OnShowPositionClicked() override;
+  void OnMarkAllAsRelevantClicked() override;
+  void OnMarkAllAsIrrelevantClicked() override;
+  bool HasChanges() override;
+  bool AlreadyInSamples(FeatureID const & id) override;
+  void AddNonFoundResult(FeatureID const & id) override;
+  void FlipSampleUsefulness(int index) override;
+
+private:
+  static int constexpr kInvalidIndex = -1;
+
+  void InitiateForegroundSearch(size_t index);
+
+  void OnUpdate(View::ResultType type, size_t sampleIndex, ResultsEdits::Update const & update);
+  void OnSampleUpdate(size_t sampleIndex);
+
+  void UpdateViewOnResults(search::Results const & results);
+  void ShowMarks(Context const & context);
+
+  void OnChangeAllRelevancesClicked(ResultsEdits::Relevance relevance);
+
+  template <typename Fn>
+  void ForAnyMatchingEntry(Context & context, FeatureID const & id, Fn && fn);
+
+  Framework & m_framework;
+  DataSource const & m_dataSource;
+  search::FeatureLoader m_loader;
+
+  ContextList m_contexts;
+
+  // Path to the last file search samples were loaded from or saved to.
+  std::string m_path;
+
+  int m_selectedSample = kInvalidIndex;
+
+  SearchRequestRunner m_runner;
+
+  ThreadChecker m_threadChecker;
+};

libs/search/search_quality/assessment_tool/main_view.cpp

@@ -0,0 +1,516 @@
+#include "main_view.hpp"
+
+#include "feature_info_dialog.hpp"
+#include "helpers.hpp"
+#include "model.hpp"
+#include "results_view.hpp"
+#include "sample_view.hpp"
+#include "samples_view.hpp"
+
+#include "qt/qt_common/map_widget.hpp"
+#include "qt/qt_common/scale_slider.hpp"
+
+#include "map/framework.hpp"
+#include "map/place_page_info.hpp"
+
+#include "geometry/mercator.hpp"
+
+#include "base/assert.hpp"
+#include "base/checked_cast.hpp"
+#include "base/string_utils.hpp"
+
+#include <limits>
+
+#include <QtCore/Qt>
+#include <QtGui/QCloseEvent>
+#include <QtGui/QIntValidator>
+#include <QtGui/QKeySequence>
+#include <QtWidgets/QApplication>
+#include <QtWidgets/QDialog>
+#include <QtWidgets/QDialogButtonBox>
+#include <QtWidgets/QDockWidget>
+#include <QtWidgets/QFileDialog>
+#include <QtWidgets/QFormLayout>
+#include <QtWidgets/QHBoxLayout>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QLineEdit>
+#include <QtWidgets/QMenuBar>
+#include <QtWidgets/QMessageBox>
+#include <QtWidgets/QToolBar>
+
+using Relevance = search::Sample::Result::Relevance;
+
+namespace
+{
+char const kJSON[] = "JSON Lines files (*.jsonl)";
+}  // namespace
+
+MainView::MainView(Framework & framework, QRect const & screenGeometry) : m_framework(framework)
+{
+  setGeometry(screenGeometry);
+
+  setWindowTitle(tr("Assessment tool"));
+  InitMapWidget();
+  InitDocks();
+  InitMenuBar();
+
+  m_framework.SetPlacePageListeners([this]()
+  {
+    auto const & info = m_framework.GetCurrentPlacePageInfo();
+    auto const & selectedFeature = info.GetID();
+    if (!selectedFeature.IsValid())
+      return;
+    m_selectedFeature = selectedFeature;
+
+    if (m_skipFeatureInfoDialog)
+    {
+      m_skipFeatureInfoDialog = false;
+      return;
+    }
+
+    auto mapObject = m_framework.GetMapObjectByID(selectedFeature);
+    if (!mapObject.GetID().IsValid())
+      return;
+
+    auto const address = m_framework.GetAddressAtPoint(mapObject.GetMercator());
+    FeatureInfoDialog dialog(this /* parent */, mapObject, address, m_sampleLocale);
+    dialog.exec();
+  }, [this]() { m_selectedFeature = FeatureID(); }, {} /* onUpdate */, {} /* onSwitchFullScreenMode */);
+}
+
+MainView::~MainView()
+{
+  if (m_framework.GetDrapeEngine() != nullptr)
+  {
+    m_framework.EnterBackground();
+    m_framework.DestroyDrapeEngine();
+  }
+}
+
+void MainView::SetSamples(ContextList::SamplesSlice const & samples)
+{
+  m_samplesView->SetSamples(samples);
+  m_sampleView->Clear();
+  m_initiateBackgroundSearch->setEnabled(true);
+}
+
+void MainView::OnSearchStarted()
+{
+  m_state = State::Search;
+  m_sampleView->OnSearchStarted();
+}
+
+void MainView::OnSearchCompleted()
+{
+  m_state = State::AfterSearch;
+  m_sampleView->OnSearchCompleted();
+}
+
+void MainView::ShowSample(size_t sampleIndex, search::Sample const & sample, std::optional<m2::PointD> const & position,
+                          bool isUseless, bool hasEdits)
+{
+  m_sampleLocale = sample.m_locale;
+
+  MoveViewportToRect(sample.m_viewport);
+
+  m_sampleView->SetContents(sample, position);
+  m_sampleView->show();
+
+  OnResultChanged(sampleIndex, ResultType::Found, ResultsEdits::Update::MakeAll());
+  OnResultChanged(sampleIndex, ResultType::NonFound, ResultsEdits::Update::MakeAll());
+  OnSampleChanged(sampleIndex, isUseless, hasEdits);
+}
+
+void MainView::AddFoundResults(search::Results const & results)
+{
+  m_sampleView->AddFoundResults(results);
+}
+
+void MainView::ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                                   std::vector<ResultsEdits::Entry> const & entries)
+{
+  m_sampleView->ShowNonFoundResults(results, entries);
+}
+
+void MainView::ShowMarks(Context const & context)
+{
+  m_sampleView->ClearSearchResultMarks();
+  m_sampleView->ShowFoundResultsMarks(context.m_foundResults);
+  m_sampleView->ShowNonFoundResultsMarks(context.m_nonFoundResults, context.m_nonFoundResultsEdits.GetEntries());
+}
+
+void MainView::MoveViewportToResult(search::Result const & result)
+{
+  m_skipFeatureInfoDialog = true;
+  m_framework.SelectSearchResult(result, false /* animation */);
+}
+
+void MainView::MoveViewportToResult(search::Sample::Result const & result)
+{
+  int constexpr kViewportAroundResultSizeM = 100;
+  auto const rect = mercator::RectByCenterXYAndSizeInMeters(result.m_pos, kViewportAroundResultSizeM);
+  MoveViewportToRect(rect);
+}
+
+void MainView::MoveViewportToRect(m2::RectD const & rect)
+{
+  m_framework.ShowRect(rect, -1 /* maxScale */, false /* animation */);
+}
+
+void MainView::OnResultChanged(size_t sampleIndex, ResultType type, ResultsEdits::Update const & update)
+{
+  m_samplesView->OnUpdate(sampleIndex);
+
+  if (!m_samplesView->IsSelected(sampleIndex))
+    return;
+
+  switch (type)
+  {
+  case ResultType::Found: m_sampleView->GetFoundResultsView().Update(update); break;
+  case ResultType::NonFound: m_sampleView->GetNonFoundResultsView().Update(update); break;
+  }
+}
+
+void MainView::OnSampleChanged(size_t sampleIndex, bool isUseless, bool hasEdits)
+{
+  m_samplesView->OnUpdate(sampleIndex);
+  if (!m_samplesView->IsSelected(sampleIndex))
+    return;
+  SetSampleDockTitle(isUseless, hasEdits);
+  m_sampleView->OnUselessnessChanged(isUseless);
+}
+
+void MainView::OnSamplesChanged(bool hasEdits)
+{
+  SetSamplesDockTitle(hasEdits);
+  m_save->setEnabled(hasEdits);
+  m_saveAs->setEnabled(hasEdits);
+}
+
+void MainView::SetResultsEdits(size_t sampleIndex, ResultsEdits & foundResultsEdits,
+                               ResultsEdits & nonFoundResultsEdits)
+{
+  CHECK(m_samplesView->IsSelected(sampleIndex), ());
+  m_sampleView->SetResultsEdits(foundResultsEdits, nonFoundResultsEdits);
+}
+
+void MainView::ShowError(std::string const & msg)
+{
+  QMessageBox box(QMessageBox::Critical /* icon */, tr("Error") /* title */, QString::fromStdString(msg) /* text */,
+                  QMessageBox::Ok /* buttons */, this /* parent */);
+  box.exec();
+}
+
+void MainView::Clear()
+{
+  m_samplesView->Clear();
+  SetSamplesDockTitle(false /* hasEdits */);
+
+  m_sampleView->Clear();
+  SetSampleDockTitle(false /* isUseless */, false /* hasEdits */);
+
+  m_skipFeatureInfoDialog = false;
+  m_sampleLocale.clear();
+}
+
+void MainView::closeEvent(QCloseEvent * event)
+{
+  if (TryToSaveEdits(tr("Save changes before closing?")) == SaveResult::Cancelled)
+    event->ignore();
+  else
+    event->accept();
+}
+
+void MainView::OnSampleSelected(QItemSelection const & current)
+{
+  CHECK(m_model, ());
+  auto const indexes = current.indexes();
+  for (auto const & index : indexes)
+    m_model->OnSampleSelected(index.row());
+}
+
+void MainView::OnResultSelected(QItemSelection const & current)
+{
+  CHECK(m_model, ());
+  auto const indexes = current.indexes();
+  for (auto const & index : indexes)
+    m_model->OnResultSelected(index.row());
+}
+
+void MainView::OnNonFoundResultSelected(QItemSelection const & current)
+{
+  CHECK(m_model, ());
+  auto const indexes = current.indexes();
+  for (auto const & index : indexes)
+    m_model->OnNonFoundResultSelected(index.row());
+}
+
+void MainView::InitMenuBar()
+{
+  auto * bar = menuBar();
+
+  auto * fileMenu = bar->addMenu(tr("&File"));
+
+  {
+    auto * open = new QAction(tr("&Open samples..."), this /* parent */);
+    open->setShortcuts(QKeySequence::Open);
+    open->setStatusTip(tr("Open the file with samples for assessment"));
+    connect(open, &QAction::triggered, this, &MainView::Open);
+    fileMenu->addAction(open);
+  }
+
+  {
+    m_save = new QAction(tr("Save samples"), this /* parent */);
+    m_save->setShortcuts(QKeySequence::Save);
+    m_save->setStatusTip(tr("Save the file with assessed samples"));
+    m_save->setEnabled(false);
+    connect(m_save, &QAction::triggered, this, &MainView::Save);
+    fileMenu->addAction(m_save);
+  }
+
+  {
+    m_saveAs = new QAction(tr("Save samples as..."), this /* parent */);
+    m_saveAs->setShortcuts(QKeySequence::SaveAs);
+    m_saveAs->setStatusTip(tr("Save the file with assessed samples"));
+    m_saveAs->setEnabled(false);
+    connect(m_saveAs, &QAction::triggered, this, &MainView::SaveAs);
+    fileMenu->addAction(m_saveAs);
+  }
+
+  {
+    m_initiateBackgroundSearch = new QAction(tr("Initiate background search"), this /* parent */);
+    m_initiateBackgroundSearch->setShortcut(static_cast<int>(Qt::CTRL) | static_cast<int>(Qt::Key_I));
+    m_initiateBackgroundSearch->setStatusTip(tr("Search in the background for the queries from a selected range"));
+    m_initiateBackgroundSearch->setEnabled(false);
+    connect(m_initiateBackgroundSearch, &QAction::triggered, this, &MainView::InitiateBackgroundSearch);
+    fileMenu->addAction(m_initiateBackgroundSearch);
+  }
+
+  fileMenu->addSeparator();
+
+  {
+    auto * quit = new QAction(tr("&Quit"), this /* parent */);
+    quit->setShortcuts(QKeySequence::Quit);
+    quit->setStatusTip(tr("Exit the tool"));
+    connect(quit, &QAction::triggered, this, &QWidget::close);
+    fileMenu->addAction(quit);
+  }
+
+  auto * viewMenu = bar->addMenu(tr("&View"));
+
+  {
+    CHECK(m_samplesDock != nullptr, ());
+    viewMenu->addAction(m_samplesDock->toggleViewAction());
+  }
+
+  {
+    CHECK(m_sampleDock != nullptr, ());
+    viewMenu->addAction(m_sampleDock->toggleViewAction());
+  }
+}
+
+void MainView::InitMapWidget()
+{
+  auto * widget = new QWidget(this /* parent */);
+  auto * layout = BuildLayoutWithoutMargins<QHBoxLayout>(widget /* parent */);
+  widget->setLayout(layout);
+
+  {
+    auto * mapWidget = new qt::common::MapWidget(m_framework, false /* screenshotMode */, widget /* parent */);
+    connect(mapWidget, &qt::common::MapWidget::OnContextMenuRequested,
+            [this](QPoint const & p) { AddSelectedFeature(p); });
+    auto * toolBar = new QToolBar(widget /* parent */);
+    toolBar->setOrientation(Qt::Vertical);
+    toolBar->setIconSize(QSize(32, 32));
+    qt::common::ScaleSlider::Embed(Qt::Vertical, *toolBar, *mapWidget);
+
+    layout->addWidget(mapWidget);
+    layout->addWidget(toolBar);
+  }
+
+  setCentralWidget(widget);
+}
+
+void MainView::InitDocks()
+{
+  m_samplesView = new SamplesView(this /* parent */);
+
+  {
+    auto * model = m_samplesView->selectionModel();
+    connect(model, &QItemSelectionModel::selectionChanged, this, &MainView::OnSampleSelected);
+  }
+
+  connect(m_samplesView, &SamplesView::FlipSampleUsefulness,
+          [this](int index) { m_model->FlipSampleUsefulness(index); });
+
+  m_samplesDock = CreateDock(*m_samplesView);
+  addDockWidget(Qt::LeftDockWidgetArea, m_samplesDock);
+  SetSamplesDockTitle(false /* hasEdits */);
+
+  m_sampleView = new SampleView(this /* parent */, m_framework);
+
+  connect(m_sampleView, &SampleView::OnShowViewportClicked, [this]() { m_model->OnShowViewportClicked(); });
+  connect(m_sampleView, &SampleView::OnShowPositionClicked, [this]() { m_model->OnShowPositionClicked(); });
+
+  connect(m_sampleView, &SampleView::OnMarkAllAsRelevantClicked, [this]() { m_model->OnMarkAllAsRelevantClicked(); });
+  connect(m_sampleView, &SampleView::OnMarkAllAsIrrelevantClicked,
+          [this]() { m_model->OnMarkAllAsIrrelevantClicked(); });
+
+  {
+    auto const & view = m_sampleView->GetFoundResultsView();
+    connect(&view, &ResultsView::OnResultSelected, [this](int index) { m_model->OnResultSelected(index); });
+  }
+
+  {
+    auto const & view = m_sampleView->GetNonFoundResultsView();
+    connect(&view, &ResultsView::OnResultSelected, [this](int index) { m_model->OnNonFoundResultSelected(index); });
+  }
+
+  m_sampleDock = CreateDock(*m_sampleView);
+  connect(m_sampleDock, &QDockWidget::dockLocationChanged,
+          [this](Qt::DockWidgetArea area) { m_sampleView->OnLocationChanged(area); });
+  addDockWidget(Qt::RightDockWidgetArea, m_sampleDock);
+  SetSampleDockTitle(false /* isUseless */, false /* hasEdits */);
+}
+
+void MainView::Open()
+{
+  CHECK(m_model, ());
+
+  if (TryToSaveEdits(tr("Save changes before opening samples?")) == SaveResult::Cancelled)
+    return;
+
+  auto const name = QFileDialog::getOpenFileName(this /* parent */, tr("Open samples..."), QString() /* dir */, kJSON);
+  auto const file = name.toStdString();
+  if (file.empty())
+    return;
+
+  m_model->Open(file);
+}
+
+void MainView::Save()
+{
+  m_model->Save();
+}
+
+void MainView::SaveAs()
+{
+  auto const name =
+      QFileDialog::getSaveFileName(this /* parent */, tr("Save samples as..."), QString() /* dir */, kJSON);
+  auto const file = name.toStdString();
+  if (!file.empty())
+    m_model->SaveAs(file);
+}
+
+void MainView::InitiateBackgroundSearch()
+{
+  QDialog dialog(this);
+  QFormLayout form(&dialog);
+
+  form.addRow(new QLabel("Queries range"));
+
+  QValidator * validator = new QIntValidator(0, std::numeric_limits<int>::max(), this);
+
+  QLineEdit * lineEditFrom = new QLineEdit(&dialog);
+  form.addRow(new QLabel("First"), lineEditFrom);
+  lineEditFrom->setValidator(validator);
+
+  QLineEdit * lineEditTo = new QLineEdit(&dialog);
+  form.addRow(new QLabel("Last"), lineEditTo);
+  lineEditTo->setValidator(validator);
+
+  QDialogButtonBox buttonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel, Qt::Horizontal, &dialog);
+  form.addRow(&buttonBox);
+
+  connect(&buttonBox, &QDialogButtonBox::accepted, &dialog, &QDialog::accept);
+  connect(&buttonBox, &QDialogButtonBox::rejected, &dialog, &QDialog::reject);
+
+  if (dialog.exec() != QDialog::Accepted)
+    return;
+
+  std::string const strFrom = lineEditFrom->text().toStdString();
+  std::string const strTo = lineEditTo->text().toStdString();
+  uint64_t from = 0;
+  uint64_t to = 0;
+  if (!strings::to_uint64(strFrom, from))
+  {
+    LOG(LERROR, ("Could not parse number from", strFrom));
+    return;
+  }
+  if (!strings::to_uint64(strTo, to))
+  {
+    LOG(LERROR, ("Could not parse number from", strTo));
+    return;
+  }
+
+  m_model->InitiateBackgroundSearch(base::checked_cast<size_t>(from), base::checked_cast<size_t>(to));
+}
+
+void MainView::SetSamplesDockTitle(bool hasEdits)
+{
+  CHECK(m_samplesDock, ());
+  if (hasEdits)
+    m_samplesDock->setWindowTitle(tr("Samples *"));
+  else
+    m_samplesDock->setWindowTitle(tr("Samples"));
+}
+
+void MainView::SetSampleDockTitle(bool isUseless, bool hasEdits)
+{
+  CHECK(m_sampleDock, ());
+  std::string title = "Sample";
+  if (hasEdits)
+    title += " *";
+  if (isUseless)
+    title += " (useless)";
+  m_sampleDock->setWindowTitle(tr(title.data()));
+}
+
+MainView::SaveResult MainView::TryToSaveEdits(QString const & msg)
+{
+  CHECK(m_model, ());
+
+  if (!m_model->HasChanges())
+    return SaveResult::NoEdits;
+
+  QMessageBox box(QMessageBox::Question /* icon */, tr("Save edits?") /* title */, msg /* text */,
+                  QMessageBox::Save | QMessageBox::Discard | QMessageBox::Cancel /* buttons */, this /* parent */);
+  auto const button = box.exec();
+  switch (button)
+  {
+  case QMessageBox::Save: Save(); return SaveResult::Saved;
+  case QMessageBox::Discard: return SaveResult::Discarded;
+  case QMessageBox::Cancel: return SaveResult::Cancelled;
+  }
+
+  CHECK(false, ());
+  return SaveResult::Cancelled;
+}
+
+void MainView::AddSelectedFeature(QPoint const & p)
+{
+  auto const selectedFeature = m_selectedFeature;
+
+  if (!selectedFeature.IsValid())
+    return;
+
+  if (m_state != State::AfterSearch)
+    return;
+
+  if (m_model->AlreadyInSamples(selectedFeature))
+    return;
+
+  QMenu menu;
+  auto const * action = menu.addAction("Add to non-found results");
+  connect(action, &QAction::triggered, [this, selectedFeature]() { m_model->AddNonFoundResult(selectedFeature); });
+  menu.exec(p);
+}
+
+QDockWidget * MainView::CreateDock(QWidget & widget)
+{
+  auto * dock = new QDockWidget(QString(), this /* parent */, Qt::Widget);
+  dock->setFeatures(QDockWidget::DockWidgetClosable | QDockWidget::DockWidgetMovable |
+                    QDockWidget::DockWidgetFloatable);
+  dock->setWidget(&widget);
+  return dock;
+}

libs/search/search_quality/assessment_tool/main_view.hpp

@@ -0,0 +1,128 @@
+#pragma once
+#include "view.hpp"
+
+#include "indexer/feature_decl.hpp"
+
+#include <QtWidgets/QMainWindow>
+
+class Framework;
+class QDockWidget;
+class QItemSelection;
+class SamplesView;
+class SampleView;
+
+namespace qt
+{
+namespace common
+{
+class MapWidget;
+}
+}  // namespace qt
+
+class MainView
+  : public QMainWindow
+  , public View
+{
+  Q_OBJECT
+
+public:
+  explicit MainView(Framework & framework, QRect const & screenGeometry);
+  ~MainView() override;
+
+  // View overrides:
+  void SetSamples(ContextList::SamplesSlice const & samples) override;
+  void OnSearchStarted() override;
+  void OnSearchCompleted() override;
+  void ShowSample(size_t sampleIndex, search::Sample const & sample, std::optional<m2::PointD> const & position,
+                  bool isUseless, bool hasEdits) override;
+
+  void AddFoundResults(search::Results const & results) override;
+  void ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                           std::vector<ResultsEdits::Entry> const & entries) override;
+
+  void ShowMarks(Context const & context) override;
+
+  void MoveViewportToResult(search::Result const & result) override;
+  void MoveViewportToResult(search::Sample::Result const & result) override;
+  void MoveViewportToRect(m2::RectD const & rect) override;
+
+  void OnResultChanged(size_t sampleIndex, ResultType type, ResultsEdits::Update const & update) override;
+  void SetResultsEdits(size_t sampleIndex, ResultsEdits & foundResultsResultsEdits,
+                       ResultsEdits & nonFoundResultsResultsEdits) override;
+  void OnSampleChanged(size_t sampleIndex, bool isUseless, bool hasEdits) override;
+  void OnSamplesChanged(bool hasEdits) override;
+
+  void ShowError(std::string const & msg) override;
+
+  void Clear() override;
+
+protected:
+  // QMainWindow overrides:
+  void closeEvent(QCloseEvent * event) override;
+
+private slots:
+  void OnSampleSelected(QItemSelection const & current);
+  void OnResultSelected(QItemSelection const & current);
+  void OnNonFoundResultSelected(QItemSelection const & current);
+
+private:
+  enum class State
+  {
+    BeforeSearch,
+    Search,
+    AfterSearch
+  };
+
+  friend std::string DebugPrint(State state)
+  {
+    switch (state)
+    {
+    case State::BeforeSearch: return "BeforeSearch";
+    case State::Search: return "Search";
+    case State::AfterSearch: return "AfterSearch";
+    }
+  }
+
+  enum class SaveResult
+  {
+    NoEdits,
+    Saved,
+    Discarded,
+    Cancelled
+  };
+
+  void InitMapWidget();
+  void InitDocks();
+  void InitMenuBar();
+
+  void Open();
+  void Save();
+  void SaveAs();
+  void InitiateBackgroundSearch();
+
+  void SetSamplesDockTitle(bool hasEdits);
+  void SetSampleDockTitle(bool isUseless, bool hasEdits);
+  SaveResult TryToSaveEdits(QString const & msg);
+
+  void AddSelectedFeature(QPoint const & p);
+
+  QDockWidget * CreateDock(QWidget & widget);
+
+  Framework & m_framework;
+
+  SamplesView * m_samplesView = nullptr;
+  QDockWidget * m_samplesDock = nullptr;
+
+  SampleView * m_sampleView = nullptr;
+  QDockWidget * m_sampleDock = nullptr;
+
+  QAction * m_save = nullptr;
+  QAction * m_saveAs = nullptr;
+  QAction * m_initiateBackgroundSearch = nullptr;
+
+  State m_state = State::BeforeSearch;
+  FeatureID m_selectedFeature;
+
+  bool m_skipFeatureInfoDialog = false;
+  std::string m_sampleLocale;
+};

libs/search/search_quality/assessment_tool/model.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <string>
+
+class View;
+struct FeatureID;
+
+class Model
+{
+public:
+  virtual ~Model() = default;
+
+  void SetView(View & view) { m_view = &view; }
+
+  virtual void Open(std::string const & path) = 0;
+  virtual void Save() = 0;
+  virtual void SaveAs(std::string const & path) = 0;
+
+  // Initiates the search in the background on all samples
+  // in the 1-based range [|from|, |to|], both ends included.
+  // Another background search that may currently be running will be cancelled
+  // but the results for already completed requests will not be discarded.
+  //
+  // Does nothing if the range is invalid.
+  virtual void InitiateBackgroundSearch(size_t from, size_t to) = 0;
+
+  virtual void OnSampleSelected(int index) = 0;
+  virtual void OnResultSelected(int index) = 0;
+  virtual void OnNonFoundResultSelected(int index) = 0;
+  virtual void OnShowViewportClicked() = 0;
+  virtual void OnShowPositionClicked() = 0;
+  virtual void OnMarkAllAsRelevantClicked() = 0;
+  virtual void OnMarkAllAsIrrelevantClicked() = 0;
+  virtual bool HasChanges() = 0;
+
+  virtual bool AlreadyInSamples(FeatureID const & id) = 0;
+  virtual void AddNonFoundResult(FeatureID const & id) = 0;
+  virtual void FlipSampleUsefulness(int index) = 0;
+
+protected:
+  View * m_view = nullptr;
+};

libs/search/search_quality/assessment_tool/result_view.cpp

@@ -0,0 +1,185 @@
+#include "search/search_quality/assessment_tool/result_view.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/assessment_tool/helpers.hpp"
+
+#include <memory>
+#include <utility>
+
+#include <QtWidgets/QHBoxLayout>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QRadioButton>
+#include <QtWidgets/QVBoxLayout>
+
+using namespace std;
+
+namespace
+{
+QLabel * CreateLabel(QWidget & parent)
+{
+  QLabel * label = new QLabel(&parent);
+  label->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Minimum);
+  label->setWordWrap(true);
+  return label;
+}
+
+void SetText(QLabel & label, string const & text)
+{
+  if (text.empty())
+  {
+    label.hide();
+    return;
+  }
+
+  label.setText(ToQString(text));
+  label.show();
+}
+
+string GetResultType(search::Sample::Result const & result)
+{
+  return strings::JoinStrings(result.m_types, ", ");
+}
+
+string GetResultType(search::Result const & result)
+{
+  return (result.GetResultType() == search::Result::Type::Feature ? result.GetLocalizedFeatureType() : "");
+}
+}  // namespace
+
+ResultView::ResultView(string const & name, string const & type, string const & address, QWidget & parent)
+  : QWidget(&parent)
+{
+  Init();
+  SetContents(name, type, address);
+  setEnabled(false);
+  setObjectName("result");
+}
+
+ResultView::ResultView(search::Result const & result, QWidget & parent)
+  : ResultView(result.GetString(), GetResultType(result), result.GetAddress(), parent)
+{}
+
+ResultView::ResultView(search::Sample::Result const & result, QWidget & parent)
+  : ResultView(strings::ToUtf8(result.m_name), GetResultType(result), {} /* address */, parent)
+{}
+
+void ResultView::SetEditor(ResultsEdits::Editor && editor)
+{
+  m_editor = make_unique<ResultsEdits::Editor>(std::move(editor));
+
+  UpdateRelevanceRadioButtons();
+
+  setEnabled(true);
+}
+
+void ResultView::Update()
+{
+  if (!m_editor)
+  {
+    setStyleSheet("");
+    return;
+  }
+
+  if (m_editor->GetType() == ResultsEdits::Entry::Type::Created)
+    setStyleSheet("#result {background: rgba(173, 223, 173, 50%)}");
+  else if (m_editor->HasChanges())
+    setStyleSheet("#result {background: rgba(255, 255, 200, 50%)}");
+  else
+    setStyleSheet("");
+
+  UpdateRelevanceRadioButtons();
+}
+
+void ResultView::Init()
+{
+  auto * layout = new QVBoxLayout(this /* parent */);
+  layout->setSizeConstraint(QLayout::SetMinimumSize);
+  setLayout(layout);
+
+  m_name = CreateLabel(*this /* parent */);
+  layout->addWidget(m_name);
+
+  m_type = CreateLabel(*this /* parent */);
+  m_type->setStyleSheet("QLabel { font-size : 8pt }");
+  layout->addWidget(m_type);
+
+  m_address = CreateLabel(*this /* parent */);
+  m_address->setStyleSheet("QLabel { color : green ; font-size : 8pt }");
+  layout->addWidget(m_address);
+
+  {
+    auto * group = new QWidget(this /* parent */);
+    auto * groupLayout = new QHBoxLayout(group /* parent */);
+    group->setLayout(groupLayout);
+
+    m_harmful = CreateRatioButton("Harmful", *groupLayout);
+    m_irrelevant = CreateRatioButton("Irrelevant", *groupLayout);
+    m_relevant = CreateRatioButton("Relevant", *groupLayout);
+    m_vital = CreateRatioButton("Vital", *groupLayout);
+
+    layout->addWidget(group);
+  }
+}
+
+void ResultView::SetContents(string const & name, string const & type, string const & address)
+{
+  SetText(*m_name, name);
+  SetText(*m_type, type);
+  SetText(*m_address, address);
+
+  m_harmful->setChecked(false);
+  m_irrelevant->setChecked(false);
+  m_relevant->setChecked(false);
+  m_vital->setChecked(false);
+}
+
+QRadioButton * ResultView::CreateRatioButton(string const & text, QLayout & layout)
+{
+  QRadioButton * radio = new QRadioButton(QString::fromStdString(text), this /* parent */);
+  layout.addWidget(radio);
+
+  connect(radio, &QRadioButton::toggled, this, &ResultView::OnRelevanceChanged);
+  return radio;
+}
+
+void ResultView::OnRelevanceChanged()
+{
+  if (!m_editor)
+    return;
+
+  auto relevance = Relevance::Irrelevant;
+  if (m_harmful->isChecked())
+    relevance = Relevance::Harmful;
+  else if (m_irrelevant->isChecked())
+    relevance = Relevance::Irrelevant;
+  else if (m_relevant->isChecked())
+    relevance = Relevance::Relevant;
+  else if (m_vital->isChecked())
+    relevance = Relevance::Vital;
+
+  m_editor->Set(relevance);
+}
+
+void ResultView::UpdateRelevanceRadioButtons()
+{
+  if (!m_editor)
+    return;
+
+  m_harmful->setChecked(false);
+  m_irrelevant->setChecked(false);
+  m_relevant->setChecked(false);
+  m_vital->setChecked(false);
+
+  auto const & r = m_editor->Get();
+
+  if (!r)
+    return;
+
+  switch (*r)
+  {
+  case Relevance::Harmful: m_harmful->setChecked(true); break;
+  case Relevance::Irrelevant: m_irrelevant->setChecked(true); break;
+  case Relevance::Relevant: m_relevant->setChecked(true); break;
+  case Relevance::Vital: m_vital->setChecked(true); break;
+  }
+}

libs/search/search_quality/assessment_tool/result_view.hpp

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/edits.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include <memory>
+#include <string>
+
+#include <QtWidgets/QWidget>
+
+class QLabel;
+class QRadioButton;
+
+namespace search
+{
+class Result;
+}
+
+class ResultView : public QWidget
+{
+public:
+  using Relevance = search::Sample::Result::Relevance;
+
+  ResultView(search::Result const & result, QWidget & parent);
+  ResultView(search::Sample::Result const & result, QWidget & parent);
+
+  void SetEditor(ResultsEdits::Editor && editor);
+
+  void Update();
+
+private:
+  ResultView(std::string const & name, std::string const & type, std::string const & address, QWidget & parent);
+
+  void Init();
+  void SetContents(std::string const & name, std::string const & type, std::string const & address);
+
+  QRadioButton * CreateRatioButton(std::string const & label, QLayout & layout);
+  void OnRelevanceChanged();
+  void UpdateRelevanceRadioButtons();
+
+  QLabel * m_name = nullptr;
+  QLabel * m_type = nullptr;
+  QLabel * m_address = nullptr;
+
+  QRadioButton * m_harmful = nullptr;
+  QRadioButton * m_irrelevant = nullptr;
+  QRadioButton * m_relevant = nullptr;
+  QRadioButton * m_vital = nullptr;
+
+  std::unique_ptr<ResultsEdits::Editor> m_editor;
+};

libs/search/search_quality/assessment_tool/results_view.cpp

@@ -0,0 +1,108 @@
+#include "search/search_quality/assessment_tool/results_view.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/assessment_tool/result_view.hpp"
+
+#include "base/assert.hpp"
+
+#include <QtWidgets/QListWidgetItem>
+
+ResultsView::ResultsView(QWidget & parent) : QListWidget(&parent)
+{
+  setAlternatingRowColors(true);
+
+  connect(selectionModel(), &QItemSelectionModel::selectionChanged, [&](QItemSelection const & current)
+  {
+    auto const indexes = current.indexes();
+    for (auto const & index : indexes)
+      emit OnResultSelected(index.row());
+  });
+
+  connect(this, &ResultsView::itemClicked, [&](QListWidgetItem * item)
+  {
+    auto const index = indexFromItem(item);
+    emit OnResultSelected(index.row());
+  });
+}
+
+void ResultsView::Add(search::Result const & result)
+{
+  AddImpl(result, false /* hidden */);
+  if (result.HasPoint())
+    m_hasResultsWithPoints = true;
+}
+
+void ResultsView::Add(search::Sample::Result const & result, ResultsEdits::Entry const & entry)
+{
+  AddImpl(result, entry.m_deleted /* hidden */);
+}
+
+ResultView & ResultsView::Get(size_t i)
+{
+  CHECK_LESS(i, Size(), ());
+  return *m_results[i];
+}
+
+ResultView const & ResultsView::Get(size_t i) const
+{
+  CHECK_LESS(i, Size(), ());
+  return *m_results[i];
+}
+
+void ResultsView::Update(ResultsEdits::Update const & update)
+{
+  switch (update.m_type)
+  {
+  case ResultsEdits::Update::Type::Single:
+  {
+    CHECK_LESS(update.m_index, m_results.size(), ());
+    m_results[update.m_index]->Update();
+    break;
+  }
+  case ResultsEdits::Update::Type::All:
+  {
+    for (auto * result : m_results)
+      result->Update();
+    break;
+  }
+  case ResultsEdits::Update::Type::Add:
+  {
+    CHECK_LESS(update.m_index, m_results.size(), ());
+    m_results[update.m_index]->Update();
+    break;
+  }
+  case ResultsEdits::Update::Type::Delete:
+  {
+    auto const index = update.m_index;
+    CHECK_LESS(index, Size(), ());
+    item(static_cast<int>(index))->setHidden(true);
+    break;
+  }
+  case ResultsEdits::Update::Type::Resurrect:
+    auto const index = update.m_index;
+    CHECK_LESS(index, Size(), ());
+    item(static_cast<int>(index))->setHidden(false);
+    break;
+  };
+}
+
+void ResultsView::Clear()
+{
+  m_results.clear();
+  m_hasResultsWithPoints = false;
+  clear();
+}
+
+template <typename Result>
+void ResultsView::AddImpl(Result const & result, bool hidden)
+{
+  auto * item = new QListWidgetItem(this /* parent */);
+  item->setHidden(hidden);
+  addItem(item);
+
+  auto * view = new ResultView(result, *this /* parent */);
+  item->setSizeHint(view->minimumSizeHint());
+  setItemWidget(item, view);
+
+  m_results.push_back(view);
+}

libs/search/search_quality/assessment_tool/results_view.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/edits.hpp"
+
+#include <cstddef>
+#include <vector>
+
+#include <QtWidgets/QListWidget>
+
+class QWidget;
+class ResultView;
+
+namespace search
+{
+class Result;
+}
+
+class ResultsView : public QListWidget
+{
+  Q_OBJECT
+
+public:
+  explicit ResultsView(QWidget & parent);
+
+  void Add(search::Result const & result);
+  void Add(search::Sample::Result const & result, ResultsEdits::Entry const & entry);
+
+  ResultView & Get(size_t i);
+  ResultView const & Get(size_t i) const;
+  void Update(ResultsEdits::Update const & update);
+
+  size_t Size() const { return m_results.size(); }
+  bool HasResultsWithPoints() const { return m_hasResultsWithPoints; }
+
+  void Clear();
+
+signals:
+  void OnResultSelected(int index);
+
+private:
+  template <typename Result>
+  void AddImpl(Result const & result, bool hidden);
+
+  std::vector<ResultView *> m_results;
+  bool m_hasResultsWithPoints = false;
+};

libs/search/search_quality/assessment_tool/sample_view.cpp

@@ -0,0 +1,387 @@
+#include "sample_view.hpp"
+
+#include "helpers.hpp"
+#include "result_view.hpp"
+#include "results_view.hpp"
+
+#include "qt/qt_common/spinner.hpp"
+
+#include "map/bookmark_manager.hpp"
+#include "map/framework.hpp"
+#include "map/search_mark.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include <QtGui/QStandardItem>
+#include <QtGui/QStandardItemModel>
+#include <QtWidgets/QComboBox>
+#include <QtWidgets/QGroupBox>
+#include <QtWidgets/QHBoxLayout>
+#include <QtWidgets/QHeaderView>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QLineEdit>
+#include <QtWidgets/QListWidget>
+#include <QtWidgets/QMenu>
+#include <QtWidgets/QPushButton>
+#include <QtWidgets/QVBoxLayout>
+
+namespace
+{
+template <typename Layout>
+Layout * BuildSubLayout(QLayout & mainLayout, QWidget & parent, QWidget ** box)
+{
+  *box = new QWidget(&parent);
+  auto * subLayout = BuildLayoutWithoutMargins<Layout>(*box /* parent */);
+  (*box)->setLayout(subLayout);
+  mainLayout.addWidget(*box);
+  return subLayout;
+}
+
+template <typename Layout>
+Layout * BuildSubLayout(QLayout & mainLayout, QWidget & parent)
+{
+  QWidget * box = nullptr;
+  return BuildSubLayout<Layout>(mainLayout, parent, &box);
+}
+
+void SetVerticalStretch(QWidget & widget, int stretch)
+{
+  auto policy = widget.sizePolicy();
+  policy.setVerticalStretch(stretch);
+  widget.setSizePolicy(policy);
+}
+}  // namespace
+
+SampleView::SampleView(QWidget * parent, Framework & framework) : QWidget(parent), m_framework(framework)
+{
+  m_framework.GetBookmarkManager().GetEditSession().SetIsVisible(UserMark::Type::SEARCH, true);
+  m_framework.GetBookmarkManager().GetEditSession().SetIsVisible(UserMark::Type::COLORED, true);
+
+  auto * mainLayout = BuildLayout<QVBoxLayout>(this /* parent */);
+
+  // When the dock for SampleView is attached to the right side of the
+  // screen, we don't need left margin, because of zoom in/zoom out
+  // slider. In other cases, it's better to keep left margin as is.
+  m_defaultMargins = mainLayout->contentsMargins();
+  m_rightAreaMargins = m_defaultMargins;
+  m_rightAreaMargins.setLeft(0);
+
+  {
+    m_query = new QLabel(this /* parent */);
+    m_query->setToolTip(tr("Query text"));
+    m_query->setWordWrap(true);
+    m_query->hide();
+    mainLayout->addWidget(m_query);
+  }
+
+  {
+    m_langs = new QLabel(this /* parent */);
+    m_langs->setToolTip(tr("Query input language"));
+    m_langs->hide();
+    mainLayout->addWidget(m_langs);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QHBoxLayout>(*mainLayout, *this /* parent */);
+
+    m_showViewport = new QPushButton(tr("Show viewport"), this /* parent */);
+    connect(m_showViewport, &QPushButton::clicked, [this]() { emit OnShowViewportClicked(); });
+    layout->addWidget(m_showViewport);
+
+    m_showPosition = new QPushButton(tr("Show position"), this /* parent */);
+    connect(m_showPosition, &QPushButton::clicked, [this]() { emit OnShowPositionClicked(); });
+    layout->addWidget(m_showPosition);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QVBoxLayout>(*mainLayout, *this /* parent */, &m_relatedQueriesBox);
+    SetVerticalStretch(*m_relatedQueriesBox, 1 /* stretch */);
+
+    layout->addWidget(new QLabel(tr("Related queries")));
+
+    m_relatedQueries = new QListWidget();
+    layout->addWidget(m_relatedQueries);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QHBoxLayout>(*mainLayout, *this /* parent */);
+
+    m_markAllAsRelevant = new QPushButton(tr("Mark all as Relevant"), this /* parent */);
+    connect(m_markAllAsRelevant, &QPushButton::clicked, [this]() { emit OnMarkAllAsRelevantClicked(); });
+    layout->addWidget(m_markAllAsRelevant);
+
+    m_markAllAsIrrelevant = new QPushButton(tr("Mark all as Irrelevant"), this /* parent */);
+    connect(m_markAllAsIrrelevant, &QPushButton::clicked, [this]() { emit OnMarkAllAsIrrelevantClicked(); });
+    layout->addWidget(m_markAllAsIrrelevant);
+  }
+
+  {
+    m_uselessnessLabel = new QLabel(this /* parent */);
+    m_uselessnessLabel->setText(tr("Sample is marked as useless"));
+    m_uselessnessLabel->hide();
+    mainLayout->addWidget(m_uselessnessLabel);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QVBoxLayout>(*mainLayout, *this /* parent */, &m_foundResultsBox);
+    SetVerticalStretch(*m_foundResultsBox, 4 /* stretch */);
+
+    {
+      auto * subLayout = BuildSubLayout<QHBoxLayout>(*layout, *this /* parent */);
+      subLayout->addWidget(new QLabel(tr("Found results")));
+
+      m_spinner = new Spinner();
+      subLayout->addWidget(&m_spinner->AsWidget());
+    }
+
+    m_foundResults = new ResultsView(*this /* parent */);
+    layout->addWidget(m_foundResults);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QVBoxLayout>(*mainLayout, *this /* parent */, &m_nonFoundResultsBox);
+    SetVerticalStretch(*m_nonFoundResultsBox, 2 /* stretch */);
+
+    layout->addWidget(new QLabel(tr("Non found results")));
+
+    m_nonFoundResults = new ResultsView(*this /* parent */);
+    m_nonFoundResults->setContextMenuPolicy(Qt::CustomContextMenu);
+    connect(m_nonFoundResults, &ResultsView::customContextMenuRequested, [&](QPoint pos)
+    {
+      pos = m_nonFoundResults->mapToGlobal(pos);
+
+      auto const items = m_nonFoundResults->selectedItems();
+      for (auto const * item : items)
+      {
+        int const row = m_nonFoundResults->row(item);
+
+        QMenu menu;
+        auto const * action = menu.addAction("Remove result");
+        connect(action, &QAction::triggered, [this, row]() { OnRemoveNonFoundResult(row); });
+
+        menu.exec(pos);
+      }
+    });
+    layout->addWidget(m_nonFoundResults);
+  }
+
+  setLayout(mainLayout);
+
+  Clear();
+}
+
+void SampleView::SetContents(search::Sample const & sample, std::optional<m2::PointD> const & position)
+{
+  if (!sample.m_query.empty())
+  {
+    m_query->setText(ToQString(sample.m_query));
+    m_query->show();
+  }
+  if (!sample.m_locale.empty())
+  {
+    m_langs->setText(ToQString(sample.m_locale));
+    m_langs->show();
+  }
+  m_showViewport->setEnabled(true);
+
+  m_relatedQueries->clear();
+  for (auto const & query : sample.m_relatedQueries)
+    m_relatedQueries->addItem(ToQString(query));
+  if (m_relatedQueries->count() != 0)
+    m_relatedQueriesBox->show();
+
+  ClearAllResults();
+  m_position = position;
+  if (m_position)
+    ShowUserPosition(*m_position);
+  else
+    HideUserPosition();
+}
+
+void SampleView::OnSearchStarted()
+{
+  m_spinner->Show();
+  m_showPosition->setEnabled(false);
+
+  m_markAllAsRelevant->setEnabled(false);
+  m_markAllAsIrrelevant->setEnabled(false);
+}
+
+void SampleView::OnSearchCompleted()
+{
+  m_spinner->Hide();
+  auto const resultsAvailable = m_foundResults->HasResultsWithPoints();
+  if (m_position)
+  {
+    if (resultsAvailable)
+      m_showPosition->setText(tr("Show position and top results"));
+    else
+      m_showPosition->setText(tr("Show position"));
+    m_showPosition->setEnabled(true);
+  }
+  else if (resultsAvailable)
+  {
+    m_showPosition->setText(tr("Show results"));
+    m_showPosition->setEnabled(true);
+  }
+  else
+  {
+    m_showPosition->setEnabled(false);
+  }
+
+  m_markAllAsRelevant->setEnabled(resultsAvailable);
+  m_markAllAsIrrelevant->setEnabled(resultsAvailable);
+}
+
+void SampleView::AddFoundResults(search::Results const & results)
+{
+  /// @todo Should clear previous m_foundResults.
+  for (auto const & res : results)
+    m_foundResults->Add(res);
+}
+
+void SampleView::ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                                     std::vector<ResultsEdits::Entry> const & entries)
+{
+  CHECK_EQUAL(results.size(), entries.size(), ());
+
+  m_nonFoundResults->Clear();
+
+  bool allDeleted = true;
+  for (size_t i = 0; i < results.size(); ++i)
+  {
+    m_nonFoundResults->Add(results[i], entries[i]);
+    if (!entries[i].m_deleted)
+      allDeleted = false;
+  }
+  if (!allDeleted)
+    m_nonFoundResultsBox->show();
+}
+
+void SampleView::ShowFoundResultsMarks(search::Results const & results)
+{
+  /// @todo Should clear previous _found_ results marks, but keep _nonfound_ if any.
+  m_framework.FillSearchResultsMarks(false /* clear */, results);
+}
+
+void SampleView::ShowNonFoundResultsMarks(std::vector<search::Sample::Result> const & results,
+                                          std::vector<ResultsEdits::Entry> const & entries)
+
+{
+  CHECK_EQUAL(results.size(), entries.size(), ());
+
+  auto editSession = m_framework.GetBookmarkManager().GetEditSession();
+  editSession.SetIsVisible(UserMark::Type::SEARCH, true);
+
+  for (size_t i = 0; i < results.size(); ++i)
+  {
+    auto const & result = results[i];
+    auto const & entry = entries[i];
+    if (entry.m_deleted)
+      continue;
+
+    auto * mark = editSession.CreateUserMark<SearchMarkPoint>(result.m_pos);
+    mark->SetNotFoundType();
+  }
+}
+
+void SampleView::ClearSearchResultMarks()
+{
+  m_framework.GetBookmarkManager().GetEditSession().ClearGroup(UserMark::Type::SEARCH);
+}
+
+void SampleView::ClearAllResults()
+{
+  m_foundResults->Clear();
+  m_nonFoundResults->Clear();
+  m_nonFoundResultsBox->hide();
+  ClearSearchResultMarks();
+}
+
+void SampleView::SetResultsEdits(ResultsEdits & resultsResultsEdits, ResultsEdits & nonFoundResultsEdits)
+{
+  SetResultsEdits(*m_foundResults, resultsResultsEdits);
+  SetResultsEdits(*m_nonFoundResults, nonFoundResultsEdits);
+  m_nonFoundResultsEdits = &nonFoundResultsEdits;
+}
+
+void SampleView::OnUselessnessChanged(bool isUseless)
+{
+  if (isUseless)
+  {
+    m_uselessnessLabel->show();
+    m_foundResultsBox->hide();
+    m_nonFoundResultsBox->hide();
+    m_markAllAsRelevant->hide();
+    m_markAllAsIrrelevant->hide();
+  }
+  else
+  {
+    m_uselessnessLabel->hide();
+    m_foundResultsBox->show();
+    m_markAllAsRelevant->show();
+    m_markAllAsIrrelevant->show();
+  }
+}
+
+void SampleView::Clear()
+{
+  m_query->hide();
+  m_langs->hide();
+
+  m_showViewport->setEnabled(false);
+  m_showPosition->setEnabled(false);
+
+  m_markAllAsRelevant->setEnabled(false);
+  m_markAllAsIrrelevant->setEnabled(false);
+
+  m_relatedQueriesBox->hide();
+
+  ClearAllResults();
+  HideUserPosition();
+  m_position = std::nullopt;
+  OnSearchCompleted();
+}
+
+void SampleView::OnLocationChanged(Qt::DockWidgetArea area)
+{
+  if (area == Qt::RightDockWidgetArea)
+    layout()->setContentsMargins(m_rightAreaMargins);
+  else
+    layout()->setContentsMargins(m_defaultMargins);
+}
+
+void SampleView::SetResultsEdits(ResultsView & results, ResultsEdits & edits)
+{
+  size_t const numRelevances = edits.GetRelevances().size();
+  CHECK_EQUAL(results.Size(), numRelevances, ());
+  for (size_t i = 0; i < numRelevances; ++i)
+    results.Get(i).SetEditor(ResultsEdits::Editor(edits, i));
+}
+
+void SampleView::OnRemoveNonFoundResult(int row)
+{
+  m_nonFoundResultsEdits->Delete(row);
+}
+
+void SampleView::ShowUserPosition(m2::PointD const & position)
+{
+  // Clear the old position.
+  HideUserPosition();
+
+  auto es = m_framework.GetBookmarkManager().GetEditSession();
+  auto mark = es.CreateUserMark<ColoredMarkPoint>(position);
+  mark->SetColor(dp::Color(200, 100, 240, 255) /* purple */);
+  mark->SetRadius(8.0f);
+  m_positionMarkId = mark->GetId();
+}
+
+void SampleView::HideUserPosition()
+{
+  if (m_positionMarkId == kml::kInvalidMarkId)
+    return;
+
+  m_framework.GetBookmarkManager().GetEditSession().DeleteUserMark(m_positionMarkId);
+  m_positionMarkId = kml::kInvalidMarkId;
+}

libs/search/search_quality/assessment_tool/sample_view.hpp

@@ -0,0 +1,102 @@
+#pragma once
+#include "edits.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "geometry/point2d.hpp"
+
+#include "kml/type_utils.hpp"
+
+#include <optional>
+
+#include <QtCore/QMargins>
+#include <QtWidgets/QWidget>
+
+class Framework;
+class QLabel;
+class QListWidget;
+class QPushButton;
+class ResultsView;
+class Spinner;
+
+class SampleView : public QWidget
+{
+  Q_OBJECT
+
+public:
+  using Relevance = search::Sample::Result::Relevance;
+
+  SampleView(QWidget * parent, Framework & framework);
+
+  void SetContents(search::Sample const & sample, std::optional<m2::PointD> const & position);
+  void OnSearchStarted();
+  void OnSearchCompleted();
+
+  void AddFoundResults(search::Results const & results);
+  void ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                           std::vector<ResultsEdits::Entry> const & entries);
+
+  void ShowFoundResultsMarks(search::Results const & results);
+  void ShowNonFoundResultsMarks(std::vector<search::Sample::Result> const & results,
+                                std::vector<ResultsEdits::Entry> const & entries);
+  void ClearSearchResultMarks();
+
+  void SetResultsEdits(ResultsEdits & resultsResultsEdits, ResultsEdits & nonFoundResultsResultsEdits);
+
+  void OnUselessnessChanged(bool isUseless);
+
+  void Clear();
+
+  ResultsView & GetFoundResultsView() { return *m_foundResults; }
+  ResultsView & GetNonFoundResultsView() { return *m_nonFoundResults; }
+
+  void OnLocationChanged(Qt::DockWidgetArea area);
+
+signals:
+  void OnShowViewportClicked();
+  void OnShowPositionClicked();
+  void OnMarkAllAsRelevantClicked();
+  void OnMarkAllAsIrrelevantClicked();
+
+private:
+  void ClearAllResults();
+  void SetResultsEdits(ResultsView & results, ResultsEdits & edits);
+  void OnRemoveNonFoundResult(int row);
+
+  void ShowUserPosition(m2::PointD const & position);
+  void HideUserPosition();
+
+  Framework & m_framework;
+
+  Spinner * m_spinner = nullptr;
+
+  QLabel * m_query = nullptr;
+  QLabel * m_langs = nullptr;
+
+  QListWidget * m_relatedQueries = nullptr;
+  QWidget * m_relatedQueriesBox = nullptr;
+
+  QPushButton * m_showViewport = nullptr;
+  QPushButton * m_showPosition = nullptr;
+
+  QPushButton * m_markAllAsRelevant = nullptr;
+  QPushButton * m_markAllAsIrrelevant = nullptr;
+
+  QLabel * m_uselessnessLabel = nullptr;
+
+  ResultsView * m_foundResults = nullptr;
+  QWidget * m_foundResultsBox = nullptr;
+
+  ResultsView * m_nonFoundResults = nullptr;
+  QWidget * m_nonFoundResultsBox = nullptr;
+
+  ResultsEdits * m_nonFoundResultsEdits = nullptr;
+
+  QMargins m_rightAreaMargins;
+  QMargins m_defaultMargins;
+
+  kml::MarkId m_positionMarkId = kml::kInvalidMarkId;
+
+  std::optional<m2::PointD> m_position;
+};

libs/search/search_quality/assessment_tool/samples_view.cpp

@@ -0,0 +1,77 @@
+#include "search/search_quality/assessment_tool/samples_view.hpp"
+
+#include "search/search_quality/assessment_tool/helpers.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+
+#include <string>
+
+#include <QtGui/QAction>
+#include <QtGui/QContextMenuEvent>
+#include <QtGui/QStandardItem>
+#include <QtWidgets/QHeaderView>
+#include <QtWidgets/QMenu>
+
+// SamplesView::Model ------------------------------------------------------------------------------
+SamplesView::Model::Model(QWidget * parent) : QStandardItemModel(0 /* rows */, 1 /* columns */, parent) {}
+
+QVariant SamplesView::Model::data(QModelIndex const & index, int role) const
+{
+  auto const row = index.row();
+
+  if (role == Qt::DisplayRole && m_samples.IsValid())
+    return QString::fromStdString(m_samples.GetLabel(row));
+  if (role == Qt::BackgroundRole && m_samples.IsValid())
+  {
+    if (m_samples.IsChanged(row))
+      return QBrush(QColor(0xFF, 0xFF, 0xC8));
+
+    if (m_samples.GetSearchState(row) == Context::SearchState::InQueue)
+      return QBrush(QColor(0xFF, 0xCC, 0x66));
+
+    if (m_samples.GetSearchState(row) == Context::SearchState::Completed)
+      return QBrush(QColor(0xCA, 0xFE, 0xDB));
+
+    return QBrush(Qt::transparent);
+  }
+  return QStandardItemModel::data(index, role);
+}
+
+// SamplesView -------------------------------------------------------------------------------------
+SamplesView::SamplesView(QWidget * parent) : QTableView(parent)
+{
+  setEditTriggers(QAbstractItemView::NoEditTriggers);
+  setSelectionMode(QAbstractItemView::SingleSelection);
+
+  {
+    auto * header = horizontalHeader();
+    header->setStretchLastSection(true /* stretch */);
+    header->hide();
+  }
+
+  m_model = new Model(this /* parent */);
+  // TODO: Do not invoke virtual functions from constructor.
+  setModel(m_model);
+}
+
+bool SamplesView::IsSelected(size_t index) const
+{
+  return selectionModel()->isRowSelected(base::checked_cast<int>(index), QModelIndex());
+}
+
+void SamplesView::contextMenuEvent(QContextMenuEvent * event)
+{
+  QModelIndex modelIndex = selectionModel()->currentIndex();
+  if (!modelIndex.isValid())
+    return;
+
+  int const index = modelIndex.row();
+  bool const isUseless = m_model->SampleIsUseless(index);
+
+  QMenu menu(this);
+  auto const text = std::string(isUseless ? "unmark" : "mark") + " sample as useless";
+  auto const * action = menu.addAction(text.c_str());
+  connect(action, &QAction::triggered, [this, index]() { emit FlipSampleUsefulness(index); });
+  menu.exec(event->globalPos());
+}

libs/search/search_quality/assessment_tool/samples_view.hpp

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/context.hpp"
+
+#include "base/checked_cast.hpp"
+
+#include <cstddef>
+#include <vector>
+
+#include <QtGui/QStandardItemModel>
+#include <QtWidgets/QMainWindow>
+#include <QtWidgets/QTableView>
+
+class SamplesView : public QTableView
+{
+  Q_OBJECT
+
+public:
+  explicit SamplesView(QWidget * parent);
+
+  void SetSamples(ContextList::SamplesSlice const & samples) { m_model->SetSamples(samples); }
+  bool IsSelected(size_t index) const;
+  void OnUpdate(size_t index) { m_model->OnUpdate(index); }
+  void Clear() { m_model->SetSamples(ContextList::SamplesSlice{}); }
+
+  // QMainWindow overrides:
+  void contextMenuEvent(QContextMenuEvent * event) override;
+
+signals:
+  void FlipSampleUsefulness(int index);
+
+private:
+  class Model : public QStandardItemModel
+  {
+  public:
+    explicit Model(QWidget * parent);
+
+    void SetSamples(ContextList::SamplesSlice const & samples)
+    {
+      removeRows(0, rowCount());
+      m_samples = samples;
+      if (m_samples.IsValid())
+        insertRows(0, base::checked_cast<int>(m_samples.Size()));
+    }
+
+    void OnUpdate(size_t index)
+    {
+      auto const ix = createIndex(base::checked_cast<int>(index) /* row */, 0 /* column */);
+
+      // Need to refresh view when some item is updated.
+      emit dataChanged(ix, ix);
+    }
+
+    bool SampleIsUseless(int index) const { return m_samples.IsUseless(index); }
+
+    // QStandardItemModel overrides:
+    QVariant data(QModelIndex const & index, int role = Qt::DisplayRole) const override;
+
+  private:
+    ContextList::SamplesSlice m_samples;
+  };
+
+  Model * m_model = nullptr;
+};

libs/search/search_quality/assessment_tool/search_request_runner.cpp

@@ -0,0 +1,253 @@
+#include "search/search_quality/assessment_tool/search_request_runner.hpp"
+
+#include "search/feature_loader.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <utility>
+
+using namespace std;
+
+SearchRequestRunner::SearchRequestRunner(Framework & framework, DataSource const & dataSource, ContextList & contexts,
+                                         UpdateViewOnResults && updateViewOnResults,
+                                         UpdateSampleSearchState && updateSampleSearchState)
+  : m_framework(framework)
+  , m_dataSource(dataSource)
+  , m_contexts(contexts)
+  , m_updateViewOnResults(std::move(updateViewOnResults))
+  , m_updateSampleSearchState(std::move(updateSampleSearchState))
+{}
+
+void SearchRequestRunner::InitiateForegroundSearch(size_t index)
+{
+  RunRequest(index, false /* background */, m_foregroundTimestamp);
+}
+
+void SearchRequestRunner::InitiateBackgroundSearch(size_t from, size_t to)
+{
+  // 1 <= from <= to <= m_contexts.Size().
+  if (from < 1 || from > to || to > m_contexts.Size())
+  {
+    LOG(LINFO, ("Could not initiate search in the range", from, to, "Total samples:", m_contexts.Size()));
+    return;
+  }
+
+  ResetBackgroundSearch();
+
+  // Convert to 0-based.
+  --from;
+  --to;
+  m_backgroundFirstIndex = from;
+  m_backgroundLastIndex = to;
+  m_backgroundNumProcessed = 0;
+
+  for (size_t index = from; index <= to; ++index)
+  {
+    if (m_contexts[index].m_searchState == Context::SearchState::Untouched)
+    {
+      m_contexts[index].m_searchState = Context::SearchState::InQueue;
+      m_backgroundQueue.push(index);
+      m_updateSampleSearchState(index);
+    }
+    else
+    {
+      CHECK(m_contexts[index].m_searchState == Context::SearchState::Completed, ());
+      ++m_backgroundNumProcessed;
+      LOG(LINFO, ("Using results from an earlier search for request number", index + 1));
+      if (m_backgroundNumProcessed == to - from + 1)
+        PrintBackgroundSearchStats();
+    }
+  }
+
+  size_t const numThreads = m_framework.GetSearchAPI().GetEngine().GetNumThreads();
+  for (size_t i = 0; i < numThreads; ++i)
+    RunNextBackgroundRequest(m_backgroundTimestamp);
+}
+
+void SearchRequestRunner::ResetForegroundSearch()
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  ++m_foregroundTimestamp;
+  if (auto handle = m_foregroundQueryHandle.lock())
+    handle->Cancel();
+}
+
+void SearchRequestRunner::ResetBackgroundSearch()
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  ++m_backgroundTimestamp;
+
+  queue<size_t>().swap(m_backgroundQueue);
+  m_backgroundNumProcessed = 0;
+
+  bool cancelledAny = false;
+  for (auto const & entry : m_backgroundQueryHandles)
+  {
+    auto handle = entry.second.lock();
+    if (handle)
+    {
+      handle->Cancel();
+      cancelledAny = true;
+    }
+  }
+  m_backgroundQueryHandles.clear();
+
+  if (cancelledAny)
+  {
+    for (size_t index = m_backgroundFirstIndex; index <= m_backgroundLastIndex; ++index)
+    {
+      if (m_contexts[index].m_searchState == Context::SearchState::InQueue)
+      {
+        m_contexts[index].m_searchState = Context::SearchState::Untouched;
+        m_updateSampleSearchState(index);
+      }
+    }
+  }
+
+  m_backgroundFirstIndex = kInvalidIndex;
+  m_backgroundLastIndex = kInvalidIndex;
+}
+
+void SearchRequestRunner::RunNextBackgroundRequest(size_t timestamp)
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  if (m_backgroundQueue.empty())
+    return;
+
+  size_t index = m_backgroundQueue.front();
+  m_backgroundQueue.pop();
+
+  RunRequest(index, true /* background */, timestamp);
+}
+
+void SearchRequestRunner::RunRequest(size_t index, bool background, size_t timestamp)
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  auto const & context = m_contexts[index];
+  auto const & sample = context.m_sample;
+
+  search::SearchParams params;
+  sample.FillSearchParams(params);
+  params.m_onResults = [=, this](search::Results const & results)
+  {
+    vector<optional<ResultsEdits::Relevance>> relevances;
+    vector<size_t> goldenMatching;
+    vector<size_t> actualMatching;
+
+    if (results.IsEndedNormal())
+    {
+      // Can't use MainModel's m_loader here due to thread-safety issues.
+      search::FeatureLoader loader(m_dataSource);
+      search::Matcher matcher(loader);
+
+      vector<search::Result> const actual(results.begin(), results.end());
+      matcher.Match(sample, actual, goldenMatching, actualMatching);
+      relevances.resize(actual.size());
+      for (size_t i = 0; i < goldenMatching.size(); ++i)
+      {
+        auto const j = goldenMatching[i];
+        if (j != search::Matcher::kInvalidId)
+        {
+          CHECK_LESS(j, relevances.size(), ());
+          relevances[j] = sample.m_results[i].m_relevance;
+        }
+      }
+
+      LOG(LINFO, ("Request number", index + 1, "has been processed in the", background ? "background" : "foreground"));
+    }
+
+    GetPlatform().RunTask(Platform::Thread::Gui,
+                          [this, background, timestamp, index, results, relevances, goldenMatching, actualMatching]
+    {
+      size_t const latestTimestamp = background ? m_backgroundTimestamp : m_foregroundTimestamp;
+      if (timestamp != latestTimestamp)
+        return;
+
+      auto & context = m_contexts[index];
+
+      context.m_foundResults = results;
+
+      if (results.IsEndMarker())
+      {
+        if (results.IsEndedNormal())
+          context.m_searchState = Context::SearchState::Completed;
+        else
+          context.m_searchState = Context::SearchState::Untouched;
+        m_updateSampleSearchState(index);
+      }
+
+      if (results.IsEndedNormal())
+      {
+        if (!context.m_initialized)
+        {
+          context.m_foundResultsEdits.Reset(relevances);
+          context.m_goldenMatching = goldenMatching;
+          context.m_actualMatching = actualMatching;
+
+          {
+            vector<optional<ResultsEdits::Relevance>> relevances;
+
+            auto & nonFound = context.m_nonFoundResults;
+            CHECK(nonFound.empty(), ());
+            for (size_t i = 0; i < context.m_goldenMatching.size(); ++i)
+            {
+              auto const j = context.m_goldenMatching[i];
+              if (j != search::Matcher::kInvalidId)
+                continue;
+              nonFound.push_back(context.m_sample.m_results[i]);
+              relevances.emplace_back(nonFound.back().m_relevance);
+            }
+            context.m_nonFoundResultsEdits.Reset(relevances);
+          }
+
+          context.m_initialized = true;
+        }
+
+        if (background)
+        {
+          ++m_backgroundNumProcessed;
+          m_backgroundQueryHandles.erase(index);
+          if (m_backgroundNumProcessed == m_backgroundLastIndex - m_backgroundFirstIndex + 1)
+            PrintBackgroundSearchStats();
+          else
+            RunNextBackgroundRequest(timestamp);
+        }
+      }
+
+      if (!background)
+        m_updateViewOnResults(results);
+    });
+  };
+
+  auto & engine = m_framework.GetSearchAPI().GetEngine();
+  if (background)
+    m_backgroundQueryHandles[index] = engine.Search(std::move(params));
+  else
+    m_foregroundQueryHandle = engine.Search(std::move(params));
+}
+
+void SearchRequestRunner::PrintBackgroundSearchStats() const
+{
+  LOG(LINFO, ("All requests from", m_backgroundFirstIndex + 1, "to", m_backgroundLastIndex + 1, "have been processed"));
+
+  vector<size_t> vitals;
+  vitals.reserve(m_backgroundLastIndex - m_backgroundFirstIndex + 1);
+  for (size_t index = m_backgroundFirstIndex; index <= m_backgroundLastIndex; ++index)
+  {
+    auto const & entries = m_contexts[index].m_foundResultsEdits.GetEntries();
+    bool const foundVital = any_of(entries.begin(), entries.end(), [](ResultsEdits::Entry const & e)
+    { return e.m_currRelevance == search::Sample::Result::Relevance::Vital; });
+    if (foundVital)
+      vitals.emplace_back(index + 1);
+  }
+
+  LOG(LINFO, ("Vital results found:", vitals.size(), "out of", m_backgroundLastIndex - m_backgroundFirstIndex + 1));
+  LOG(LINFO, ("Vital results found for these queries (1-based):", vitals));
+}

libs/search/search_quality/assessment_tool/search_request_runner.hpp

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/context.hpp"
+
+#include "map/framework.hpp"
+
+#include "base/thread_checker.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <map>
+#include <memory>
+#include <queue>
+#include <vector>
+
+// A proxy for SearchAPI/SearchEngine.
+// This class updates the Model's |m_contexts| directly (from the main thread) and updates
+// the View via the |m_updateViewOnResults| and |m_updateSampleSearchState| callbacks.
+class SearchRequestRunner
+{
+public:
+  using UpdateViewOnResults = std::function<void(search::Results const & results)>;
+  using UpdateSampleSearchState = std::function<void(size_t index)>;
+
+  SearchRequestRunner(Framework & framework, DataSource const & dataSource, ContextList & contexts,
+                      UpdateViewOnResults && updateViewOnResults, UpdateSampleSearchState && updateSampleSearchState);
+
+  void InitiateForegroundSearch(size_t index);
+
+  void InitiateBackgroundSearch(size_t from, size_t to);
+
+  void ResetForegroundSearch();
+
+  void ResetBackgroundSearch();
+
+private:
+  static size_t constexpr kInvalidIndex = std::numeric_limits<size_t>::max();
+
+  // Tries to run the unprocessed request with the smallest index, if there is one.
+  void RunNextBackgroundRequest(size_t timestamp);
+
+  void RunRequest(size_t index, bool background, size_t timestamp);
+
+  void PrintBackgroundSearchStats() const;
+
+  Framework & m_framework;
+
+  DataSource const & m_dataSource;
+
+  ContextList & m_contexts;
+
+  UpdateViewOnResults m_updateViewOnResults;
+  UpdateSampleSearchState m_updateSampleSearchState;
+
+  std::weak_ptr<search::ProcessorHandle> m_foregroundQueryHandle;
+  std::map<size_t, std::weak_ptr<search::ProcessorHandle>> m_backgroundQueryHandles;
+
+  size_t m_foregroundTimestamp = 0;
+  size_t m_backgroundTimestamp = 0;
+
+  size_t m_backgroundFirstIndex = kInvalidIndex;
+  size_t m_backgroundLastIndex = kInvalidIndex;
+  std::queue<size_t> m_backgroundQueue;
+  size_t m_backgroundNumProcessed = 0;
+
+  ThreadChecker m_threadChecker;
+};

libs/search/search_quality/assessment_tool/view.hpp

@@ -0,0 +1,69 @@
+#pragma once
+#include "context.hpp"
+#include "edits.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "geometry/point2d.hpp"
+#include "geometry/rect2d.hpp"
+
+#include <cstddef>
+#include <optional>
+#include <string>
+#include <vector>
+
+class ResultsEdits;
+class Model;
+
+class View
+{
+public:
+  enum class ResultType
+  {
+    Found,
+    NonFound
+  };
+
+  virtual ~View() = default;
+
+  void SetModel(Model & model) { m_model = &model; }
+
+  virtual void SetSamples(ContextList::SamplesSlice const & samples) = 0;
+  virtual void OnSearchStarted() = 0;
+  virtual void OnSearchCompleted() = 0;
+  virtual void ShowSample(size_t index, search::Sample const & sample, std::optional<m2::PointD> const & position,
+                          bool isUseless, bool hasEdits) = 0;
+
+  virtual void AddFoundResults(search::Results const & results) = 0;
+  virtual void ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                                   std::vector<ResultsEdits::Entry> const & entries) = 0;
+
+  virtual void ShowMarks(Context const & context) = 0;
+
+  virtual void MoveViewportToResult(search::Result const & result) = 0;
+  virtual void MoveViewportToResult(search::Sample::Result const & result) = 0;
+  virtual void MoveViewportToRect(m2::RectD const & rect) = 0;
+
+  virtual void OnResultChanged(size_t sampleIndex, ResultType type, ResultsEdits::Update const & update) = 0;
+  virtual void SetResultsEdits(size_t index, ResultsEdits & foundResultsEdits, ResultsEdits & nonFoundResultsEdits) = 0;
+  virtual void OnSampleChanged(size_t sampleIndex, bool isUseless, bool hasEdits) = 0;
+  virtual void OnSamplesChanged(bool hasEdits) = 0;
+
+  virtual void ShowError(std::string const & msg) = 0;
+
+  virtual void Clear() = 0;
+
+protected:
+  Model * m_model = nullptr;
+};
+
+inline std::string DebugPrint(View::ResultType type)
+{
+  switch (type)
+  {
+  case View::ResultType::Found: return "Found";
+  case View::ResultType::NonFound: return "NonFound";
+  }
+  return "Unknown";
+}

libs/search/search_quality/clusterize-tag-values.lisp

@@ -0,0 +1,333 @@
+#|
+exec /usr/bin/env sbcl --noinform --quit --eval "(defparameter *script-name* \"$0\")" --load "$0" --end-toplevel-options "$@"
+|#
+
+;;; This script clusterizes values from the taginfo database and
+;;; prints information about clusters.
+
+;;; Silently loads sqlite.
+(with-open-file (*standard-output* "/dev/null"
+                                   :direction :output
+                                   :if-exists :supersede)
+  (ql:quickload "sqlite"))
+
+(defun latin-char-p (char)
+  (or (and (char>= char #\a) (char<= char #\z))
+      (and (char>= char #\A) (char<= char #\Z))))
+
+(defun starts-with (text prefix)
+  "Returns non-nil if text starts with prefix."
+  (and (>= (length text) (length prefix))
+       (loop for u being the element of text
+          for v being the element of prefix
+          always (char= u v))))
+
+(defun get-postcode-pattern (postcode fn)
+  "Simplifies postcode in the following way:
+   * all latin letters are replaced by 'A'
+   * all digits are replaced by 'N'
+   * hyphens and dots are replaced by a space
+   * other characters are capitalized
+
+   This format follows https://en.wikipedia.org/wiki/List_of_postal_codes.
+  "
+  (let ((pattern (map 'string #'(lambda (c) (cond ((latin-char-p c) #\A)
+                                                  ((digit-char-p c) #\N)
+                                                  ((or (char= #\- c) (char= #\. c)) #\Space)
+                                                  (T c)))
+                      (string-upcase postcode))))
+    (funcall fn postcode pattern)))
+
+
+(defun get-phone-or-flat-pattern (phone fn)
+  "Simplifies phone or flat numbers in the following way:
+   * all letters are replaced by 'A'
+   * all digits are replaced by 'N'
+   * other characters are capitalized
+  "
+  (let ((pattern (map 'string #'(lambda (c) (cond ((alpha-char-p c) #\A)
+                                                  ((digit-char-p c) #\N)
+                                                  (T c)))
+                      (string-upcase phone))))
+    (funcall fn phone pattern)))
+
+(defun group-by (cmp list)
+  "cmp -> [a] -> [[a]]
+
+   Groups equal adjacent elements of the list. Equality is checked with cmp.
+  "
+  (let ((buckets
+           (reduce #'(lambda (buckets cur)
+                       (cond ((null buckets) (cons (list cur) nil))
+                             ((funcall cmp (caar buckets) cur)
+                              (cons (cons cur (car buckets)) (cdr buckets)))
+                             (T (cons (list cur) buckets))))
+                   list :initial-value nil)))
+    (reverse (mapcar #'reverse buckets))))
+
+(defun split-by (fn list)
+  "fn -> [a] -> [[a]]
+
+   Splits list by separators, where separators are defined by fn
+   predicate.
+  "
+  (loop for e in list
+     with buckets = nil
+     for prev-sep = T then cur-sep
+     for cur-sep = (funcall fn e)
+     do (cond (cur-sep T)
+              (prev-sep (push (list e) buckets))
+              (T (push e (car buckets))))
+     finally (return (reverse (mapcar #'reverse buckets)))))
+
+(defun split-string-by (fn string)
+  "fn -> string -> [string]
+
+   Splits string by separators, where separators are defined by fn
+   predicate.
+  "
+  (mapcar #'(lambda (list) (concatenate 'string list))
+          (split-by fn (concatenate 'list string))))
+
+(defun drop-while (fn list)
+  (cond ((null list) nil)
+        ((funcall fn (car list)) (drop-while fn (cdr list)))
+        (T list)))
+
+(defun take-while (fn list)
+  (if (null list)
+      nil
+      (loop for value in list
+         while (funcall fn value)
+         collecting value)))
+
+(defparameter *building-synonyms*
+  '("building" "bldg" "bld" "bl" "unit" "block" "blk"
+    "корпус" "корп" "кор" "литер" "лит" "строение" "стр" "блок" "бл"))
+
+(defparameter *house-number-seps* '(#\Space #\Tab #\" #\\ #\( #\) #\. #\# #\~))
+(defparameter *house-number-groups-seps* '(#\, #\| #\; #\+))
+
+(defun building-synonym-p (s)
+  (find s *building-synonyms* :test #'string=))
+
+(defun short-building-synonym-p (s)
+  (or (string= "к" s) (string= "с" s)))
+
+(defstruct token value type)
+
+(defun get-char-type (c)
+  (cond ((digit-char-p c) :number)
+        ((find c *house-number-seps* :test #'char=) :separator)
+        ((find c *house-number-groups-seps* :test #'char=) :group-separator)
+        ((char= c #\-) :hyphen)
+        ((char= c #\/) :slash)
+        (T :string)))
+
+(defun transform-string-token (fn value)
+  "Transforms building token value into one or more tokens in
+   accordance to its value.  For example, 'литA' is transformed to
+   tokens 'лит' (building part) and 'А' (letter).
+  "
+  (flet ((emit (value type) (funcall fn value type)))
+    (cond ((building-synonym-p value)
+           (emit value :building-part))
+          ((and (= 4 (length value))
+                (starts-with value "лит"))
+           (emit (subseq value 0 3) :building-part)
+           (emit (subseq value 3) :letter))
+          ((and (= 2 (length value))
+                (short-building-synonym-p (subseq value 0 1)))
+           (emit (subseq value 0 1) :building-part)
+           (emit (subseq value 1) :letter))
+          ((= 1 (length value))
+           (emit value (if (short-building-synonym-p value)
+                           :letter-or-building-part
+                           :letter)))
+          (T (emit value :string)))))
+
+(defun tokenize-house-number (house-number)
+  "house-number => [token]"
+  (let ((parts (group-by #'(lambda (lhs rhs)
+                             (eq (get-char-type lhs) (get-char-type rhs)))
+                         (string-downcase house-number)))
+        (tokens nil))
+    (flet ((add-token (value type) (push (make-token :value value :type type) tokens)))
+      (dolist (part parts)
+        (let ((value (concatenate 'string part))
+              (type (get-char-type (car part))))
+          (case type
+            (:string (transform-string-token #'add-token value))
+            (:separator T)
+            (otherwise (add-token value type)))))
+      (loop for prev = nil then curr
+         for curr in tokens
+         do (when (eq :letter-or-building-part (token-type curr))
+              (cond ((null prev)
+                     (setf (token-type curr) :letter))
+                    ((eq :number (token-type prev))
+                     (setf (token-type curr) :building-part)))))
+      (reverse tokens))))
+
+(defun house-number-with-optional-suffix-p (tokens)
+  (case (length tokens)
+    (1 (eq (token-type (first tokens)) :number))
+    (2 (let ((first-type (token-type (first tokens)))
+             (second-type (token-type (second tokens))))
+         (and (eq first-type :number)
+              (or (eq second-type :string)
+                  (eq second-type :letter)
+                  (eq second-type :letter-or-building-part)))))
+    (otherwise nil)))
+
+(defun get-house-number-sub-numbers (house-number)
+  "house-number => [[token]]
+
+   As house-number can be actually a collection of separated house
+   numbers, this function returns a list of possible house numbers.
+   Current implementation splits house number if and only if
+   house-number matches the following rule:
+
+   NUMBERS ::= (NUMBER STRING-SUFFIX?) | (NUMBER STRING-SUFFIX?) SEP NUMBERS
+  "
+  (let* ((tokens (tokenize-house-number house-number))
+         (groups (split-by #'(lambda (token) (eq :group-separator (token-type token))) tokens)))
+    (if (every #'house-number-with-optional-suffix-p groups)
+        groups
+        (list tokens))))
+
+(defun join-house-number-tokens (tokens)
+  "Joins token values with spaces."
+  (format nil "~{~a~^ ~}" (mapcar #'token-value tokens)))
+
+(defun join-house-number-parse (tokens)
+  "Joins parsed house number tokens with spaces."
+  (format nil "~{~a~^ ~}"
+          (mapcar #'(lambda (token)
+                      (let ((token-type (token-type token))
+                            (token-value (token-value token)))
+                        (case token-type
+                          (:number "N")
+                          (:building-part "B")
+                          (:letter "L")
+                          (:letter-or-building-part "U")
+                          (:string "S")
+                          ((:hyphen :slash :group-separator) token-value)
+                          (otherwise (assert NIL NIL (format nil "Unknown token type: ~a"
+                                                             token-type))))))
+                  tokens)))
+
+(defun get-house-number-pattern (house-number fn)
+  (dolist (number (get-house-number-sub-numbers house-number))
+    (let ((house-number (join-house-number-tokens number))
+          (pattern (join-house-number-parse number)))
+      (funcall fn house-number pattern))))
+
+(defun get-house-number-strings (house-number fn)
+  "Returns all strings from the house number."
+  (dolist (number (get-house-number-sub-numbers house-number))
+    (dolist (string (mapcar #'token-value
+                            (remove-if-not #'(lambda (token)
+                                               (case (token-type token)
+                                                 ((:string
+                                                   :letter
+                                                   :letter-or-building-part
+                                                   :building-part) T)
+                                                 (otherwise nil)))
+                                           number)))
+      (funcall fn string string))))
+
+(defstruct type-settings
+  pattern-simplifier
+  field-name)
+
+(defparameter *value-type-settings*
+  `(:postcode ,(make-type-settings :pattern-simplifier #'get-postcode-pattern
+                                   :field-name "addr:postcode")
+              :phone ,(make-type-settings :pattern-simplifier #'get-phone-or-flat-pattern
+                                          :field-name "contact:phone")
+              :flat ,(make-type-settings :pattern-simplifier #'get-phone-or-flat-pattern
+                                         :field-name "addr:flats")
+              :house-number ,(make-type-settings :pattern-simplifier #'get-house-number-pattern
+                                                 :field-name "addr:housenumber")
+              :house-number-strings ,(make-type-settings
+                                      :pattern-simplifier #'get-house-number-strings
+                                      :field-name "addr:housenumber")))
+
+(defstruct cluster
+  "A cluster of values with the same pattern, i.e.  all six-digits
+   series or all four-digits-two-letters series."
+  (key "") (num-samples 0) (samples nil))
+
+(defun add-sample (cluster sample &optional (count 1))
+  "Adds a value sample to a cluster of samples."
+  (push sample (cluster-samples cluster))
+  (incf (cluster-num-samples cluster) count))
+
+(defparameter *seps* '(#\Space #\Tab #\Newline #\Backspace #\Return #\Rubout #\Linefeed #\"))
+
+(defun trim (string)
+  "Removes leading and trailing garbage from a string."
+  (string-trim *seps* string))
+
+(defun get-pattern-clusters (values simplifier)
+  "Constructs a list of clusters by a list of values."
+  (let ((table (make-hash-table :test #'equal))
+        (clusters nil))
+    (loop for (value count) in values
+       do (funcall simplifier (trim value)
+                   #'(lambda (value pattern)
+                       (let ((cluster (gethash pattern table (make-cluster :key pattern))))
+                         (add-sample cluster value count)
+                         (setf (gethash pattern table) cluster)))))
+    (maphash #'(lambda (pattern cluster)
+                 (declare (ignore pattern))
+                 (push cluster clusters))
+             table)
+    clusters))
+
+(defun make-keyword (name) (values (intern (string-upcase name) "KEYWORD")))
+
+(when (/= 3 (length *posix-argv*))
+  (format t "Usage: ~a value path-to-taginfo-db.db~%" *script-name*)
+  (format t "~%value can be one of the following:~%")
+  (format t "~{    ~a~%~}"
+          (loop for field in *value-type-settings* by #'cddr collecting (string-downcase field)))
+  (exit :code -1))
+
+(defparameter *value-type* (second *posix-argv*))
+(defparameter *db-path* (third *posix-argv*))
+
+(defparameter *type-settings* (getf *value-type-settings* (make-keyword *value-type*)))
+
+(defparameter *values*
+  (sqlite:with-open-database (db *db-path*)
+    (let ((query (format nil "select value, count_all from tags where key=\"~a\";"
+                         (type-settings-field-name *type-settings*))))
+      (sqlite:execute-to-list db query))))
+
+(defparameter *clusters*
+  (sort (get-pattern-clusters *values* (type-settings-pattern-simplifier *type-settings*))
+        #'(lambda (lhs rhs) (> (cluster-num-samples lhs)
+                               (cluster-num-samples rhs)))))
+
+(defparameter *total*
+  (loop for cluster in *clusters*
+     summing (cluster-num-samples cluster)))
+
+(format t "Total: ~a~%" *total*)
+(loop for cluster in *clusters*
+   for prev-prefix-sum = 0 then curr-prefix-sum
+   for curr-prefix-sum = (+ prev-prefix-sum (cluster-num-samples cluster))
+   do (let ((key (cluster-key cluster))
+            (num-samples (cluster-num-samples cluster))
+            (samples (cluster-samples cluster)))
+        ; Prints number of values in a cluster, accumulated
+        ; percent of values clustered so far, simplified version
+        ; of a value and examples of values.
+        (format t "~a (~2$%) ~a [~{~a~^, ~}~:[~;, ...~]]~%"
+                num-samples
+                (* 100 (/ curr-prefix-sum *total*))
+                key
+                (subseq samples 0 (min (length samples) 5))
+                (> num-samples 5))))

libs/search/search_quality/download-maps.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# Downloads all maps necessary for learning to rank to the current
+# directory.
+
+ALL=
+VERSION=
+BASE="http://direct.mapswithme.com/direct"
+
+display_usage() {
+    echo "Usage: $0 -v [version] -a -h"
+    echo "    -v  version of maps to download"
+    echo "    -a  download all maps of the specified version"
+    echo "    -c  continue getting partially-downloaded files; new files on the server must be exactly the same as the ones in the previous attempt"
+    echo "    -h  display this message"
+}
+
+while getopts ":acv:h" opt
+do
+    case "$opt" in
+        a) ALL=1
+           ;;
+        c) RESUME_PARTIAL="-c"
+           ;;
+        v) VERSION="$OPTARG"
+           ;;
+        h) display_usage
+           exit -1
+           ;;
+        \?) echo "Invalid option: -$OPTARG" 1>&2
+            display_usage
+            exit -1
+            ;;
+        :) echo "Option -$OPTARG requires an argument" 1>&2
+           display_usage
+           exit -1
+           ;;
+    esac
+done
+
+if [ -z "$VERSION" ]
+then
+    echo "Version of maps is not specified." 1>&2
+    exit -1
+fi
+
+if ! curl "$BASE/" 2>/dev/null |
+        sed -n 's/^.*href="\(.*\)\/".*$/\1/p' |
+        grep -v "^../$" | grep -q "$VERSION"
+then
+    echo "Invalid version: $VERSION" 1>&2
+    exit -1
+fi
+
+NAMES=("Australia_Brisbane.mwm"
+       "Belarus_Hrodna*.mwm"
+       "Belarus_Minsk*.mwm"
+       "Canada_Ontario_London.mwm"
+       "Canada_Quebek_Montreal.mwm"
+       "Germany_*.mwm"
+       "Russia_*.mwm"
+       "UK_England_*.mwm"
+       "US_California_*.mwm"
+       "US_Maryland_*.mwm")
+
+DIR="$BASE/$VERSION"
+
+if [ "$ALL" ]
+then
+    echo "Downloading all maps..."
+
+    files=$(curl "$DIR/" 2>/dev/null | sed -n 's/^.*href="\(.*\.mwm\)".*$/\1/p')
+
+    set -e
+    set -x
+    for file in $files
+    do
+        wget $RESUME_PARTIAL -np -nd "$DIR/$file"
+    done
+else
+    echo "Downloading maps..."
+
+    set -e
+    set -x
+    for name in ${NAMES[@]}
+    do
+        wget $RESUME_PARTIAL -r -np -nd -A "$name" "$DIR/"
+    done
+fi

libs/search/search_quality/features_collector_tool/CMakeLists.txt

@@ -0,0 +1,11 @@
+project(features_collector_tool)
+
+set(SRC features_collector_tool.cpp)
+
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  search_quality
+  search_tests_support
+  gflags::gflags
+)

libs/search/search_quality/features_collector_tool/features_collector_tool.cpp

@@ -0,0 +1,202 @@
+#include "search/search_quality/helpers.hpp"
+#include "search/search_quality/matcher.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "search/search_tests_support/test_search_engine.hpp"
+#include "search/search_tests_support/test_search_request.hpp"
+
+#include "search/feature_loader.hpp"
+#include "search/ranking_info.hpp"
+#include "search/result.hpp"
+
+#include "indexer/classificator_loader.hpp"
+#include "indexer/data_source.hpp"
+
+#include "platform/platform_tests_support/helpers.hpp"
+
+#include "platform/local_country_file_utils.hpp"
+#include "platform/platform.hpp"
+
+#include "base/string_utils.hpp"
+
+#include <fstream>
+#include <iostream>
+#include <limits>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gflags/gflags.h>
+
+using namespace search::search_quality;
+using namespace search::tests_support;
+using namespace search;
+using namespace std;
+
+DEFINE_int32(num_threads, 1, "Number of search engine threads");
+DEFINE_string(data_path, "", "Path to data directory (resources dir)");
+DEFINE_string(mwm_path, "", "Path to mwm files (writable dir)");
+DEFINE_string(stats_path, "", "Path to store stats about queries results (default: stderr)");
+DEFINE_string(json_in, "", "Path to the json file with samples (default: stdin)");
+
+struct Stats
+{
+  // Indexes of not-found VITAL or RELEVANT results.
+  vector<size_t> m_notFound;
+};
+
+void GetContents(istream & is, string & contents)
+{
+  string line;
+  while (getline(is, line))
+  {
+    contents.append(line);
+    contents.push_back('\n');
+  }
+}
+
+void DisplayStats(ostream & os, vector<Sample> const & samples, vector<Stats> const & stats)
+{
+  auto const n = samples.size();
+  ASSERT_EQUAL(stats.size(), n, ());
+
+  size_t numWarnings = 0;
+  for (auto const & stat : stats)
+    if (!stat.m_notFound.empty())
+      ++numWarnings;
+
+  if (numWarnings == 0)
+  {
+    os << "All " << stats.size() << " queries are OK." << endl;
+    return;
+  }
+
+  os << numWarnings << " warnings." << endl;
+  for (size_t i = 0; i < n; ++i)
+  {
+    if (stats[i].m_notFound.empty())
+      continue;
+    os << "Query #" << i + 1 << " \"" << strings::ToUtf8(samples[i].m_query) << "\":" << endl;
+    for (auto const & j : stats[i].m_notFound)
+      os << "Not found: " << DebugPrint(samples[i].m_results[j]) << endl;
+  }
+}
+
+int main(int argc, char * argv[])
+{
+  platform::tests_support::ChangeMaxNumberOfOpenFiles(kMaxOpenFiles);
+  CheckLocale();
+
+  gflags::SetUsageMessage("Features collector tool.");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  SetPlatformDirs(FLAGS_data_path, FLAGS_mwm_path);
+
+  classificator::Load();
+
+  FrozenDataSource dataSource;
+  InitDataSource(dataSource, "" /* mwmListPath */);
+
+  auto engine = InitSearchEngine(dataSource, "en" /* locale */, FLAGS_num_threads);
+  engine->InitAffiliations();
+
+  vector<Sample> samples;
+  {
+    string lines;
+    if (FLAGS_json_in.empty())
+    {
+      GetContents(cin, lines);
+    }
+    else
+    {
+      ifstream ifs(FLAGS_json_in);
+      if (!ifs.is_open())
+      {
+        cerr << "Can't open input json file." << endl;
+        return -1;
+      }
+      GetContents(ifs, lines);
+    }
+
+    if (!Sample::DeserializeFromJSONLines(lines, samples))
+    {
+      cerr << "Can't parse input json file." << endl;
+      return -1;
+    }
+  }
+
+  vector<Stats> stats(samples.size());
+  FeatureLoader loader(dataSource);
+  Matcher matcher(loader);
+
+  vector<unique_ptr<TestSearchRequest>> requests;
+  requests.reserve(samples.size());
+
+  for (auto const & sample : samples)
+  {
+    search::SearchParams params;
+    sample.FillSearchParams(params);
+    params.m_batchSize = 100;
+    params.m_maxNumResults = 300;
+    requests.push_back(make_unique<TestSearchRequest>(*engine, params));
+    requests.back()->Start();
+  }
+
+  cout << "SampleId,";
+  RankingInfo::PrintCSVHeader(cout);
+  cout << ",Relevance" << endl;
+  for (size_t i = 0; i < samples.size(); ++i)
+  {
+    requests[i]->Wait();
+    auto const & sample = samples[i];
+    auto const & results = requests[i]->Results();
+
+    vector<size_t> goldenMatching;
+    vector<size_t> actualMatching;
+    matcher.Match(sample, results, goldenMatching, actualMatching);
+
+    for (size_t j = 0; j < results.size(); ++j)
+    {
+      if (results[j].GetResultType() != Result::Type::Feature)
+        continue;
+      if (actualMatching[j] == Matcher::kInvalidId)
+        continue;
+
+      auto const & info = results[j].GetRankingInfo();
+      cout << i << ",";
+      info.ToCSV(cout);
+
+      auto const relevance = sample.m_results[actualMatching[j]].m_relevance;
+      cout << "," << DebugPrint(relevance) << endl;
+    }
+
+    auto & s = stats[i];
+    for (size_t j = 0; j < goldenMatching.size(); ++j)
+    {
+      auto const wasNotFound = goldenMatching[j] == Matcher::kInvalidId ||
+                               goldenMatching[j] >= search::SearchParams::kDefaultNumResultsEverywhere;
+      auto const isRelevant = sample.m_results[j].m_relevance == Sample::Result::Relevance::Relevant ||
+                              sample.m_results[j].m_relevance == Sample::Result::Relevance::Vital;
+      if (wasNotFound && isRelevant)
+        s.m_notFound.push_back(j);
+    }
+    requests[i].reset();
+  }
+
+  if (FLAGS_stats_path.empty())
+  {
+    cerr << string(34, '=') << " Statistics " << string(34, '=') << endl;
+    DisplayStats(cerr, samples, stats);
+  }
+  else
+  {
+    ofstream ofs(FLAGS_stats_path);
+    if (!ofs.is_open())
+    {
+      cerr << "Can't open output file for stats." << endl;
+      return -1;
+    }
+    DisplayStats(ofs, samples, stats);
+  }
+  return 0;
+}

libs/search/search_quality/gen-samples.lisp

@@ -0,0 +1,135 @@
+#|
+exec /usr/bin/env sbcl --noinform --quit --load "$0" --end-toplevel-options "$@"
+|#
+
+;;; Silently loads :cl-json.
+(with-open-file (*standard-output* "/dev/null"
+                                   :direction :output
+                                   :if-exists :supersede)
+  (ql:quickload "cl-json"))
+
+(defparameter *samples* nil)
+
+(defparameter *minx* -180)
+(defparameter *maxx* 180)
+(defparameter *miny* -180)
+(defparameter *maxy* 180)
+
+(defun deg-to-rad (deg) (/ (* deg pi) 180))
+(defun rad-to-deg (rad) (/ (* rad 180) pi))
+
+(defun clamp (value min max)
+  (cond ((< value min) min)
+        ((> value max) max)
+        (t value)))
+
+(defun clamp-x (x) (clamp x *minx* *maxx*))
+(defun clamp-y (y) (clamp y *miny* *maxy*))
+
+(defun lon-to-x (lon) lon)
+(defun lat-to-y (lat)
+  (let* ((sinx (sin (deg-to-rad (clamp lat -86.0 86.0))))
+         (result (rad-to-deg (* 0.5
+                                (log (/ (+ 1.0 sinx)
+                                        (- 1.0 sinx)))))))
+    (clamp-y result)))
+
+(defclass pos () ((x :initarg :x)
+                  (y :initarg :y)))
+(defclass viewport () ((minx :initarg :minx)
+                       (miny :initarg :miny)
+                       (maxx :initarg :maxx)
+                       (maxy :initarg :maxy)))
+
+(defun position-x-y (x y)
+  (assert (and (>= x *minx*) (<= x *maxx*)))
+  (assert (and (>= y *miny*) (<= y *maxy*)))
+  (make-instance 'pos :x x :y y))
+
+(defun position-lat-lon (lat lon)
+  (position-x-y (lon-to-x lon) (lat-to-y lat)))
+
+(defun viewport (&key minx miny maxx maxy)
+  (assert (<= minx maxx))
+  (assert (<= miny maxy))
+  (make-instance 'viewport :minx minx :maxx maxx :miny miny :maxy maxy))
+
+(defclass result ()
+  ((name :initarg :name)
+   (relevancy :initarg :relevancy)
+   (types :initarg :types)
+   (position :initarg :position)
+   (house-number :initarg :house-number)))
+
+(defun make-result (relevancy name types position &key (house-number ""))
+  (make-instance 'result
+                 :name name
+                 :relevancy relevancy
+                 :types types
+                 :position position
+                 :house-number house-number))
+
+(defmacro vital (&rest args)
+  `(make-result 'vital ,@args))
+
+(defmacro relevant (&rest args)
+  `(make-result 'relevant ,@args))
+
+(defmacro irrelevant (&rest args)
+  `(make-result 'irrelevant ,@args))
+
+(defmacro harmful (&rest args)
+  `(make-result 'harmful ,@args))
+
+(defclass sample ()
+  ((query :initarg :query)
+   (locale :initarg :locale)
+   (position :initarg :position)
+   (viewport :initarg :viewport)
+   (results :initarg :results)))
+
+(defun make-sample (query locale position viewport results)
+  (make-instance 'sample
+                 :query query
+                 :locale locale
+                 :position position
+                 :viewport viewport
+                 :results results))
+
+(defmacro with-gensyms ((&rest syms) &rest body)
+  `(let ,(loop for sym in syms
+            collecting `(,sym (gensym)))
+     ,@body))
+
+(defmacro defsample (&rest args)
+  `(push (make-sample ,@args) *samples*))
+
+(defmacro scoped-samples ((locale position viewport) &rest body)
+  (with-gensyms (ls ps vs)
+    `(let ((,ls ,locale)
+           (,ps ,position)
+           (,vs ,viewport))
+       (flet ((def (query results)
+                (defsample query ,ls ,ps ,vs results)))
+         ,@body))))
+
+(defun power-set (seq)
+  (unless seq (return-from power-set '(())))
+  (let ((x (car seq))
+        (ps (power-set (cdr seq))))
+    (concatenate 'list ps (mapcar #'(lambda (xs) (cons x xs)) ps))))
+
+(defun join-strings (strings)
+  "Joins a list of strings with spaces between them."
+  (with-output-to-string (s)
+    (format s "~{~a~^ ~}" strings)))
+
+;;; Loads samples specification from standard input.
+(load *standard-input*)
+
+(format *error-output* "Num samples: ~a~%" (length *samples*))
+(format *error-output* "Num results: ~a~%"
+        (loop for sample in *samples*
+             summing (length (slot-value sample 'results))))
+
+(format t "~{~a~%~}" (mapcar #'json:encode-json-to-string (reverse *samples*)))

libs/search/search_quality/helpers.cpp

@@ -0,0 +1,157 @@
+#include "search/search_quality/helpers.hpp"
+
+#include "indexer/data_source.hpp"
+
+#include "platform/local_country_file.hpp"
+#include "platform/local_country_file_utils.hpp"
+#include "platform/platform.hpp"
+
+#include "coding/reader.hpp"
+
+#include "geometry/mercator.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+#include "base/string_utils.hpp"
+
+#include <fstream>
+#include <limits>
+#include <utility>
+
+#include "defines.hpp"
+
+#include "cppjansson/cppjansson.hpp"
+
+namespace search
+{
+namespace search_quality
+{
+using namespace std;
+
+namespace
+{
+uint64_t ReadVersionFromHeader(platform::LocalCountryFile const & mwm)
+{
+  vector<string> const kSpecialFiles = {WORLD_FILE_NAME, WORLD_COASTS_FILE_NAME};
+  for (auto const & name : kSpecialFiles)
+    if (mwm.GetCountryName() == name)
+      return mwm.GetVersion();
+
+  return version::MwmVersion::Read(FilesContainerR(mwm.GetPath(MapFileType::Map))).GetVersion();
+}
+}  // namespace
+
+void CheckLocale()
+{
+  string const kJson = "{\"coord\":123.456}";
+  string const kErrorMsg = "Bad locale. Consider setting LC_ALL=C";
+
+  double coord;
+  {
+    base::Json root(kJson.c_str());
+    FromJSONObject(root.get(), "coord", coord);
+  }
+
+  string line;
+  {
+    auto root = base::NewJSONObject();
+    ToJSONObject(*root, "coord", coord);
+
+    unique_ptr<char, JSONFreeDeleter> buffer(json_dumps(root.get(), JSON_COMPACT));
+
+    line.append(buffer.get());
+  }
+
+  CHECK_EQUAL(line, kJson, (kErrorMsg));
+
+  {
+    string const kTest = "123.456";
+    double value;
+    VERIFY(strings::to_double(kTest, value), (kTest));
+    CHECK_EQUAL(strings::to_string(value), kTest, (kErrorMsg));
+  }
+}
+
+void ReadStringsFromFile(string const & path, vector<string> & result)
+{
+  ifstream stream(path.c_str());
+  CHECK(stream.is_open(), ("Can't open", path));
+
+  string s;
+  while (getline(stream, s))
+  {
+    strings::Trim(s);
+    if (!s.empty())
+      result.emplace_back(s);
+  }
+}
+
+void SetPlatformDirs(string const & dataPath, string const & mwmPath)
+{
+  Platform & platform = GetPlatform();
+
+  if (!dataPath.empty())
+    platform.SetResourceDir(dataPath);
+
+  if (!mwmPath.empty())
+    platform.SetWritableDirForTests(mwmPath);
+
+  LOG(LINFO, ("writable dir =", platform.WritableDir()));
+  LOG(LINFO, ("resources dir =", platform.ResourcesDir()));
+}
+
+void InitViewport(string viewportName, m2::RectD & viewport)
+{
+  map<string, m2::RectD> const kViewports = {{"default", m2::RectD(m2::PointD(0.0, 0.0), m2::PointD(1.0, 1.0))},
+                                             {"moscow", mercator::RectByCenterLatLonAndSizeInMeters(55.7, 37.7, 5000)},
+                                             {"london", mercator::RectByCenterLatLonAndSizeInMeters(51.5, 0.0, 5000)},
+                                             {"zurich", mercator::RectByCenterLatLonAndSizeInMeters(47.4, 8.5, 5000)}};
+
+  auto it = kViewports.find(viewportName);
+  if (it == kViewports.end())
+  {
+    LOG(LINFO, ("Unknown viewport name:", viewportName, "; setting to default"));
+    viewportName = "default";
+    it = kViewports.find(viewportName);
+  }
+  CHECK(it != kViewports.end(), ());
+  viewport = it->second;
+  LOG(LINFO, ("Viewport is set to:", viewportName, DebugPrint(viewport)));
+}
+
+void InitDataSource(FrozenDataSource & dataSource, string const & mwmListPath)
+{
+  vector<platform::LocalCountryFile> mwms;
+  if (!mwmListPath.empty())
+  {
+    vector<string> availableMwms;
+    ReadStringsFromFile(mwmListPath, availableMwms);
+    for (auto const & countryName : availableMwms)
+      mwms.emplace_back(GetPlatform().WritableDir(), platform::CountryFile(countryName), 0);
+  }
+  else
+  {
+    platform::FindAllLocalMapsAndCleanup(numeric_limits<int64_t>::max() /* the latest version */, mwms);
+  }
+
+  LOG(LINFO, ("Initializing the data source with the following mwms:"));
+  for (auto & mwm : mwms)
+  {
+    mwm.SyncWithDisk();
+    LOG(LINFO, (mwm.GetCountryName(), ReadVersionFromHeader(mwm)));
+    dataSource.RegisterMap(mwm);
+  }
+  LOG(LINFO, ());
+}
+
+unique_ptr<search::tests_support::TestSearchEngine> InitSearchEngine(DataSource & dataSource, string const & locale,
+                                                                     size_t numThreads)
+{
+  search::Engine::Params params;
+  params.m_locale = locale;
+  params.m_numThreads = base::checked_cast<size_t>(numThreads);
+
+  return make_unique<search::tests_support::TestSearchEngine>(dataSource, params);
+}
+}  // namespace search_quality
+}  // namespace search

libs/search/search_quality/helpers.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "search/search_tests_support/test_search_engine.hpp"
+
+#include "geometry/rect2d.hpp"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class DataSource;
+class FrozenDataSource;
+
+namespace search
+{
+namespace search_quality
+{
+// todo(@m) We should not need that much.
+size_t constexpr kMaxOpenFiles = 4000;
+
+void CheckLocale();
+
+void ReadStringsFromFile(std::string const & path, std::vector<std::string> & result);
+
+void SetPlatformDirs(std::string const & dataPath, std::string const & mwmPath);
+
+void InitViewport(std::string viewportName, m2::RectD & viewport);
+
+void InitDataSource(FrozenDataSource & dataSource, std::string const & mwmListPath);
+
+std::unique_ptr<search::tests_support::TestSearchEngine> InitSearchEngine(DataSource & dataSource,
+                                                                          std::string const & locale,
+                                                                          size_t numThreads);
+}  // namespace search_quality
+}  // namespace search

libs/search/search_quality/helpers_json.cpp

@@ -0,0 +1,105 @@
+#include "search/search_quality/helpers_json.hpp"
+
+namespace m2
+{
+using std::string;
+
+namespace
+{
+void ParsePoint(json_t * root, m2::PointD & point)
+{
+  FromJSONObject(root, "x", point.x);
+  FromJSONObject(root, "y", point.y);
+}
+}  // namespace
+
+void FromJSONObject(json_t * root, char const * field, RectD & rect)
+{
+  json_t * r = base::GetJSONObligatoryField(root, field);
+  double minX, minY, maxX, maxY;
+  FromJSONObject(r, "minx", minX);
+  FromJSONObject(r, "miny", minY);
+  FromJSONObject(r, "maxx", maxX);
+  FromJSONObject(r, "maxy", maxY);
+  rect.setMinX(minX);
+  rect.setMinY(minY);
+  rect.setMaxX(maxX);
+  rect.setMaxY(maxY);
+}
+
+void ToJSONObject(json_t & root, char const * field, RectD const & rect)
+{
+  auto json = base::NewJSONObject();
+  ToJSONObject(*json, "minx", rect.minX());
+  ToJSONObject(*json, "miny", rect.minY());
+  ToJSONObject(*json, "maxx", rect.maxX());
+  ToJSONObject(*json, "maxy", rect.maxY());
+  json_object_set_new(&root, field, json.release());
+}
+
+void FromJSONObject(json_t * root, string const & field, RectD & rect)
+{
+  FromJSONObject(root, field.c_str(), rect);
+}
+
+void ToJSONObject(json_t & root, string const & field, RectD const & rect)
+{
+  ToJSONObject(root, field.c_str(), rect);
+}
+
+void FromJSONObject(json_t * root, char const * field, PointD & point)
+{
+  json_t * p = base::GetJSONObligatoryField(root, field);
+  ParsePoint(p, point);
+}
+
+void FromJSONObject(json_t * root, string const & field, PointD & point)
+{
+  FromJSONObject(root, field.c_str(), point);
+}
+
+void FromJSONObjectOptional(json_t * root, char const * field, std::optional<PointD> & point)
+{
+  json_t * p = base::GetJSONOptionalField(root, field);
+  if (!p || base::JSONIsNull(p))
+  {
+    point = std::nullopt;
+    return;
+  }
+
+  PointD parsed;
+  ParsePoint(p, parsed);
+  point = parsed;
+}
+
+void ToJSONObject(json_t & root, char const * field, PointD const & point)
+{
+  auto json = base::NewJSONObject();
+  ToJSONObject(*json, "x", point.x);
+  ToJSONObject(*json, "y", point.y);
+  json_object_set_new(&root, field, json.release());
+}
+
+void FromJSONObjectOptional(json_t * root, string const & field, std::optional<PointD> & point)
+{
+  FromJSONObjectOptional(root, field.c_str(), point);
+}
+
+void ToJSONObject(json_t & root, string const & field, PointD const & point)
+{
+  ToJSONObject(root, field.c_str(), point);
+}
+
+void ToJSONObject(json_t & root, char const * field, std::optional<PointD> const & point)
+{
+  if (point)
+    ToJSONObject(root, field, *point);
+  else
+    ToJSONObject(root, field, base::NewJSONNull());
+}
+
+void ToJSONObject(json_t & root, string const & field, std::optional<PointD> const & point)
+{
+  ToJSONObject(root, field.c_str(), point);
+}
+}  // namespace m2

libs/search/search_quality/helpers_json.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "geometry/point2d.hpp"
+#include "geometry/rect2d.hpp"
+
+#include <optional>
+#include <string>
+
+#include "cppjansson/cppjansson.hpp"
+
+namespace m2
+{
+void FromJSONObject(json_t * root, char const * field, RectD & rect);
+void ToJSONObject(json_t & root, char const * field, RectD const & rect);
+void FromJSONObject(json_t * root, std::string const & field, RectD & rect);
+void ToJSONObject(json_t & root, std::string const & field, RectD const & rect);
+
+void FromJSONObject(json_t * root, char const * field, PointD & point);
+void FromJSONObjectOptional(json_t * root, char const * field, std::optional<PointD> & point);
+void FromJSONObject(json_t * root, std::string const & field, PointD & point);
+void FromJSONObjectOptional(json_t * root, std::string const & field, std::optional<PointD> & point);
+
+void ToJSONObject(json_t & root, char const * field, PointD const & point);
+void ToJSONObject(json_t & root, std::string const & field, PointD const & point);
+void ToJSONObject(json_t & root, char const * field, std::optional<PointD> const & point);
+void ToJSONObject(json_t & root, std::string const & field, std::optional<PointD> const & point);
+}  // namespace m2

libs/search/search_quality/matcher.cpp

@@ -0,0 +1,203 @@
+#include "search/search_quality/matcher.hpp"
+
+#include "search/feature_loader.hpp"
+#include "search/house_numbers_matcher.hpp"
+
+#include "indexer/feature.hpp"
+#include "indexer/feature_algo.hpp"
+#include "indexer/road_shields_parser.hpp"
+#include "indexer/search_string_utils.hpp"
+
+#include "base/control_flow.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+
+namespace search
+{
+namespace
+{
+template <typename Iter>
+bool StartsWithHouseNumber(Iter beg, Iter end)
+{
+  using namespace search::house_numbers;
+
+  std::string s;
+  for (auto it = beg; it != end; ++it)
+  {
+    if (!s.empty())
+      s.append(" ");
+    s.append(*it);
+    if (LooksLikeHouseNumber(s, false /* isPrefix */))
+      return true;
+  }
+  return false;
+}
+
+// todo(@m) This function looks very slow.
+template <typename Iter>
+bool EndsWithHouseNumber(Iter beg, Iter end)
+{
+  using namespace search::house_numbers;
+
+  if (beg == end)
+    return false;
+
+  std::string s;
+  for (auto it = --end;; --it)
+  {
+    if (s.empty())
+      s = *it;
+    else
+      s = *it + " " + s;
+    if (LooksLikeHouseNumber(s, false /* isPrefix */))
+      return true;
+    if (it == beg)
+      break;
+  }
+  return false;
+}
+
+std::vector<std::string> NormalizeAndTokenizeAsUtf8(std::string_view str)
+{
+  std::vector<std::string> res;
+  ForEachNormalizedToken(str, [&res](strings::UniString const & token) { res.push_back(strings::ToUtf8(token)); });
+  return res;
+}
+
+bool StreetMatches(std::string_view name, std::vector<std::string> const & queryTokens)
+{
+  auto const nameTokens = NormalizeAndTokenizeAsUtf8(name);
+  if (nameTokens.empty())
+    return false;
+
+  for (size_t i = 0; i + nameTokens.size() <= queryTokens.size(); ++i)
+  {
+    bool found = true;
+    for (size_t j = 0; j < nameTokens.size(); ++j)
+    {
+      if (queryTokens[i + j] != nameTokens[j])
+      {
+        found = false;
+        break;
+      }
+    }
+
+    if (!found)
+      continue;
+
+    if (!EndsWithHouseNumber(queryTokens.begin(), queryTokens.begin() + i) &&
+        !StartsWithHouseNumber(queryTokens.begin() + i + nameTokens.size(), queryTokens.end()))
+    {
+      return true;
+    }
+  }
+
+  return false;
+}
+}  // namespace
+
+Matcher::Matcher(FeatureLoader & loader) : m_loader(loader) {}
+
+void Matcher::Match(Sample const & goldenSample, std::vector<Result> const & actual,
+                    std::vector<size_t> & goldenMatching, std::vector<size_t> & actualMatching)
+{
+  auto const & golden = goldenSample.m_results;
+
+  auto const n = golden.size();
+  auto const m = actual.size();
+
+  goldenMatching.assign(n, kInvalidId);
+  actualMatching.assign(m, kInvalidId);
+
+  // TODO (@y, @m): use Kuhn algorithm here for maximum matching.
+  for (size_t i = 0; i < n; ++i)
+  {
+    if (goldenMatching[i] != kInvalidId)
+      continue;
+    auto const & g = golden[i];
+
+    for (size_t j = 0; j < m; ++j)
+    {
+      if (actualMatching[j] != kInvalidId)
+        continue;
+
+      auto const & a = actual[j];
+      if (Matches(goldenSample.m_query, g, a))
+      {
+        goldenMatching[i] = j;
+        actualMatching[j] = i;
+        break;
+      }
+    }
+  }
+}
+
+bool Matcher::Matches(strings::UniString const & query, Sample::Result const & golden, search::Result const & actual)
+{
+  if (actual.GetResultType() != Result::Type::Feature)
+    return false;
+
+  auto ft = m_loader.Load(actual.GetFeatureID());
+  if (!ft)
+    return false;
+
+  return Matches(query, golden, *ft);
+}
+
+bool Matcher::Matches(strings::UniString const & query, Sample::Result const & golden, FeatureType & ft)
+{
+  auto const queryTokens = NormalizeAndTokenizeAsUtf8(ToUtf8(query));
+
+  bool nameMatches = false;
+
+  // The golden result may have an empty name. What is more likely, though, is that the
+  // sample was not obtained as a result of a previous run of our search_quality tools.
+  // Probably it originates from a third-party source.
+  if (golden.m_name.empty())
+  {
+    if (ft.GetGeomType() == feature::GeomType::Line)
+    {
+      for (auto const & name : ftypes::GetRoadShieldsNames(ft))
+      {
+        if (StreetMatches(name, queryTokens))
+        {
+          nameMatches = true;
+          break;
+        }
+      }
+    }
+    else
+    {
+      // Don't try to guess: it's enough to match by distance.
+      // |ft| with GeomType::Point here is usually a POI and |ft| with GeomType::Area is a building.
+      nameMatches = true;
+    }
+  }
+
+  ft.ForEachName([&queryTokens, &ft, &golden, &nameMatches](int8_t /* lang */, std::string_view name)
+  {
+    if (NormalizeAndSimplifyString(ToUtf8(golden.m_name)) == NormalizeAndSimplifyString(name))
+    {
+      nameMatches = true;
+      return base::ControlFlow::Break;
+    }
+
+    if (golden.m_name.empty() && ft.GetGeomType() == feature::GeomType::Line && StreetMatches(name, queryTokens))
+    {
+      nameMatches = true;
+      return base::ControlFlow::Break;
+    }
+
+    return base::ControlFlow::Continue;
+  });
+
+  bool houseNumberMatches = true;
+  std::string const & hn = ft.GetHouseNumber();
+  if (!golden.m_houseNumber.empty() && !hn.empty())
+    houseNumberMatches = golden.m_houseNumber == hn;
+
+  /// @todo Where are 50 meters came from?
+  return (nameMatches && houseNumberMatches && feature::GetMinDistanceMeters(ft, golden.m_pos) < 50.0);
+}
+}  // namespace search

libs/search/search_quality/matcher.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "search/search_quality/sample.hpp"
+
+#include "search/result.hpp"
+
+#include "base/string_utils.hpp"
+
+#include <cstddef>
+#include <limits>
+#include <vector>
+
+class FeatureType;
+
+namespace search
+{
+class FeatureLoader;
+
+class Matcher
+{
+public:
+  inline static size_t constexpr kInvalidId = std::numeric_limits<size_t>::max();
+
+  explicit Matcher(FeatureLoader & loader);
+
+  // Matches the results loaded from |goldenSample| with |actual| results
+  // found by the search engine using the params from the Sample.
+  // goldenMatching[i] is the index of the result in |actual| that matches
+  // the sample result number i.
+  // actualMatching[j] is the index of the sample in |golden| that matches
+  // the golden result number j.
+  void Match(Sample const & goldenSample, std::vector<Result> const & actual, std::vector<size_t> & goldenMatching,
+             std::vector<size_t> & actualMatching);
+
+  bool Matches(strings::UniString const & query, Sample::Result const & golden, Result const & actual);
+  bool Matches(strings::UniString const & query, Sample::Result const & golden, FeatureType & ft);
+
+private:
+  FeatureLoader & m_loader;
+};
+}  // namespace search

libs/search/search_quality/merge_samples.py

@@ -0,0 +1,200 @@
+import argparse
+import copy
+import json
+
+
+def read_samples(path):
+    with open(path, "r") as f:
+        return [json.loads(line) for line in f]
+
+
+def doubles_equal(a, b):
+    EPS = 1e-6
+    return abs(a - b) < EPS
+
+
+def points_equal(a, b):
+    return doubles_equal(a["x"], b["x"]) and doubles_equal(a["y"], b["y"])
+
+
+def rects_equal(a, b):
+    for key in ["minx", "miny", "maxx", "maxy"]:
+        if not doubles_equal(a[key], b[key]):
+            return False
+    return True
+
+
+def samples_equal(a, b):
+    """
+    Returns whether two samples originate from the same unassessed sample,
+    i.e. found results and uselessness are not taken into account.
+    """
+    for field in ["query", "locale"]:
+        if a[field] != b[field]:
+            return False
+
+    pos_key = "position"
+    bad_point = {"x": -1, "y": -1}
+    a_pos = a[pos_key] or bad_point
+    b_pos = b[pos_key] or bad_point
+    if not points_equal(a_pos, b_pos):
+        return False
+
+    vp_key = "viewport"
+    if not rects_equal(a[vp_key], b[vp_key]):
+        return False
+
+    return True
+
+
+def results_equal(a, b):
+    """
+    Returns whether two found results are equal before assessing.
+    Does not take relevance into account.
+    """
+    pos_key = "position"
+    name_key = "name"
+    hn_key = "houseNumber"
+    types_key = "types"
+    if a[name_key] != b[name_key]:
+        return False
+    if not points_equal(a[pos_key], b[pos_key]):
+        return False
+    if a[hn_key] != b[hn_key]:
+        return False
+    if sorted(a[types_key]) != sorted(b[types_key]):
+        return False
+    return True
+
+
+def greedily_match_results(a, b):
+    match_to_a = [-1] * len(a)
+    match_to_b = [-1] * len(b)
+    for i in range(len(a)):
+        if match_to_a[i] >= 0:
+            continue
+        for j in range(len(b)):
+            if match_to_b[j] >= 0:
+                continue
+            if results_equal(a[i], b[j]):
+                match_to_a[i] = j
+                match_to_b[j] = i
+                break
+    return match_to_a, match_to_b
+
+
+def merge_relevancies(a, b):
+    """
+    The second element of the returned tuple is True
+    iff the relevancies are compatible and can be merged.
+    Two relevancies are incompatible if one of them is a
+    "strong" one, i.e. either "harmful" or "vital", and
+    the other is "weak" and has the opposite sign ("relevant"
+    and "irrelevant" respectively).
+
+    If a and b are compatible, then the first element of the tuple is
+    the resulting relevance. The less intuitive merge results are:
+      strong, weak -> strong
+      relevant, irrelevant -> relevant
+    """
+    RELEVANCIES = ["harmful", "irrelevant", "relevant", "vital"]
+    id_a = RELEVANCIES.index(a)
+    id_b = RELEVANCIES.index(b)
+    if id_a > id_b:
+        a, b = b, a
+        id_a, id_b = id_b, id_a
+    if id_a == 0 and id_b <= 1:
+        return a, True
+    if id_a == 1 and id_b <= 2:
+        return b, True
+    if id_a > 1:
+        return b, True
+    return a, False
+
+
+def merge_two_samples(a, b, line_number):
+    if not samples_equal(a, b):
+        raise Exception(f"Tried to merge two non-equivalent samples, line {line_number}")
+
+    useless_key = "useless"
+    useless_a = a.get(useless_key, False)
+    useless_b = b.get(useless_key, False)
+
+    if useless_a != useless_b:
+        print(line_number, "useless:", useless_a, useless_b)
+
+    if useless_a and not useless_b:
+        return b
+    if useless_b:
+        return a
+
+    # Both are not useless.
+
+    res_key = "results"
+    match_to_a, match_to_b = greedily_match_results(a[res_key], b[res_key])
+
+    lst = [x for x in match_to_a if x < 0]
+
+    rel_key = "relevancy"
+
+    c = copy.deepcopy(a)
+    c[res_key] = []
+
+    for i, j in enumerate(match_to_a):
+        if j < 0:
+            continue
+        res = a[res_key][i]
+        ra = a[res_key][i][rel_key]
+        rb = b[res_key][j][rel_key]
+        rc, ok = merge_relevancies(ra, rb)
+        res[rel_key] = rc
+        if not ok:
+            print(line_number, ra, rb, a[res_key][i]["name"])
+            continue
+        c[res_key].append(res)
+
+    # Add all unmatched results as is.
+    def add_unmatched(x, match_to_x):
+        c[res_key].extend(
+            x[res_key][i]
+            for i in range(len(match_to_x))
+            if match_to_x[i] < 0
+        )
+
+    add_unmatched(a, match_to_a)
+    add_unmatched(b, match_to_b)
+
+    return c
+
+
+def merge_two_files(path0, path1, path_out):
+    """
+    Merges two .jsonl files. The files must contain the same number
+    of samples, one JSON-encoded sample per line. The first sample
+    of the first file will be merged with the first sample of the second
+    file, etc.
+    """
+    a = read_samples(path0)
+    b = read_samples(path1)
+    if len(a) != len(b):
+        raise Exception(f"Different sizes of samples are not supported: {len(a)} and {len(b)}")
+
+    result = [merge_two_samples(a[i], b[i], i+1) for i in range(len(a))]
+
+    with open(path_out, "w") as f:
+        for sample in result:
+            f.write(json.dumps(sample) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Utilities to merge assessed samples from different assessors")
+    parser.add_argument("--input0", required=True, dest="input0", help="Path to the first input file")
+    parser.add_argument("--input1", required=True, dest="input1", help="Path to the second input file")
+    parser.add_argument("--output", required=True, dest="output", help="Path to the output file")
+    args = parser.parse_args()
+
+    merge_two_files(args.input0, args.input1, args.output)
+
+
+if __name__ == "__main__":
+    main()

libs/search/search_quality/sample.cpp

@@ -0,0 +1,307 @@
+#include "search/search_quality/sample.hpp"
+
+#include "search/search_params.hpp"
+#include "search/search_quality/helpers_json.hpp"
+
+#include "indexer/feature.hpp"
+#include "indexer/feature_algo.hpp"
+#include "indexer/feature_data.hpp"
+
+#include "base/logging.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+#include <memory>
+#include <sstream>
+
+namespace search
+{
+using namespace base;
+using namespace std;
+
+namespace
+{
+bool LessRect(m2::RectD const & lhs, m2::RectD const & rhs)
+{
+  if (lhs.minX() != rhs.minX())
+    return lhs.minX() < rhs.minX();
+  if (lhs.minY() != rhs.minY())
+    return lhs.minY() < rhs.minY();
+  if (lhs.maxX() != rhs.maxX())
+    return lhs.maxX() < rhs.maxX();
+  if (lhs.maxY() != rhs.maxY())
+    return lhs.maxY() < rhs.maxY();
+  return false;
+}
+
+template <typename T>
+bool Less(vector<T> lhs, vector<T> rhs)
+{
+  sort(lhs.begin(), lhs.end());
+  sort(rhs.begin(), rhs.end());
+  return lhs < rhs;
+}
+
+template <typename T>
+bool Equal(vector<T> lhs, vector<T> rhs)
+{
+  sort(lhs.begin(), lhs.end());
+  sort(rhs.begin(), rhs.end());
+  return lhs == rhs;
+}
+}  // namespace
+
+// static
+Sample::Result Sample::Result::Build(FeatureType & ft, Relevance relevance)
+{
+  Sample::Result r;
+  r.m_pos = feature::GetCenter(ft);
+  r.m_name = strings::MakeUniString(ft.GetReadableName());
+  r.m_houseNumber = ft.GetHouseNumber();
+  r.m_types = feature::TypesHolder(ft).ToObjectNames();
+  r.m_relevance = relevance;
+  return r;
+}
+
+bool Sample::Result::operator<(Sample::Result const & rhs) const
+{
+  if (m_pos != rhs.m_pos)
+    return m_pos < rhs.m_pos;
+  if (m_name != rhs.m_name)
+    return m_name < rhs.m_name;
+  if (m_houseNumber != rhs.m_houseNumber)
+    return m_houseNumber < rhs.m_houseNumber;
+  if (m_relevance != rhs.m_relevance)
+    return m_relevance < rhs.m_relevance;
+  return Less(m_types, rhs.m_types);
+}
+
+bool Sample::Result::operator==(Sample::Result const & rhs) const
+{
+  // Note: Strict equality for points and viewports.
+  return m_pos == rhs.m_pos && m_name == rhs.m_name && m_houseNumber == rhs.m_houseNumber &&
+         Equal(m_types, rhs.m_types) && m_relevance == rhs.m_relevance;
+}
+
+bool Sample::DeserializeFromJSON(string const & jsonStr)
+{
+  try
+  {
+    base::Json root(jsonStr.c_str());
+    DeserializeFromJSONImpl(root.get());
+    return true;
+  }
+  catch (base::Json::Exception const & e)
+  {
+    LOG(LWARNING, ("Can't parse sample:", e.Msg(), jsonStr));
+  }
+  return false;
+}
+
+base::JSONPtr Sample::SerializeToJSON() const
+{
+  auto json = base::NewJSONObject();
+  SerializeToJSONImpl(*json);
+  return json;
+}
+
+bool Sample::operator<(Sample const & rhs) const
+{
+  if (m_query != rhs.m_query)
+    return m_query < rhs.m_query;
+  if (m_locale != rhs.m_locale)
+    return m_locale < rhs.m_locale;
+  if (m_pos != rhs.m_pos)
+    return m_pos < rhs.m_pos;
+  if (m_viewport != rhs.m_viewport)
+    return LessRect(m_viewport, rhs.m_viewport);
+  if (!Equal(m_results, rhs.m_results))
+    return Less(m_results, rhs.m_results);
+  return Less(m_relatedQueries, rhs.m_relatedQueries);
+}
+
+bool Sample::operator==(Sample const & rhs) const
+{
+  return !(*this < rhs) && !(rhs < *this);
+}
+
+// static
+bool Sample::DeserializeFromJSONLines(string const & lines, vector<Sample> & samples)
+{
+  istringstream is(lines);
+  string line;
+  vector<Sample> result;
+
+  while (getline(is, line))
+  {
+    if (line.empty())
+      continue;
+
+    Sample sample;
+    if (!sample.DeserializeFromJSON(line))
+      return false;
+    result.emplace_back(std::move(sample));
+  }
+
+  samples.insert(samples.end(), result.begin(), result.end());
+  return true;
+}
+
+// static
+void Sample::SerializeToJSONLines(vector<Sample> const & samples, string & lines)
+{
+  for (auto const & sample : samples)
+  {
+    unique_ptr<char, JSONFreeDeleter> buffer(json_dumps(sample.SerializeToJSON().get(), JSON_COMPACT));
+    lines.append(buffer.get());
+    lines.push_back('\n');
+  }
+}
+
+void Sample::DeserializeFromJSONImpl(json_t * root)
+{
+  FromJSONObject(root, "query", m_query);
+  FromJSONObject(root, "locale", m_locale);
+  FromJSONObjectOptional(root, "position", m_pos);
+  FromJSONObject(root, "viewport", m_viewport);
+  FromJSONObjectOptional(root, "results", m_results);
+  FromJSONObjectOptional(root, "related_queries", m_relatedQueries);
+  FromJSONObjectOptionalField(root, "useless", m_useless);
+}
+
+void Sample::SerializeToJSONImpl(json_t & root) const
+{
+  ToJSONObject(root, "query", m_query);
+  ToJSONObject(root, "locale", m_locale);
+  ToJSONObject(root, "position", m_pos);
+  ToJSONObject(root, "viewport", m_viewport);
+  ToJSONObject(root, "results", m_results);
+  ToJSONObject(root, "related_queries", m_relatedQueries);
+  if (m_useless)
+    ToJSONObject(root, "useless", m_useless);
+}
+
+void Sample::FillSearchParams(search::SearchParams & params) const
+{
+  params.m_query = strings::ToUtf8(m_query);
+  params.m_inputLocale = m_locale;
+  params.m_viewport = m_viewport;
+  params.m_mode = Mode::Everywhere;
+  params.m_position = m_pos.value_or(m2::PointD());
+  params.m_needAddress = true;
+  params.m_suggestsEnabled = false;
+  params.m_needHighlighting = false;
+  params.m_useDebugInfo = true;  // for RankingInfo printing
+}
+
+void FromJSONObject(json_t * root, char const * field, Sample::Result::Relevance & relevance)
+{
+  string r;
+  FromJSONObject(root, field, r);
+  if (r == "harmful")
+    relevance = search::Sample::Result::Relevance::Harmful;
+  else if (r == "irrelevant")
+    relevance = search::Sample::Result::Relevance::Irrelevant;
+  else if (r == "relevant")
+    relevance = search::Sample::Result::Relevance::Relevant;
+  else if (r == "vital")
+    relevance = search::Sample::Result::Relevance::Vital;
+  else
+    CHECK(false, ("Unknown relevance:", r));
+}
+
+void ToJSONObject(json_t & root, char const * field, Sample::Result::Relevance relevance)
+{
+  using Relevance = Sample::Result::Relevance;
+
+  string r;
+  switch (relevance)
+  {
+  case Relevance::Harmful: r = "harmful"; break;
+  case Relevance::Irrelevant: r = "irrelevant"; break;
+  case Relevance::Relevant: r = "relevant"; break;
+  case Relevance::Vital: r = "vital"; break;
+  }
+
+  json_object_set_new(&root, field, json_string(r.c_str()));
+}
+
+void FromJSONObject(json_t * root, string const & field, Sample::Result::Relevance & relevance)
+{
+  FromJSONObject(root, field.c_str(), relevance);
+}
+
+void ToJSONObject(json_t & root, string const & field, Sample::Result::Relevance relevance)
+{
+  ToJSONObject(root, field.c_str(), relevance);
+}
+
+void FromJSON(json_t * root, Sample::Result & result)
+{
+  FromJSONObject(root, "position", result.m_pos);
+  FromJSONObject(root, "name", result.m_name);
+  FromJSONObject(root, "houseNumber", result.m_houseNumber);
+  FromJSONObject(root, "types", result.m_types);
+  FromJSONObject(root, "relevancy", result.m_relevance);
+}
+
+base::JSONPtr ToJSON(Sample::Result const & result)
+{
+  auto root = base::NewJSONObject();
+  ToJSONObject(*root, "position", result.m_pos);
+  ToJSONObject(*root, "name", result.m_name);
+  ToJSONObject(*root, "houseNumber", result.m_houseNumber);
+  ToJSONObject(*root, "types", result.m_types);
+  ToJSONObject(*root, "relevancy", result.m_relevance);
+  return root;
+}
+
+string DebugPrint(Sample::Result::Relevance r)
+{
+  switch (r)
+  {
+  case Sample::Result::Relevance::Harmful: return "Harmful";
+  case Sample::Result::Relevance::Irrelevant: return "Irrelevant";
+  case Sample::Result::Relevance::Relevant: return "Relevant";
+  case Sample::Result::Relevance::Vital: return "Vital";
+  }
+  return "Unknown";
+}
+
+string DebugPrint(Sample::Result const & r)
+{
+  ostringstream oss;
+  oss << "relevance: " << DebugPrint(r.m_relevance) << " ";
+  oss << "name: " << DebugPrint(r.m_name) << " ";
+  oss << "house number: " << r.m_houseNumber << " ";
+  oss << "pos: " << DebugPrint(r.m_pos) << " ";
+  oss << "types: [";
+  for (size_t i = 0; i < r.m_types.size(); ++i)
+  {
+    if (i > 0)
+      oss << " ";
+    oss << r.m_types[i];
+  }
+  oss << "]";
+  return oss.str();
+}
+
+string DebugPrint(Sample const & s)
+{
+  ostringstream oss;
+  oss << "[";
+  oss << "query: " << DebugPrint(s.m_query) << ", ";
+  oss << "locale: " << s.m_locale << ", ";
+  oss << "pos: " << (s.m_pos ? DebugPrint(*s.m_pos) : "null") << ", ";
+  oss << "viewport: " << DebugPrint(s.m_viewport) << ", ";
+  oss << "results: [";
+  for (size_t i = 0; i < s.m_results.size(); ++i)
+  {
+    if (i > 0)
+      oss << ", ";
+    oss << DebugPrint(s.m_results[i]);
+  }
+  oss << "]";
+  return oss.str();
+}
+}  // namespace search

libs/search/search_quality/sample.hpp

@@ -0,0 +1,109 @@
+#pragma once
+
+#include "geometry/point2d.hpp"
+#include "geometry/rect2d.hpp"
+
+#include "base/string_utils.hpp"
+
+#include "cppjansson/cppjansson.hpp"
+
+#include <optional>
+#include <string>
+#include <vector>
+
+class FeatureType;
+
+namespace search
+{
+struct SearchParams;
+
+struct Sample
+{
+  struct Result
+  {
+    enum class Relevance
+    {
+      // A result that should not be present and it's hard (for the user)
+      // to explain why it is there, i.e. it is a waste of time even
+      // to try to understand what this result is about.
+      Harmful,
+
+      // A result that is irrelevant to the query but at
+      // least it is easy to explain why it showed up.
+      Irrelevant,
+
+      // A result that is relevant to the query.
+      Relevant,
+
+      // A result that definetely should be present, preferably
+      // at a position close to the beginning.
+      Vital
+    };
+
+    static Result Build(FeatureType & ft, Relevance relevance);
+
+    bool operator<(Result const & rhs) const;
+
+    bool operator==(Result const & rhs) const;
+
+    m2::PointD m_pos = m2::PointD(0, 0);
+
+    strings::UniString m_name;
+    std::string m_houseNumber;
+    std::vector<std::string> m_types;  // OMaps types, not OSM types.
+    Relevance m_relevance = Relevance::Irrelevant;
+  };
+
+  bool DeserializeFromJSON(std::string const & jsonStr);
+  base::JSONPtr SerializeToJSON() const;
+
+  static bool DeserializeFromJSONLines(std::string const & lines, std::vector<Sample> & samples);
+  static void SerializeToJSONLines(std::vector<Sample> const & samples, std::string & lines);
+
+  bool operator<(Sample const & rhs) const;
+
+  bool operator==(Sample const & rhs) const;
+
+  void DeserializeFromJSONImpl(json_t * root);
+  void SerializeToJSONImpl(json_t & root) const;
+
+  void FillSearchParams(search::SearchParams & params) const;
+
+  strings::UniString m_query;
+  std::string m_locale;
+  std::optional<m2::PointD> m_pos;
+  m2::RectD m_viewport = m2::RectD(0, 0, 0, 0);
+  std::vector<Result> m_results;
+  std::vector<strings::UniString> m_relatedQueries;
+
+  // A useless sample is usually a result of the user exploring
+  // the search engine without a clear search intent or a sample
+  // that cannot be assessed properly using only the data available
+  // to the engine (for example, related queries may help a lot but
+  // are not expected to be available, or local knowledge of the area
+  // is needed).
+  // More examples:
+  // * A sample whose requests is precisely about a particular street
+  //   in a particular city is useless if the assessor is sure that
+  //   there is no such street in this city.
+  // * On the other hand, if there is such a street (or, more often,
+  //   a building) as indicated by other data sources but the engine
+  //   still could not find it because of its absense in our
+  //   data, the sample is NOT useless.
+  bool m_useless = false;
+};
+
+void FromJSONObject(json_t * root, char const * field, Sample::Result::Relevance & relevance);
+void ToJSONObject(json_t & root, char const * field, Sample::Result::Relevance relevance);
+void FromJSONObject(json_t * root, std::string const & field, Sample::Result::Relevance & relevance);
+void ToJSONObject(json_t & root, std::string const & field, Sample::Result::Relevance relevance);
+
+void FromJSON(json_t * root, Sample::Result & result);
+base::JSONPtr ToJSON(Sample::Result const & result);
+
+std::string DebugPrint(Sample::Result::Relevance r);
+
+std::string DebugPrint(Sample::Result const & r);
+
+std::string DebugPrint(Sample const & s);
+}  // namespace search

libs/search/search_quality/samples_generation_tool/CMakeLists.txt

@@ -0,0 +1,11 @@
+project(samples_generation_tool)
+
+set(SRC samples_generation_tool.cpp)
+
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  search_quality
+  search_tests_support
+  gflags::gflags
+)

libs/search/search_quality/samples_generation_tool/samples_generation_tool.cpp

@@ -0,0 +1,458 @@
+#include "search/search_quality/helpers.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "search/categories_cache.hpp"
+#include "search/house_numbers_matcher.hpp"
+#include "search/mwm_context.hpp"
+#include "search/reverse_geocoder.hpp"
+
+#include "indexer/classificator_loader.hpp"
+#include "indexer/data_source.hpp"
+#include "indexer/feature_algo.hpp"
+#include "indexer/ftypes_matcher.hpp"
+
+#include "platform/platform_tests_support/helpers.hpp"
+
+#include "platform/platform.hpp"
+
+#include "geometry/mercator.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+#include <cstring>  // strlen
+#include <fstream>
+#include <map>
+#include <random>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <gflags/gflags.h>
+
+using namespace search::search_quality;
+using namespace search;
+using namespace std;
+
+double constexpr kMaxDistanceToObjectM = 7500.0;
+double constexpr kMinViewportSizeM = 100.0;
+double constexpr kMaxViewportSizeM = 5000.0;
+size_t constexpr kMaxSamplesPerMwm = 20;
+
+DEFINE_string(data_path, "", "Path to data directory (resources dir).");
+DEFINE_string(mwm_path, "", "Path to mwm files (writable dir).");
+DEFINE_string(out_buildings_path, "buildings.json", "Path to output file for buildings samples.");
+DEFINE_string(out_cafes_path, "cafes.json", "Path to output file for cafes samples.");
+DEFINE_double(max_distance_to_object, kMaxDistanceToObjectM, "Maximal distance from user position to object (meters).");
+DEFINE_double(min_viewport_size, kMinViewportSizeM, "Minimal size of viewport (meters).");
+DEFINE_double(max_viewport_size, kMaxViewportSizeM, "Maximal size of viewport (meters).");
+DEFINE_uint64(max_samples_per_mwm, kMaxSamplesPerMwm,
+              "Maximal number of samples of each type (buildings/cafes) per mwm.");
+DEFINE_bool(add_misprints, false, "Add random misprints.");
+DEFINE_bool(add_cafe_address, false, "Add address.");
+DEFINE_bool(add_cafe_type, false, "Add cafe type (restaurant/cafe/bar) in local language.");
+
+std::random_device g_rd;
+mt19937 g_rng(g_rd());
+
+enum class RequestType
+{
+  Building,
+  Cafe
+};
+
+bool GetRandomBool()
+{
+  std::uniform_int_distribution<> dis(0, 1);
+  return dis(g_rng) == 0;
+}
+
+// Leaves first letter as is.
+void EraseRandom(strings::UniString & uni)
+{
+  if (uni.size() <= 1)
+    return;
+
+  uniform_int_distribution<size_t> dis(1, uni.size() - 1);
+  auto const it = uni.begin() + dis(g_rng);
+  uni.erase(it, it + 1);
+}
+
+// Leaves first letter as is.
+void SwapRandom(strings::UniString & uni)
+{
+  if (uni.size() <= 2)
+    return;
+
+  uniform_int_distribution<size_t> dis(1, uni.size() - 2);
+  auto const index = dis(g_rng);
+  swap(uni[index], uni[index + 1]);
+}
+
+void AddRandomMisprint(strings::UniString & str)
+{
+  // todo(@t.yan): Disable for hieroglyphs, consider implementing InsertRandom.
+  if (GetRandomBool())
+    return EraseRandom(str);
+  return SwapRandom(str);
+}
+
+void AddMisprints(string & str)
+{
+  if (!FLAGS_add_misprints)
+    return;
+
+  auto tokens = strings::Tokenize(str, " -&");
+  str.clear();
+  for (size_t i = 0; i < tokens.size(); ++i)
+  {
+    auto const & token = tokens[i];
+    auto uni = strings::MakeUniString(token);
+    if (uni.size() > 4 && g_rng() % 4)
+      AddRandomMisprint(uni);
+    if (uni.size() > 8 && g_rng() % 4)
+      AddRandomMisprint(uni);
+
+    str += strings::ToUtf8(uni);
+    if (i != tokens.size() - 1)
+      str += " ";
+  }
+}
+
+map<string, vector<string>> const kStreetSynonyms = {{"улица", {"ул", "у"}},
+                                                     {"проспект", {"пр-т", "пр", "пркт", "прт", "пр-кт"}},
+                                                     {"переулок", {"пер"}},
+                                                     {"проезд", {"пр-д", "пр", "прд"}},
+                                                     {"аллея", {"ал"}},
+                                                     {"бульвар", {"б-р", "бр"}},
+                                                     {"набережная", {"наб", "наб-я"}},
+                                                     {"шоссе", {"шос"}},
+                                                     {"вyлица", {"вул"}},
+                                                     {"площадь", {"пл", "площ"}},
+                                                     {"тупик", {"туп"}},
+                                                     {"street", {"str", "st"}},
+                                                     {"avenue", {"ave", "av"}},
+                                                     {"boulevard", {"bld", "blv", "bv", "blvd"}},
+                                                     {"drive", {"dr"}},
+                                                     {"highway", {"hw", "hwy"}},
+                                                     {"road", {"rd"}},
+                                                     {"square", {"sq"}}};
+
+void ModifyStreet(string & str)
+{
+  auto tokens = strings::Tokenize<std::string>(str, " -&");
+  str.clear();
+
+  auto const isStreetSynonym = [](string const & s) { return kStreetSynonyms.find(s) != kStreetSynonyms.end(); };
+
+  auto const synonymIt = find_if(tokens.begin(), tokens.end(), isStreetSynonym);
+  if (synonymIt != tokens.end() && find_if(synonymIt + 1, tokens.end(), isStreetSynonym) == tokens.end())
+  {
+    // Only one street synonym.
+    if (GetRandomBool())
+    {
+      if (GetRandomBool())
+      {
+        auto const & synonyms = kStreetSynonyms.at(*synonymIt);
+        uniform_int_distribution<size_t> dis(0, synonyms.size() - 1);
+        *synonymIt = synonyms[dis(g_rng)];
+      }
+      else
+      {
+        tokens.erase(synonymIt);
+      }
+    }
+    // Else leave as is.
+  }
+
+  if (tokens.empty())
+    return;
+
+  str = strings::JoinStrings(tokens, " ");
+  AddMisprints(str);
+}
+
+void ModifyHouse(uint8_t lang, string & str)
+{
+  if (str.empty())
+    return;
+
+  if (lang == StringUtf8Multilang::GetLangIndex("ru") && isdigit(str[0]))
+  {
+    uniform_int_distribution<size_t> dis(0, 4);
+    auto const r = dis(g_rng);
+    if (r == 0)
+      str = "д " + str;
+    if (r == 1)
+      str = "д" + str;
+    if (r == 2)
+      str = "дом " + str;
+    if (r == 3)
+      str = "д. " + str;
+    // Else leave housenumber as is.
+  }
+}
+
+m2::PointD GenerateNearbyPosition(m2::PointD const & point)
+{
+  auto const maxDistance = FLAGS_max_distance_to_object;
+  uniform_real_distribution<double> dis(-maxDistance, maxDistance);
+  return mercator::GetSmPoint(point, dis(g_rng) /* dX */, dis(g_rng) /* dY */);
+}
+
+m2::RectD GenerateNearbyViewport(m2::PointD const & point)
+{
+  uniform_real_distribution<double> dis(FLAGS_min_viewport_size, FLAGS_max_viewport_size);
+  return mercator::RectByCenterXYAndSizeInMeters(GenerateNearbyPosition(point), dis(g_rng));
+}
+
+bool GetBuildingInfo(FeatureType & ft, search::ReverseGeocoder const & coder, string & street)
+{
+  std::string const & hn = ft.GetHouseNumber();
+  if (hn.empty() || !search::house_numbers::LooksLikeHouseNumber(hn, false /* prefix */))
+    return false;
+
+  street = coder.GetFeatureStreetName(ft);
+  if (street.empty())
+    return false;
+
+  return true;
+}
+
+bool GetCafeInfo(FeatureType & ft, search::ReverseGeocoder const & coder, string & street, uint32_t & cafeType,
+                 string_view & name)
+{
+  if (!ft.HasName())
+    return false;
+
+  if (!ft.GetNames().GetString(StringUtf8Multilang::kDefaultCode, name))
+    return false;
+
+  for (auto const t : feature::TypesHolder(ft))
+  {
+    if (ftypes::IsEatChecker::Instance()(t))
+    {
+      cafeType = t;
+      street = coder.GetFeatureStreetName(ft);
+      return !street.empty();
+    }
+  }
+
+  return false;
+}
+
+string ModifyAddress(string street, string house, uint8_t lang)
+{
+  if (street.empty() || house.empty())
+    return "";
+
+  ModifyStreet(street);
+  ModifyHouse(lang, house);
+
+  if (GetRandomBool())
+    return street + " " + house;
+
+  return house + " " + street;
+}
+
+string CombineRandomly(string const & mandatory, string const & optional)
+{
+  if (optional.empty() || GetRandomBool())
+    return mandatory;
+  if (GetRandomBool())
+    return optional + " " + mandatory;
+  return mandatory + " " + optional;
+}
+
+void ModifyCafe(string const & name, string const & type, string & out)
+{
+  out = FLAGS_add_cafe_type ? CombineRandomly(name, type) : name;
+  AddMisprints(out);
+}
+
+string_view GetLocalizedCafeType(unordered_map<uint32_t, StringUtf8Multilang> const & typesTranslations, uint32_t type,
+                                 uint8_t lang)
+{
+  auto const it = typesTranslations.find(type);
+  if (it == typesTranslations.end())
+    return {};
+  string_view translation;
+  if (it->second.GetString(lang, translation))
+    return translation;
+  it->second.GetString(StringUtf8Multilang::kEnglishCode, translation);
+  return translation;
+}
+
+optional<Sample> GenerateRequest(FeatureType & ft, search::ReverseGeocoder const & coder,
+                                 unordered_map<uint32_t, StringUtf8Multilang> const & typesTranslations,
+                                 vector<int8_t> const & mwmLangCodes, RequestType requestType)
+{
+  string street;
+  string cafeStr;
+  auto const lang = !mwmLangCodes.empty() ? mwmLangCodes[0] : StringUtf8Multilang::kEnglishCode;
+
+  switch (requestType)
+  {
+  case RequestType::Building:
+  {
+    if (!GetBuildingInfo(ft, coder, street))
+      return {};
+    break;
+  }
+  case RequestType::Cafe:
+  {
+    uint32_t type;
+    string_view name;
+    if (!GetCafeInfo(ft, coder, street, type, name))
+      return {};
+
+    auto const cafeType = GetLocalizedCafeType(typesTranslations, type, lang);
+    ModifyCafe(std::string(name), std::string(cafeType), cafeStr);
+    break;
+  }
+  }
+
+  auto const featureCenter = feature::GetCenter(ft);
+  auto const address = ModifyAddress(std::move(street), ft.GetHouseNumber(), lang);
+  auto query = address;
+  if (!cafeStr.empty())
+    query = FLAGS_add_cafe_address ? CombineRandomly(cafeStr, address) : cafeStr;
+
+  Sample sample;
+  sample.m_query = strings::MakeUniString(query);
+  sample.m_locale = StringUtf8Multilang::GetLangByCode(lang);
+  sample.m_pos = GenerateNearbyPosition(featureCenter);
+  sample.m_viewport = GenerateNearbyViewport(featureCenter);
+  sample.m_results.push_back(Sample::Result::Build(ft, Sample::Result::Relevance::Vital));
+
+  return sample;
+}
+
+unordered_map<uint32_t, StringUtf8Multilang> ParseStrings()
+{
+  auto const stringsFile = base::JoinPath(GetPlatform().ResourcesDir(), "strings", "types_strings.txt");
+  ifstream s(stringsFile);
+  CHECK(s.is_open(), ("Cannot open", stringsFile));
+
+  // Skip the first [[Types]] line.
+  string line;
+  getline(s, line);
+
+  uint32_t type = 0;
+  auto const typePrefixSize = strlen("[type.");
+  auto const typePostfixSize = strlen("]");
+  unordered_map<uint32_t, StringUtf8Multilang> typesTranslations;
+  while (s.good())
+  {
+    getline(s, line);
+    strings::Trim(line);
+
+    // Allow for comments starting with '#' character.
+    if (line.empty() || line[0] == '#')
+      continue;
+
+    // New type.
+    if (line[0] == '[')
+    {
+      CHECK_GREATER(line.size(), typePrefixSize + typePostfixSize, (line));
+      auto typeString = line.substr(typePrefixSize, line.size() - typePrefixSize - typePostfixSize);
+      auto const tokens = strings::Tokenize(typeString, ".");
+      type = classif().GetTypeByPath(tokens);
+    }
+    else
+    {
+      // We can not get initial type by types_strings key for some types like
+      // "amenity parking multi-storey" which is stored like amenity.parking.multi.storey
+      // and can not be tokenized correctly.
+      if (type == 0)
+        continue;
+      auto const pos = line.find("=");
+      CHECK_NOT_EQUAL(pos, string::npos, ());
+      auto lang = line.substr(0, pos);
+      strings::Trim(lang);
+      auto translation = line.substr(pos + 1);
+      strings::Trim(translation);
+      typesTranslations[type].AddString(lang, translation);
+    }
+  }
+  return typesTranslations;
+}
+
+int main(int argc, char * argv[])
+{
+  platform::tests_support::ChangeMaxNumberOfOpenFiles(kMaxOpenFiles);
+
+  gflags::SetUsageMessage("Samples generation tool.");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  SetPlatformDirs(FLAGS_data_path, FLAGS_mwm_path);
+
+  ofstream buildingsOut;
+  buildingsOut.open(FLAGS_out_buildings_path);
+  CHECK(buildingsOut.is_open(), ("Can't open output file", FLAGS_out_buildings_path));
+
+  ofstream cafesOut;
+  cafesOut.open(FLAGS_out_cafes_path);
+  CHECK(cafesOut.is_open(), ("Can't open output file", FLAGS_out_cafes_path));
+
+  classificator::Load();
+  FrozenDataSource dataSource;
+  InitDataSource(dataSource, "" /* mwmListPath */);
+  search::ReverseGeocoder const coder(dataSource);
+  auto const typesTranslations = ParseStrings();
+
+  vector<shared_ptr<MwmInfo>> mwmInfos;
+  dataSource.GetMwmsInfo(mwmInfos);
+  for (auto const & mwmInfo : mwmInfos)
+  {
+    MwmSet::MwmId const mwmId(mwmInfo);
+    LOG(LINFO, ("Start generation for", mwmId));
+
+    vector<int8_t> mwmLangCodes;
+    mwmInfo->GetRegionData().GetLanguages(mwmLangCodes);
+
+    auto handle = dataSource.GetMwmHandleById(mwmId);
+    auto & value = *handle.GetValue();
+    // WorldCoasts.
+    if (!value.HasSearchIndex())
+      continue;
+
+    MwmContext const mwmContext(std::move(handle));
+    base::Cancellable const cancellable;
+    FeaturesLoaderGuard g(dataSource, mwmId);
+
+    auto generate = [&](ftypes::BaseChecker const & checker, RequestType type, ofstream & out)
+    {
+      CategoriesCache cache(checker, cancellable);
+      auto features = cache.Get(mwmContext);
+
+      vector<uint32_t> fids;
+      features.ForEach([&fids](uint64_t fid) { fids.push_back(base::asserted_cast<uint32_t>(fid)); });
+      shuffle(fids.begin(), fids.end(), g_rng);
+
+      size_t numSamples = 0;
+      for (auto const fid : fids)
+      {
+        if (numSamples >= FLAGS_max_samples_per_mwm)
+          break;
+
+        auto ft = g.GetFeatureByIndex(fid);
+        CHECK(ft, ());
+        auto const sample = GenerateRequest(*ft, coder, typesTranslations, mwmLangCodes, type);
+        if (sample)
+        {
+          string json;
+          Sample::SerializeToJSONLines({*sample}, json);
+          out << json;
+          ++numSamples;
+        }
+      }
+    };
+
+    generate(ftypes::IsBuildingChecker::Instance(), RequestType::Building, buildingsOut);
+    generate(ftypes::IsEatChecker::Instance(), RequestType::Cafe, cafesOut);
+  }
+  return 0;
+}

libs/search/search_quality/scoring_model.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+
+from math import exp, log
+from scipy.stats import pearsonr, t
+from sklearn import svm
+from sklearn.model_selection import GridSearchCV, KFold
+from sklearn.utils import resample
+import argparse
+import collections
+import itertools
+import numpy as np
+import pandas as pd
+import random
+import sys
+
+
+MAX_DISTANCE_METERS = 2e6
+MAX_RANK = 255.0
+MAX_POPULARITY = 255.0
+RELEVANCES = {'Harmful': -3, 'Irrelevant': 0, 'Relevant': 1, 'Vital': 3}
+NAME_SCORES = ['Zero', 'Substring', 'Prefix', 'Full Match']
+SEARCH_TYPES = ['SUBPOI', 'COMPLEX_POI', 'Building', 'Street', 'Unclassified', 'Village', 'City', 'State', 'Country']
+RESULT_TYPES = ['TransportMajor', 'TransportLocal', 'Eat', 'Hotel', 'Attraction', 'Service', 'General']
+FEATURES = ['DistanceToPivot', 'Rank', 'Popularity', 'Rating', 'FalseCats', 'ErrorsMade', 'MatchedFraction',
+            'AllTokensUsed', 'ExactCountryOrCapital'] + NAME_SCORES + SEARCH_TYPES + RESULT_TYPES
+
+BOOTSTRAP_ITERATIONS = 10000
+
+
+def transform_name_score(value, categories_match):
+    if categories_match == 1:
+        return 'Zero'
+    else:
+        return value
+
+
+def normalize_data(data):
+    transform_distance = lambda v: min(v, MAX_DISTANCE_METERS) / MAX_DISTANCE_METERS
+
+    data['DistanceToPivot'] = data['DistanceToPivot'].apply(transform_distance)
+    data['Rank'] = data['Rank'].apply(lambda v: v / MAX_RANK)
+    data['Popularity'] = data['Popularity'].apply(lambda v: v / MAX_POPULARITY)
+    data['Relevance'] = data['Relevance'].apply(lambda v: RELEVANCES[v])
+
+    cats = data['PureCats'].combine(data['FalseCats'], max)
+
+    # TODO (@y, @m): do forward/backward/subset selection of features
+    # instead of this merging.  It would be great to conduct PCA on
+    # the features too.
+    data['NameScore'] = data['NameScore'].combine(cats, transform_name_score)
+
+    # Adds dummy variables to data for NAME_SCORES.
+    for ns in NAME_SCORES:
+        data[ns] = data['NameScore'].apply(lambda v: int(ns == v))
+
+    # Adds dummy variables to data for SEARCH_TYPES.
+
+    # We unify BUILDING with COMPLEX_POI and SUBPOI here, as we don't have enough
+    # training data to distinguish between them.  Remove following
+    # line as soon as the model will be changed or we will have enough
+    # training data.
+    data['SearchType'] = data['SearchType'].apply(lambda v: v if v != 'Building' and v != 'COMPLEX_POI' else 'SUBPOI')
+    for st in SEARCH_TYPES:
+        data[st] = data['SearchType'].apply(lambda v: int(st == v))
+
+    # Adds dummy variables to data for RESULT_TYPES.
+    for rt in RESULT_TYPES:
+        data[rt] = data['ResultType'].apply(lambda v: int(rt == v))
+
+
+def compute_ndcg(relevances):
+    """
+    Computes NDCG (Normalized Discounted Cumulative Gain) for a given
+    array of scores.
+    """
+
+    dcg = sum(r / log(2 + i, 2) for i, r in enumerate(relevances))
+    dcg_norm = sum(r / log(2 + i, 2) for i, r in enumerate(sorted(relevances, reverse=True)))
+    return dcg / dcg_norm if dcg_norm != 0 else 0
+
+
+def compute_ndcgs_without_ws(data):
+    """
+    Computes NDCG (Normalized Discounted Cumulative Gain) for a given
+    data. Returns an array of ndcg scores in the shape [num groups of
+    features].
+    """
+
+    grouped = data.groupby(data['SampleId'], sort=False).groups
+
+    ndcgs = []
+    for id in grouped:
+        indices = grouped[id]
+        relevances = np.array(data.ix[indices]['Relevance'])
+        ndcgs.append(compute_ndcg(relevances))
+
+    return ndcgs
+
+
+def compute_ndcgs_for_ws(data, ws):
+    """
+    Computes NDCG (Normalized Discounted Cumulative Gain) for a given
+    data and an array of coeffs in a linear model. Returns an array of
+    ndcg scores in the shape [num groups of features].
+    """
+
+    data_scores = np.array([np.dot(data.ix[i][FEATURES], ws) for i in data.index])
+    grouped = data.groupby(data['SampleId'], sort=False).groups
+
+    ndcgs = []
+    for id in grouped:
+        indices = grouped[id]
+
+        relevances = np.array(data.ix[indices]['Relevance'])
+        scores = data_scores[indices]
+
+        # Reoders relevances in accordance with decreasing scores.
+        relevances = relevances[scores.argsort()[::-1]]
+        ndcgs.append(compute_ndcg(relevances))
+
+    return ndcgs
+
+
+def transform_data(data):
+    """
+    By a given data computes x and y that can be used as an input to a
+    linear SVM.
+    """
+
+    grouped = data.groupby(data['SampleId'], sort=False)
+
+    xs, ys = [], []
+
+    # k is used to create a balanced samples set for better linear
+    # separation.
+    k = 1
+    for _, group in grouped:
+        features, relevances = group[FEATURES], group['Relevance']
+
+        n, total = len(group), 0
+        for _, (i, j) in enumerate(itertools.combinations(range(n), 2)):
+            dr = relevances.iloc[j] - relevances.iloc[i]
+            y = np.sign(dr)
+            if y == 0:
+                continue
+
+            x = np.array(features.iloc[j]) - np.array(features.iloc[i])
+
+            # Need to multiply x by average drop in NDCG when i-th and
+            # j-th are exchanged.
+            x *= abs(dr * (1 / log(j + 2, 2) - 1 / log(i + 2, 2)))
+
+            # This is needed to prevent disbalance in classes sizes.
+            if y != k:
+                x = np.negative(x)
+                y = -y
+
+            xs.append(x)
+            ys.append(y)
+            total += 1
+            k = -k
+
+        # Scales this group of features to equalize different search
+        # queries.
+        for i in range(-1, -total, -1):
+            xs[i] = xs[i] / total
+    return xs, ys
+
+
+def show_pearson_statistics(xs, ys, features):
+    """
+    Shows info about Pearson coefficient between features and
+    relevancy.
+    """
+
+    print('***** Correlation table *****')
+    print('H0 - feature not is correlated with relevancy')
+    print('H1 - feature is correlated with relevancy')
+    print()
+
+    cs, ncs = [], []
+    for i, f in enumerate(features):
+        zs = [x[i] for x in xs]
+        (c, p) = pearsonr(zs, ys)
+
+        correlated = p < 0.05
+        print('{}: pearson={:.3f}, P(H1)={}'.format(f, c, 1 - p))
+        if correlated:
+            cs.append(f)
+        else:
+            ncs.append(f)
+
+    print()
+    print('Correlated:', cs)
+    print('Non-correlated:', ncs)
+
+
+def raw_output(features, ws):
+    """
+    Prints feature-coeff pairs to the standard output.
+    """
+
+    print('{:<20}{}'.format('Feature', 'Value'))
+    print()
+    for f, w in zip(features, ws):
+        print('{:<20}{:.5f}'.format(f, w))
+
+
+def print_const(name, value):
+    print('double constexpr k{} = {:.7f};'.format(name, value))
+
+
+def print_array(name, size, values):
+    print('double constexpr {}[{}] = {{'.format(name, size))
+    print(',\n'.join('  {:.7f} /* {} */'.format(w, f) for (f, w) in values))
+    print('};')
+
+def cpp_output(features, ws):
+    """
+    Prints feature-coeff pairs in the C++-compatible format.
+    """
+
+    ns, st, rt = [], [], []
+
+    for f, w in zip(features, ws):
+        if f in NAME_SCORES:
+            ns.append((f, w))
+        elif f in SEARCH_TYPES:
+            st.append((f, w))
+        elif f in RESULT_TYPES:
+            rt.append((f, w))
+        else:
+            print_const(f, w)
+    print_array('kNameScore', 'NameScore::NAME_SCORE_COUNT', ns)
+    print_array('kType', 'Model::TYPE_COUNT', st)
+    print_array('kResultType', 'base::Underlying(ResultType::Count)', rt)
+
+
+def show_bootstrap_statistics(clf, X, y, features):
+    num_features = len(features)
+
+    coefs = []
+    for i in range(num_features):
+        coefs.append([])
+
+    for _ in range(BOOTSTRAP_ITERATIONS):
+        X_sample, y_sample = resample(X, y)
+        clf.fit(X_sample, y_sample)
+        for i, c in enumerate(get_normalized_coefs(clf)):
+            coefs[i].append(c)
+
+    subpoi_index = features.index('SUBPOI')
+    poi_index = features.index('COMPLEX_POI')
+    building_index = features.index('Building')
+    coefs[building_index] = coefs[subpoi_index]
+    coefs[poi_index] = coefs[subpoi_index]
+
+    intervals = []
+
+    print()
+    print('***** Bootstrap statistics *****')
+    print('{:<20}{:<20}{:<10}{:<10}'.format('Feature', '95% interval', 't-value', 'Pr(>|t|)'))
+    print()
+    for i, cs in enumerate(coefs):
+        values = np.array(cs)
+        lo = np.percentile(values, 2.5)
+        hi = np.percentile(values, 97.5)
+        interval = '({:.3f}, {:.3f})'.format(lo, hi)
+        tv = np.mean(values) / np.std(values)
+        pr = (1.0 - t.cdf(x=abs(tv), df=len(values))) * 0.5
+
+        stv = '{:.3f}'.format(tv)
+        spr = '{:.3f}'.format(pr)
+        print('{:<20}{:<20}{:<10}{:<10}'.format(features[i], interval, stv, spr))
+
+
+def get_normalized_coefs(clf):
+    ws = clf.coef_[0]
+    max_w = max(abs(w) for w in ws)
+    return np.divide(ws, max_w)
+
+
+def main(args):
+    data = pd.read_csv(sys.stdin)
+
+    # Drop categorial requests cause we use different ranking model for them.
+    data.drop(data[data['IsCategorialRequest'] == 1].index, inplace=True)
+    data.reset_index(inplace=True, drop=True)
+    data.drop(columns=['IsCategorialRequest', 'HasName'], inplace=True)
+
+    normalize_data(data)
+
+    ndcgs = compute_ndcgs_without_ws(data);
+    print('Current NDCG: {:.3f}, std: {:.3f}'.format(np.mean(ndcgs), np.std(ndcgs)))
+    print()
+
+    xs, ys = transform_data(data)
+
+    clf = svm.LinearSVC(random_state=args.seed)
+    cv = KFold(n_splits=5, shuffle=True, random_state=args.seed)
+
+    # "C" stands for the regularizer constant.
+    grid = {'C': np.power(10.0, np.arange(-5, 6))}
+    gs = GridSearchCV(clf, grid, scoring='roc_auc', cv=cv)
+    gs.fit(xs, ys)
+
+    print('Best params: {}'.format(gs.best_params_))
+
+    ws = get_normalized_coefs(gs.best_estimator_)
+
+    # Following code restores coeffs for merged features.
+    ws[FEATURES.index('Building')] = ws[FEATURES.index('SUBPOI')]
+    ws[FEATURES.index('COMPLEX_POI')] = ws[FEATURES.index('SUBPOI')]
+
+    ndcgs = compute_ndcgs_for_ws(data, ws)
+
+    print('NDCG mean: {:.3f}, std: {:.3f}'.format(np.mean(ndcgs), np.std(ndcgs)))
+    print('ROC AUC: {:.3f}'.format(gs.best_score_))
+
+    if args.pearson:
+        print()
+        show_pearson_statistics(xs, ys, FEATURES)
+
+    print()
+    print('***** Linear model weights *****')
+    if args.cpp:
+        cpp_output(FEATURES, ws)
+    else:
+        raw_output(FEATURES, ws)
+
+    if args.bootstrap:
+        show_bootstrap_statistics(clf, xs, ys, FEATURES)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--seed', help='random seed', type=int)
+    parser.add_argument('--pearson', help='show pearson statistics', action='store_true')
+    parser.add_argument('--cpp', help='generate output in the C++ format', action='store_true')
+    parser.add_argument('--bootstrap', help='show bootstrap confidence intervals', action='store_true')
+    args = parser.parse_args()
+    main(args)

libs/search/search_quality/search_quality_tests/CMakeLists.txt

@@ -0,0 +1,13 @@
+project(search_quality_tests)
+
+set(SRC
+  benchmark_tests.cpp
+  real_mwm_tests.cpp
+  sample_test.cpp
+)
+
+omim_add_test(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  search_quality
+)

libs/search/search_quality/search_quality_tests/benchmark_tests.cpp

@@ -0,0 +1,20 @@
+#include "testing/testing.hpp"
+
+#include "search/search_tests_support/helpers.hpp"
+
+namespace benchmark_tests
+{
+
+using BenchmarkFixture = search::tests_support::SearchTest;
+
+UNIT_CLASS_TEST(BenchmarkFixture, Smoke)
+{
+  RegisterLocalMapsInViewport(mercator::Bounds::FullRect());
+
+  SetViewport({50.1052, 8.6868}, 10000);  // Frankfurt am Main
+
+  auto request = MakeRequest("b");
+  LOG(LINFO, (request->ResponseTime().count()));
+}
+
+}  // namespace benchmark_tests

libs/search/search_quality/search_quality_tests/sample_test.cpp

@@ -0,0 +1,174 @@
+#include "testing/testing.hpp"
+
+#include "search/search_quality/sample.hpp"
+
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+#include <vector>
+
+namespace sample_test
+{
+using search::Sample;
+
+class SampleTest
+{
+public:
+  SampleTest() { Init(); }
+
+protected:
+  void Init();
+
+  Sample m_cuba;
+  Sample m_riga;
+  Sample m_tula;
+};
+
+void SampleTest::Init()
+{
+  m_cuba.m_query = strings::MakeUniString("cuba");
+  m_cuba.m_locale = "en";
+  m_cuba.m_pos = m2::PointD{37.618706, 99.53730574302003};
+  m_cuba.m_viewport = {37.1336, 67.1349, 38.0314, 67.7348};
+  Sample::Result cubaRes;
+  cubaRes.m_name = strings::MakeUniString("Cuba");
+  cubaRes.m_relevance = Sample::Result::Relevance::Relevant;
+  cubaRes.m_types.push_back("place-country");
+  cubaRes.m_pos = {-80.832886, 15.521132748163712};
+  cubaRes.m_houseNumber = "";
+  m_cuba.m_results = {cubaRes};
+  m_cuba.m_relatedQueries = {strings::MakeUniString("Cuba Libre"), strings::MakeUniString("Patria o Muerte")};
+
+  m_riga.m_query = strings::MakeUniString("riga");
+  m_riga.m_locale = "en";
+  m_riga.m_pos = m2::PointD{37.65376, 98.51110651930014};
+  m_riga.m_viewport = {37.5064, 67.0476, 37.7799, 67.304};
+  Sample::Result rigaRes;
+  rigaRes.m_name = strings::MakeUniString("Rīga");
+  rigaRes.m_relevance = Sample::Result::Relevance::Vital;
+  rigaRes.m_types.push_back("place-city-capital-2");
+  rigaRes.m_pos = {24.105186, 107.7819569220319};
+  rigaRes.m_houseNumber = "";
+  m_riga.m_results = {rigaRes, rigaRes};
+
+  m_tula.m_query = strings::MakeUniString("tula");
+  m_tula.m_locale = "en";
+  m_tula.m_pos = {};
+  m_tula.m_viewport = {37.5064, 67.0476, 37.7799, 67.304};
+}
+
+UNIT_CLASS_TEST(SampleTest, Smoke)
+{
+  auto const jsonStr = R"EOF(
+  {
+    "query": "cuba",
+    "locale": "en",
+    "position": {
+      "x": 37.618706,
+      "y": 99.53730574302003
+    },
+    "viewport": {
+      "minx": 37.1336,
+      "miny": 67.1349,
+      "maxx": 38.0314,
+      "maxy": 67.7348
+    },
+    "results": [
+      {
+        "name": "Cuba",
+        "relevancy": "relevant",
+        "types": [
+          "place-country"
+        ],
+        "position": {
+          "x": -80.832886,
+          "y": 15.521132748163712
+        },
+        "houseNumber": ""
+      }
+    ],
+    "related_queries": ["Cuba Libre", "Patria o Muerte"]
+  }
+  )EOF";
+
+  Sample s;
+  TEST(s.DeserializeFromJSON(jsonStr), ());
+  TEST_EQUAL(s, m_cuba, ());
+}
+
+UNIT_CLASS_TEST(SampleTest, BadViewport)
+{
+  auto const jsonStr = R"EOF(
+  {
+    "results": [
+      {
+        "houseNumber": "",
+        "position": {
+          "y": 15.521132748163712,
+          "x": -80.832886
+        },
+        "types": [
+          "place-country"
+        ],
+        "relevancy": "relevant",
+        "name": "Cuba"
+      }
+    ],
+    "viewport": {
+      "maxy": 67.7348,
+      "maxx": 38.0314,
+    },
+    "position": {
+      "y": 99.53730574302003,
+      "x": 37.618706
+    },
+    "locale": "en",
+    "query": "cuba"
+  }
+  )EOF";
+
+  Sample s;
+  TEST(!s.DeserializeFromJSON(jsonStr), ());
+}
+
+UNIT_CLASS_TEST(SampleTest, Arrays)
+{
+  std::string lines;
+  lines.append(
+      R"({"query": "cuba", "locale": "en", "position": {"x": 37.618706, "y": 99.53730574302003}, "viewport": {"minx": 37.1336, "miny": 67.1349, "maxx": 38.0314, "maxy": 67.7348}, "results": [{"name": "Cuba", "relevancy": "relevant", "types": ["place-country"], "position": {"x": -80.832886, "y": 15.521132748163712}, "houseNumber": ""}], "related_queries": ["Patria o Muerte", "Cuba Libre"]})");
+  lines.append("\n");
+
+  lines.append(
+      R"({"query": "riga", "locale": "en", "position": {"x": 37.65376, "y": 98.51110651930014}, "viewport": {"minx": 37.5064, "miny": 67.0476, "maxx": 37.7799, "maxy": 67.304}, "results": [{"name": "R\u012bga", "relevancy": "vital", "types": ["place-city-capital-2"], "position": {"x": 24.105186, "y": 107.7819569220319}, "houseNumber": ""}, {"name": "R\u012bga", "relevancy": "vital", "types": ["place-city-capital-2"], "position": {"x": 24.105186, "y": 107.7819569220319}, "houseNumber": ""}]})");
+  lines.append("\n");
+
+  lines.append(
+      R"({"query": "tula", "locale": "en", "viewport": {"minx": 37.5064, "miny": 67.0476, "maxx": 37.7799, "maxy": 67.304}})");
+  lines.append("\n");
+
+  std::vector<Sample> samples;
+  TEST(Sample::DeserializeFromJSONLines(lines, samples), ());
+
+  std::vector<Sample> expected = {m_cuba, m_riga, m_tula};
+
+  std::sort(samples.begin(), samples.end());
+  std::sort(expected.begin(), expected.end());
+
+  TEST_EQUAL(samples, expected, ());
+}
+
+UNIT_CLASS_TEST(SampleTest, SerDes)
+{
+  std::vector<Sample> expected = {m_cuba, m_riga, m_tula};
+
+  std::string lines;
+  Sample::SerializeToJSONLines(expected, lines);
+
+  std::vector<Sample> actual;
+  TEST(Sample::DeserializeFromJSONLines(lines, actual), ());
+
+  std::sort(expected.begin(), expected.end());
+  std::sort(actual.begin(), actual.end());
+  TEST_EQUAL(expected, actual, ());
+}
+}  // namespace sample_test

libs/search/search_quality/search_quality_tool/CMakeLists.txt

@@ -0,0 +1,11 @@
+project(search_quality_tool)
+
+set(SRC search_quality_tool.cpp)
+
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  search_tests_support
+  search_quality
+  gflags::gflags
+)

libs/search/search_quality/search_quality_tool/queries.txt

@@ -0,0 +1,41 @@
+wifi
+hotel
+Москва
+London
+кафе москва
+улица ленина
+улица Ленина дом 1
+ленина 1
+лесная 12
+ул. лесная д. 13
+улица Лесная д14
+лесная 15/1
+Ленинградский проспект 39
+Ленинградский проспект 39с79
+Ленинградский проспект 39-79
+Ленинградский проспект 39 стр. 79
+Senieji Trakai
+Trakų str
+IKEA Москва
+Красноармейская улица
+улица набережная
+квартира 44
+Tehama street
+tehama 4th street
+Улица 8 Марта
+4-я улица 8 Марта
+орехово-зуево лапина улица 78
+поварово i
+поварово 1
+чехова улица завод
+чехов улица лермонтова
+чехов улица чехова
+депутатский
+6 route des jeunes Genève
+москва 4 2 останкинская
+москва 2 4 останкинская
+улица Константина Симонова
+фрезерная 1, 2/1, стр. 1
+фрезерная 1, д. 2/1, стр. 1
+фрезерная 1, д. 2/1, стр. 10
+хиславичи улица толстого 19

libs/search/search_quality/search_quality_tool/search_quality_tool.cpp

@@ -0,0 +1,377 @@
+#include "search/search_quality/helpers.hpp"
+
+#include "search/search_tests_support/test_search_engine.hpp"
+#include "search/search_tests_support/test_search_request.hpp"
+
+#include "search/ranking_info.hpp"
+#include "search/result.hpp"
+#include "search/search_params.hpp"
+
+#include "indexer/classificator_loader.hpp"
+#include "indexer/data_source.hpp"
+#include "indexer/mwm_set.hpp"
+
+#include "platform/platform_tests_support/helpers.hpp"
+
+#include "platform/country_file.hpp"
+#include "platform/platform.hpp"
+
+#include "geometry/mercator.hpp"
+#include "geometry/point2d.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+#include "base/stl_helpers.hpp"
+#include "base/string_utils.hpp"
+#include "base/timer.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <limits>
+#include <map>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <gflags/gflags.h>
+
+using namespace search::search_quality;
+using namespace search::tests_support;
+using namespace search;
+using namespace std::chrono;
+using namespace std;
+
+DEFINE_string(data_path, "", "Path to data directory (resources dir)");
+DEFINE_string(locale, "en", "Locale of all the search queries");
+DEFINE_int32(num_threads, 1, "Number of search engine threads");
+DEFINE_string(mwm_list_path, "", "Path to a file containing the names of available mwms, one per line");
+DEFINE_string(mwm_path, "", "Path to mwm files (writable dir)");
+DEFINE_string(queries_path, "", "Path to the file with queries");
+DEFINE_int32(top, 1, "Number of top results to show for every query");
+DEFINE_string(viewport, "", "Viewport to use when searching (default, moscow, london, zurich)");
+DEFINE_string(check_completeness, "", "Path to the file with completeness data");
+DEFINE_string(ranking_csv_file, "", "File ranking info will be exported to");
+
+string const kDefaultQueriesPathSuffix = "/../search/search_quality/search_quality_tool/queries.txt";
+string const kEmptyResult = "<empty>";
+
+// Unlike strings::Tokenize, this function allows for empty tokens.
+void Split(string const & s, char delim, vector<string> & parts)
+{
+  istringstream iss(s);
+  string part;
+  while (getline(iss, part, delim))
+    parts.push_back(part);
+}
+
+struct CompletenessQuery
+{
+  DECLARE_EXCEPTION(MalformedQueryException, RootException);
+
+  explicit CompletenessQuery(string && s)
+  {
+    s.append(" ");
+
+    vector<string> parts;
+    Split(s, ';', parts);
+    if (parts.size() != 7)
+      MYTHROW(MalformedQueryException, ("Can't split", s, ", found", parts.size(), "part(s):", parts));
+
+    auto const idx = parts[0].find(':');
+    if (idx == string::npos)
+      MYTHROW(MalformedQueryException, ("Could not find \':\':", s));
+
+    string mwmName = parts[0].substr(0, idx);
+    string const kMwmSuffix = ".mwm";
+    if (!mwmName.ends_with(kMwmSuffix))
+      MYTHROW(MalformedQueryException, ("Bad mwm name:", s));
+
+    string const featureIdStr = parts[0].substr(idx + 1);
+    uint64_t featureId;
+    if (!strings::to_uint64(featureIdStr, featureId))
+      MYTHROW(MalformedQueryException, ("Bad feature id:", s));
+
+    string const type = parts[1];
+    double lon, lat;
+    if (!strings::to_double(parts[2].c_str(), lon) || !strings::to_double(parts[3].c_str(), lat))
+      MYTHROW(MalformedQueryException, ("Bad lon-lat:", s));
+
+    string const city = parts[4];
+    string const street = parts[5];
+    string const house = parts[6];
+
+    mwmName = mwmName.substr(0, mwmName.size() - kMwmSuffix.size());
+    string country = mwmName;
+    replace(country.begin(), country.end(), '_', ' ');
+
+    m_query = country + " " + city + " " + street + " " + house + " ";
+    m_mwmName = mwmName;
+    m_featureId = static_cast<uint32_t>(featureId);
+    m_lat = lat;
+    m_lon = lon;
+  }
+
+  string m_query;
+  unique_ptr<TestSearchRequest> m_request;
+  string m_mwmName;
+  uint32_t m_featureId = 0;
+  double m_lat = 0;
+  double m_lon = 0;
+};
+
+string MakePrefixFree(string const & query)
+{
+  return query + " ";
+}
+
+// If n == 1, prints the query and the top result separated by a tab.
+// Otherwise, prints the query on a separate line
+// and then prints n top results on n lines starting with tabs.
+void PrintTopResults(string const & query, vector<Result> const & results, size_t n, double elapsedSeconds)
+{
+  cout << query;
+  char timeBuf[100];
+  snprintf(timeBuf, sizeof(timeBuf), "\t[%.3fs]", elapsedSeconds);
+  if (n > 1)
+    cout << timeBuf;
+  for (size_t i = 0; i < n; ++i)
+  {
+    if (n > 1)
+      cout << endl;
+    cout << "\t";
+    if (i < results.size())
+      // todo(@m) Print more information: coordinates, viewport, etc.
+      cout << results[i].GetString();
+    else
+      cout << kEmptyResult;
+  }
+  if (n == 1)
+    cout << timeBuf;
+  cout << endl;
+}
+
+void CalcStatistics(vector<double> const & a, double & avg, double & maximum, double & var, double & stdDev)
+{
+  avg = 0;
+  maximum = 0;
+  var = 0;
+  stdDev = 0;
+
+  for (auto const x : a)
+  {
+    avg += static_cast<double>(x);
+    maximum = max(maximum, static_cast<double>(x));
+  }
+
+  double n = static_cast<double>(a.size());
+  if (a.size() > 0)
+    avg /= n;
+
+  for (double const x : a)
+    var += math::Pow2(x - avg);
+  if (a.size() > 1)
+    var /= n - 1;
+  stdDev = sqrt(var);
+}
+
+// Returns the position of the result that is expected to be found by geocoder completeness
+// tests in the |result| vector or -1 if it does not occur there.
+int FindResult(DataSource & dataSource, string const & mwmName, uint32_t const featureId, double const lat,
+               double const lon, vector<Result> const & results)
+{
+  CHECK_LESS_OR_EQUAL(results.size(), numeric_limits<int>::max(), ());
+  auto const mwmId = dataSource.GetMwmIdByCountryFile(platform::CountryFile(mwmName));
+  FeatureID const expectedFeatureId(mwmId, featureId);
+  for (size_t i = 0; i < results.size(); ++i)
+  {
+    auto const & r = results[i];
+    if (r.GetFeatureID() == expectedFeatureId)
+      return static_cast<int>(i);
+  }
+
+  // Another attempt. If the queries are stale, feature id is useless.
+  // However, some information may be recovered from (lat, lon).
+  double const kEps = 1e-2;
+  for (size_t i = 0; i < results.size(); ++i)
+  {
+    auto const & r = results[i];
+    if (r.HasPoint() && AlmostEqualAbs(r.GetFeatureCenter(), mercator::FromLatLon(lat, lon), kEps))
+    {
+      double const dist = mercator::DistanceOnEarth(r.GetFeatureCenter(), mercator::FromLatLon(lat, lon));
+      LOG(LDEBUG, ("dist =", dist));
+      return static_cast<int>(i);
+    }
+  }
+  return -1;
+}
+
+// Reads queries in the format
+//   CountryName.mwm:featureId;type;lon;lat;city;street;<housenumber or housename>
+// from |path|, executes them against the |engine| with viewport set to |viewport|
+// and reports the number of queries whose expected result is among the returned results.
+// Exact feature id is expected, but a close enough (lat, lon) is good too.
+void CheckCompleteness(string const & path, DataSource & dataSource, TestSearchEngine & engine,
+                       m2::RectD const & viewport, string const & locale)
+{
+  base::ScopedLogAbortLevelChanger const logAbortLevel(LCRITICAL);
+
+  ifstream stream(path.c_str());
+  CHECK(stream.is_open(), ("Can't open", path));
+
+  base::Timer timer;
+
+  uint32_t totalQueries = 0;
+  uint32_t malformedQueries = 0;
+  uint32_t expectedResultsFound = 0;
+  uint32_t expectedResultsTop1 = 0;
+
+  // todo(@m) Process the queries on the fly and do not keep them.
+  vector<CompletenessQuery> queries;
+
+  string line;
+  while (getline(stream, line))
+  {
+    ++totalQueries;
+    try
+    {
+      CompletenessQuery q(std::move(line));
+      q.m_request = make_unique<TestSearchRequest>(engine, q.m_query, locale, Mode::Everywhere, viewport);
+      queries.push_back(std::move(q));
+    }
+    catch (CompletenessQuery::MalformedQueryException & e)
+    {
+      LOG(LERROR, (e.what()));
+      ++malformedQueries;
+    }
+  }
+
+  for (auto & q : queries)
+  {
+    q.m_request->Run();
+
+    LOG(LDEBUG, (q.m_query, q.m_request->Results()));
+    int pos = FindResult(dataSource, q.m_mwmName, q.m_featureId, q.m_lat, q.m_lon, q.m_request->Results());
+    if (pos >= 0)
+      ++expectedResultsFound;
+    if (pos == 0)
+      ++expectedResultsTop1;
+  }
+
+  double const expectedResultsFoundPercentage =
+      totalQueries == 0 ? 0 : 100.0 * static_cast<double>(expectedResultsFound) / static_cast<double>(totalQueries);
+  double const expectedResultsTop1Percentage =
+      totalQueries == 0 ? 0 : 100.0 * static_cast<double>(expectedResultsTop1) / static_cast<double>(totalQueries);
+
+  cout << "Time spent on checking completeness: " << timer.ElapsedSeconds() << "s." << endl;
+  cout << "Total queries: " << totalQueries << endl;
+  cout << "Malformed queries: " << malformedQueries << endl;
+  cout << "Expected results found: " << expectedResultsFound << " (" << expectedResultsFoundPercentage << "%)." << endl;
+  cout << "Expected results found in the top1 slot: " << expectedResultsTop1 << " (" << expectedResultsTop1Percentage
+       << "%)." << endl;
+}
+
+void RunRequests(TestSearchEngine & engine, m2::RectD const & viewport, string queriesPath, string const & locale,
+                 string const & rankingCSVFile, size_t top)
+{
+  vector<string> queries;
+  {
+    if (queriesPath.empty())
+      queriesPath = base::JoinPath(GetPlatform().WritableDir(), kDefaultQueriesPathSuffix);
+    ReadStringsFromFile(queriesPath, queries);
+  }
+
+  vector<unique_ptr<TestSearchRequest>> requests;
+  for (size_t i = 0; i < queries.size(); ++i)
+  {
+    // todo(@m) Add a bool flag to search with prefixes?
+    requests.emplace_back(
+        make_unique<TestSearchRequest>(engine, MakePrefixFree(queries[i]), locale, Mode::Everywhere, viewport));
+  }
+
+  ofstream csv;
+  bool dumpCSV = false;
+  if (!rankingCSVFile.empty())
+  {
+    csv.open(rankingCSVFile);
+    if (!csv.is_open())
+      LOG(LERROR, ("Can't open file for CSV dump:", rankingCSVFile));
+    else
+      dumpCSV = true;
+  }
+
+  if (dumpCSV)
+  {
+    RankingInfo::PrintCSVHeader(csv);
+    csv << endl;
+  }
+
+  vector<double> responseTimes(queries.size());
+  for (size_t i = 0; i < queries.size(); ++i)
+  {
+    requests[i]->Run();
+    auto rt = duration_cast<milliseconds>(requests[i]->ResponseTime()).count();
+    responseTimes[i] = static_cast<double>(rt) / 1000;
+    PrintTopResults(MakePrefixFree(queries[i]), requests[i]->Results(), top, responseTimes[i]);
+
+    if (dumpCSV)
+    {
+      for (auto const & result : requests[i]->Results())
+      {
+        result.GetRankingInfo().ToCSV(csv);
+        csv << endl;
+      }
+    }
+  }
+
+  double averageTime;
+  double maxTime;
+  double varianceTime;
+  double stdDevTime;
+  CalcStatistics(responseTimes, averageTime, maxTime, varianceTime, stdDevTime);
+
+  cout << fixed << setprecision(3);
+  cout << endl;
+  cout << "Maximum response time: " << maxTime << "s" << endl;
+  cout << "Average response time: " << averageTime << "s"
+       << " (std. dev. " << stdDevTime << "s)" << endl;
+}
+
+int main(int argc, char * argv[])
+{
+  platform::tests_support::ChangeMaxNumberOfOpenFiles(kMaxOpenFiles);
+  CheckLocale();
+
+  gflags::SetUsageMessage("Search quality tests.");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  SetPlatformDirs(FLAGS_data_path, FLAGS_mwm_path);
+
+  classificator::Load();
+
+  FrozenDataSource dataSource;
+  InitDataSource(dataSource, FLAGS_mwm_list_path);
+
+  auto engine = InitSearchEngine(dataSource, FLAGS_locale, FLAGS_num_threads);
+  engine->InitAffiliations();
+
+  m2::RectD viewport;
+  InitViewport(FLAGS_viewport, viewport);
+
+  ios_base::sync_with_stdio(false);
+
+  if (!FLAGS_check_completeness.empty())
+  {
+    CheckCompleteness(FLAGS_check_completeness, dataSource, *engine, viewport, FLAGS_locale);
+    return 0;
+  }
+
+  RunRequests(*engine, viewport, FLAGS_queries_path, FLAGS_locale, FLAGS_ranking_csv_file,
+              static_cast<size_t>(FLAGS_top));
+  return 0;
+}