Repo created

libs/search/search_quality/assessment_tool/CMakeLists.txt

@@ -0,0 +1,49 @@
+project(assessment_tool)
+
+set(SRC
+  assessment_tool.cpp
+  context.cpp
+  context.hpp
+  edits.cpp
+  edits.hpp
+  feature_info_dialog.cpp
+  feature_info_dialog.hpp
+  helpers.cpp
+  helpers.hpp
+  main_model.cpp
+  main_model.hpp
+  main_view.cpp
+  main_view.hpp
+  model.hpp
+  result_view.cpp
+  result_view.hpp
+  results_view.cpp
+  results_view.hpp
+  sample_view.cpp
+  sample_view.hpp
+  samples_view.cpp
+  samples_view.hpp
+  search_request_runner.cpp
+  search_request_runner.hpp
+  view.hpp
+)
+
+#omim_add_executable(${PROJECT_NAME} MACOSX_BUNDLE ${SRC})
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+set_target_properties(${PROJECT_NAME} PROPERTIES AUTOMOC ON)
+
+target_link_libraries(${PROJECT_NAME}
+  search_quality
+  qt_common
+  map
+  gflags::gflags
+)
+
+# if (PLATFORM_MAC)
+#   set_target_properties(
+#     ${PROJECT_NAME}
+#     PROPERTIES
+#     MACOSX_BUNDLE_INFO_PLIST ${PROJECT_SOURCE_DIR}/Info.plist
+#   )
+# endif()

libs/search/search_quality/assessment_tool/Info.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<plist>
+  <dict>
+    <key>NSPrincipalClass</key>
+    <string>NSApplication</string>
+
+    <key>NSHighResolutionCapable</key>
+    <string>True</string>
+  </dict>
+</plist>

libs/search/search_quality/assessment_tool/assessment_tool.cpp

@@ -0,0 +1,58 @@
+#include "main_model.hpp"
+#include "main_view.hpp"
+
+#include "qt/qt_common/helpers.hpp"
+
+#include "map/framework.hpp"
+
+#include "search/search_quality/helpers.hpp"
+
+#include "platform/platform_tests_support/helpers.hpp"
+
+#include "platform/platform.hpp"
+
+#include <QtWidgets/QApplication>
+
+#include <gflags/gflags.h>
+
+DEFINE_string(resources_path, "", "Path to resources directory");
+DEFINE_string(data_path, "", "Path to data directory");
+DEFINE_string(samples_path, "", "Path to the file with samples to open on startup");
+DEFINE_uint64(num_threads, 4, "Number of search engine threads");
+
+int main(int argc, char ** argv)
+{
+  platform::tests_support::ChangeMaxNumberOfOpenFiles(search::search_quality::kMaxOpenFiles);
+
+  gflags::SetUsageMessage("Assessment tool.");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  Platform & platform = GetPlatform();
+  if (!FLAGS_resources_path.empty())
+    platform.SetResourceDir(FLAGS_resources_path);
+  if (!FLAGS_data_path.empty())
+    platform.SetWritableDirForTests(FLAGS_data_path);
+
+  Q_INIT_RESOURCE(resources_common);
+  QApplication app(argc, argv);
+
+  qt::common::SetDefaultSurfaceFormat(app.platformName());
+
+  FrameworkParams params;
+  CHECK_GREATER(FLAGS_num_threads, 0, ());
+  params.m_numSearchAPIThreads = FLAGS_num_threads;
+
+  Framework framework(params);
+  MainView view(framework, app.primaryScreen()->geometry());
+  MainModel model(framework);
+
+  model.SetView(view);
+  view.SetModel(model);
+
+  view.showMaximized();
+
+  if (!FLAGS_samples_path.empty())
+    model.Open(FLAGS_samples_path);
+
+  return app.exec();
+}

libs/search/search_quality/assessment_tool/context.cpp

@@ -0,0 +1,181 @@
+#include "search/search_quality/assessment_tool/context.hpp"
+
+#include "search/feature_loader.hpp"
+#include "search/search_quality/matcher.hpp"
+
+#include "base/assert.hpp"
+
+#include <utility>
+
+using namespace std;
+
+// Context -----------------------------------------------------------------------------------------
+void Context::Clear()
+{
+  m_goldenMatching.clear();
+  m_actualMatching.clear();
+
+  m_foundResults.Clear();
+  m_foundResultsEdits.Clear();
+
+  m_nonFoundResults.clear();
+  m_nonFoundResultsEdits.Clear();
+
+  m_sampleEdits.Clear();
+
+  m_initialized = false;
+}
+
+void Context::LoadFromSample(search::Sample const & sample)
+{
+  Clear();
+  m_sample = sample;
+  m_sampleEdits.Reset(sample.m_useless);
+}
+
+search::Sample Context::MakeSample(search::FeatureLoader & loader) const
+{
+  search::Sample outSample = m_sample;
+
+  if (!m_initialized)
+    return outSample;
+
+  auto const & foundEntries = m_foundResultsEdits.GetEntries();
+  auto const & nonFoundEntries = m_nonFoundResultsEdits.GetEntries();
+
+  auto & outResults = outSample.m_results;
+  outResults.clear();
+
+  CHECK_EQUAL(m_goldenMatching.size(), m_sample.m_results.size(), ());
+  CHECK_EQUAL(m_actualMatching.size(), foundEntries.size(), ());
+  CHECK_EQUAL(m_actualMatching.size(), m_foundResults.GetCount(), ());
+
+  // Iterates over original (loaded from the file with search samples)
+  // results first.
+  size_t k = 0;
+  for (size_t i = 0; i < m_sample.m_results.size(); ++i)
+  {
+    auto const j = m_goldenMatching[i];
+
+    if (j == search::Matcher::kInvalidId)
+    {
+      auto const & entry = nonFoundEntries[k++];
+      auto const deleted = entry.m_deleted;
+      auto const & curr = entry.m_currRelevance;
+      if (!deleted && curr)
+      {
+        auto result = m_sample.m_results[i];
+        result.m_relevance = *curr;
+        outResults.push_back(result);
+      }
+      continue;
+    }
+
+    if (!foundEntries[j].m_currRelevance)
+      continue;
+
+    auto result = m_sample.m_results[i];
+    result.m_relevance = *foundEntries[j].m_currRelevance;
+    outResults.push_back(std::move(result));
+  }
+
+  // Iterates over results retrieved during assessment.
+  for (size_t i = 0; i < m_foundResults.GetCount(); ++i)
+  {
+    auto const j = m_actualMatching[i];
+    if (j != search::Matcher::kInvalidId)
+    {
+      // This result was processed by the loop above.
+      continue;
+    }
+
+    if (!foundEntries[i].m_currRelevance)
+      continue;
+
+    auto const & result = m_foundResults[i];
+    // No need in non-feature results.
+    if (result.GetResultType() != search::Result::Type::Feature)
+      continue;
+
+    auto ft = loader.Load(result.GetFeatureID());
+    CHECK(ft, ());
+    outResults.push_back(search::Sample::Result::Build(*ft, *foundEntries[i].m_currRelevance));
+  }
+
+  outSample.m_useless = m_sampleEdits.m_currUseless;
+
+  return outSample;
+}
+
+void Context::ApplyEdits()
+{
+  if (!m_initialized)
+    return;
+  m_foundResultsEdits.Apply();
+  m_nonFoundResultsEdits.Apply();
+  m_sampleEdits.Apply();
+}
+
+// ContextList -------------------------------------------------------------------------------------
+ContextList::ContextList(OnResultsUpdate onResultsUpdate, OnResultsUpdate onNonFoundResultsUpdate,
+                         OnSampleUpdate onSampleUpdate)
+  : m_onResultsUpdate(onResultsUpdate)
+  , m_onNonFoundResultsUpdate(onNonFoundResultsUpdate)
+  , m_onSampleUpdate(onSampleUpdate)
+{}
+
+void ContextList::Resize(size_t size)
+{
+  size_t const oldSize = m_contexts.size();
+
+  for (size_t i = size; i < oldSize; ++i)
+    m_contexts[i].Clear();
+  if (size < m_contexts.size())
+    m_contexts.erase(m_contexts.begin() + size, m_contexts.end());
+
+  m_hasChanges.resize(size);
+  for (size_t i = oldSize; i < size; ++i)
+  {
+    m_contexts.emplace_back(
+        [this, i](ResultsEdits::Update const & update)
+    {
+      OnContextUpdated(i);
+      if (m_onResultsUpdate)
+        m_onResultsUpdate(i, update);
+    },
+        [this, i](ResultsEdits::Update const & update)
+    {
+      OnContextUpdated(i);
+      if (m_onNonFoundResultsUpdate)
+        m_onNonFoundResultsUpdate(i, update);
+    }, [this, i]()
+    {
+      OnContextUpdated(i);
+      if (m_onSampleUpdate)
+        m_onSampleUpdate(i);
+    });
+  }
+}
+
+vector<search::Sample> ContextList::MakeSamples(search::FeatureLoader & loader) const
+{
+  vector<search::Sample> samples;
+  for (auto const & context : m_contexts)
+    samples.push_back(context.MakeSample(loader));
+  return samples;
+}
+
+void ContextList::ApplyEdits()
+{
+  for (auto & context : m_contexts)
+    context.ApplyEdits();
+}
+
+void ContextList::OnContextUpdated(size_t index)
+{
+  if (!m_hasChanges[index] && m_contexts[index].HasChanges())
+    ++m_numChanges;
+  if (m_hasChanges[index] && !m_contexts[index].HasChanges())
+    --m_numChanges;
+  m_hasChanges[index] = m_contexts[index].HasChanges();
+}

libs/search/search_quality/assessment_tool/context.hpp

@@ -0,0 +1,139 @@
+#pragma once
+
+#include "search/result.hpp"
+#include "search/search_quality/assessment_tool/edits.hpp"
+#include "search/search_quality/matcher.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "base/string_utils.hpp"
+
+#include <cstddef>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace search
+{
+class FeatureLoader;
+}
+
+struct Context
+{
+  enum class SearchState
+  {
+    Untouched,
+    InQueue,
+    Completed
+  };
+
+  Context(ResultsEdits::OnUpdate onFoundResultsUpdate, ResultsEdits::OnUpdate onNonFoundResultsUpdate,
+          SampleEdits::OnUpdate onSampleUpdate)
+    : m_foundResultsEdits(onFoundResultsUpdate)
+    , m_nonFoundResultsEdits(onNonFoundResultsUpdate)
+    , m_sampleEdits(onSampleUpdate)
+  {}
+
+  void AddNonFoundResult(search::Sample::Result const & result)
+  {
+    CHECK_EQUAL(m_goldenMatching.size(), m_sample.m_results.size(), ());
+
+    m_sample.m_results.push_back(result);
+    m_goldenMatching.push_back(search::Matcher::kInvalidId);
+
+    m_nonFoundResults.push_back(result);
+    m_nonFoundResultsEdits.Add(result.m_relevance);
+  }
+
+  bool IsUseless() const { return m_sampleEdits.m_currUseless; }
+
+  bool HasChanges() const
+  {
+    if (m_sampleEdits.HasChanges())
+      return true;
+    if (!m_initialized)
+      return false;
+    return m_foundResultsEdits.HasChanges() || m_nonFoundResultsEdits.HasChanges();
+  }
+
+  void Clear();
+
+  void LoadFromSample(search::Sample const & sample);
+
+  // Makes sample in accordance with uncommited edits.
+  search::Sample MakeSample(search::FeatureLoader & loader) const;
+
+  // Commits all edits.
+  void ApplyEdits();
+
+  search::Sample m_sample;
+  search::Results m_foundResults;
+  ResultsEdits m_foundResultsEdits;
+
+  std::vector<size_t> m_goldenMatching;
+  std::vector<size_t> m_actualMatching;
+
+  std::vector<search::Sample::Result> m_nonFoundResults;
+  ResultsEdits m_nonFoundResultsEdits;
+
+  SampleEdits m_sampleEdits;
+
+  SearchState m_searchState = SearchState::Untouched;
+
+  bool m_initialized = false;
+};
+
+class ContextList
+{
+public:
+  class SamplesSlice
+  {
+  public:
+    SamplesSlice() = default;
+    explicit SamplesSlice(ContextList const & contexts) : m_contexts(&contexts) {}
+
+    bool IsValid() const { return m_contexts != nullptr; }
+
+    std::string GetLabel(size_t index) const { return strings::ToUtf8((*m_contexts)[index].m_sample.m_query); }
+
+    bool IsChanged(size_t index) const { return (*m_contexts)[index].HasChanges(); }
+
+    Context::SearchState GetSearchState(size_t index) const { return (*m_contexts)[index].m_searchState; }
+
+    bool IsUseless(size_t index) const { return (*m_contexts)[index].m_sampleEdits.m_currUseless; }
+
+    size_t Size() const { return m_contexts->Size(); }
+
+  private:
+    ContextList const * m_contexts = nullptr;
+  };
+
+  using OnResultsUpdate = std::function<void(size_t index, ResultsEdits::Update const & update)>;
+  using OnSampleUpdate = std::function<void(size_t index)>;
+
+  ContextList(OnResultsUpdate onResultsUpdate, OnResultsUpdate onNonFoundResultsUpdate, OnSampleUpdate onSampleUpdate);
+
+  void Resize(size_t size);
+  size_t Size() const { return m_contexts.size(); }
+
+  Context & operator[](size_t i) { return m_contexts[i]; }
+  Context const & operator[](size_t i) const { return m_contexts[i]; }
+
+  bool HasChanges() const { return m_numChanges != 0; }
+
+  // Generates search samples in accordance with uncommited edits.
+  std::vector<search::Sample> MakeSamples(search::FeatureLoader & loader) const;
+
+  // Commits all edits.
+  void ApplyEdits();
+
+private:
+  void OnContextUpdated(size_t index);
+
+  std::vector<Context> m_contexts;
+  std::vector<bool> m_hasChanges;
+  size_t m_numChanges = 0;
+
+  OnResultsUpdate m_onResultsUpdate;
+  OnResultsUpdate m_onNonFoundResultsUpdate;
+  OnSampleUpdate m_onSampleUpdate;
+};

libs/search/search_quality/assessment_tool/edits.cpp

@@ -0,0 +1,213 @@
+#include "search/search_quality/assessment_tool/edits.hpp"
+
+#include "base/assert.hpp"
+
+using namespace std;
+
+namespace
+{
+void UpdateNumEdits(ResultsEdits::Entry const & entry, ResultsEdits::Relevance const & r, size_t & numEdits)
+{
+  if (entry.m_currRelevance != entry.m_origRelevance && r == entry.m_origRelevance)
+  {
+    CHECK_GREATER(numEdits, 0, ());
+    --numEdits;
+  }
+  if (entry.m_currRelevance == entry.m_origRelevance && r != entry.m_origRelevance)
+    ++numEdits;
+}
+}  // namespace
+
+// SampleEdits -------------------------------------------------------------------------------------
+void SampleEdits::Reset(bool origUseless)
+{
+  m_origUseless = origUseless;
+  m_currUseless = origUseless;
+}
+
+void SampleEdits::FlipUsefulness()
+{
+  m_currUseless ^= true;
+  if (m_onUpdate)
+    m_onUpdate();
+}
+
+void SampleEdits::Apply()
+{
+  m_origUseless = m_currUseless;
+  if (m_onUpdate)
+    m_onUpdate();
+}
+
+// ResultsEdits::Editor ----------------------------------------------------------------------------
+ResultsEdits::Editor::Editor(ResultsEdits & parent, size_t index) : m_parent(parent), m_index(index) {}
+
+bool ResultsEdits::Editor::Set(Relevance relevance)
+{
+  return m_parent.SetRelevance(m_index, relevance);
+}
+
+optional<ResultsEdits::Relevance> const & ResultsEdits::Editor::Get() const
+{
+  return m_parent.Get(m_index).m_currRelevance;
+}
+
+bool ResultsEdits::Editor::HasChanges() const
+{
+  return m_parent.HasChanges(m_index);
+}
+
+ResultsEdits::Entry::Type ResultsEdits::Editor::GetType() const
+{
+  return m_parent.Get(m_index).m_type;
+}
+
+// ResultsEdits ------------------------------------------------------------------------------------
+void ResultsEdits::Apply()
+{
+  WithObserver(Update::MakeAll(), [this]()
+  {
+    for (auto & entry : m_entries)
+    {
+      entry.m_origRelevance = entry.m_currRelevance;
+      entry.m_type = Entry::Type::Loaded;
+    }
+    m_numEdits = 0;
+  });
+}
+
+void ResultsEdits::Reset(vector<optional<ResultsEdits::Relevance>> const & relevances)
+{
+  WithObserver(Update::MakeAll(), [this, &relevances]()
+  {
+    m_entries.resize(relevances.size());
+    for (size_t i = 0; i < m_entries.size(); ++i)
+    {
+      auto & entry = m_entries[i];
+      entry.m_origRelevance = relevances[i];
+      entry.m_currRelevance = relevances[i];
+      entry.m_deleted = false;
+      entry.m_type = Entry::Type::Loaded;
+    }
+    m_numEdits = 0;
+  });
+}
+
+bool ResultsEdits::SetRelevance(size_t index, Relevance relevance)
+{
+  return WithObserver(Update::MakeSingle(index), [this, index, relevance]()
+  {
+    CHECK_LESS(index, m_entries.size(), ());
+
+    auto & entry = m_entries[index];
+
+    UpdateNumEdits(entry, relevance, m_numEdits);
+
+    entry.m_currRelevance = relevance;
+    return entry.m_currRelevance != entry.m_origRelevance;
+  });
+}
+
+void ResultsEdits::SetAllRelevances(Relevance relevance)
+{
+  WithObserver(Update::MakeAll(), [this, relevance]()
+  {
+    for (auto & entry : m_entries)
+    {
+      UpdateNumEdits(entry, relevance, m_numEdits);
+      entry.m_currRelevance = relevance;
+    }
+  });
+}
+
+void ResultsEdits::Add(Relevance relevance)
+{
+  auto const index = m_entries.size();
+  WithObserver(Update::MakeAdd(index), [&]()
+  {
+    m_entries.emplace_back(relevance, Entry::Type::Created);
+    ++m_numEdits;
+  });
+}
+
+void ResultsEdits::Delete(size_t index)
+{
+  return WithObserver(Update::MakeDelete(index), [this, index]()
+  {
+    CHECK_LESS(index, m_entries.size(), ());
+
+    auto & entry = m_entries[index];
+    CHECK(!entry.m_deleted, ());
+    entry.m_deleted = true;
+    switch (entry.m_type)
+    {
+    case Entry::Type::Loaded: ++m_numEdits; break;
+    case Entry::Type::Created: --m_numEdits; break;
+    }
+  });
+}
+
+void ResultsEdits::Resurrect(size_t index)
+{
+  return WithObserver(Update::MakeResurrect(index), [this, index]()
+  {
+    CHECK_LESS(index, m_entries.size(), ());
+
+    auto & entry = m_entries[index];
+    CHECK(entry.m_deleted, ());
+    CHECK_GREATER(m_numEdits, 0, ());
+    entry.m_deleted = false;
+    switch (entry.m_type)
+    {
+    case Entry::Type::Loaded: --m_numEdits; break;
+    case Entry::Type::Created: ++m_numEdits; break;
+    }
+  });
+}
+
+ResultsEdits::Entry & ResultsEdits::GetEntry(size_t index)
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  return m_entries[index];
+}
+
+ResultsEdits::Entry const & ResultsEdits::GetEntry(size_t index) const
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  return m_entries[index];
+}
+
+vector<optional<ResultsEdits::Relevance>> ResultsEdits::GetRelevances() const
+{
+  vector<optional<ResultsEdits::Relevance>> relevances(m_entries.size());
+  for (size_t i = 0; i < m_entries.size(); ++i)
+    relevances[i] = m_entries[i].m_currRelevance;
+  return relevances;
+}
+
+ResultsEdits::Entry const & ResultsEdits::Get(size_t index) const
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  return m_entries[index];
+}
+
+void ResultsEdits::Clear()
+{
+  WithObserver(Update::MakeAll(), [this]()
+  {
+    m_entries.clear();
+    m_numEdits = 0;
+  });
+}
+
+bool ResultsEdits::HasChanges() const
+{
+  return m_numEdits != 0;
+}
+
+bool ResultsEdits::HasChanges(size_t index) const
+{
+  CHECK_LESS(index, m_entries.size(), ());
+  auto const & entry = m_entries[index];
+  return entry.m_currRelevance != entry.m_origRelevance;
+}

libs/search/search_quality/assessment_tool/edits.hpp

@@ -0,0 +1,153 @@
+#pragma once
+
+#include "search/search_quality/sample.hpp"
+
+#include "base/scope_guard.hpp"
+
+#include <cstddef>
+#include <functional>
+#include <limits>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+struct SampleEdits
+{
+  using OnUpdate = std::function<void()>;
+
+  explicit SampleEdits(OnUpdate onUpdate) : m_onUpdate(onUpdate) {}
+
+  void Reset(bool origUseless);
+  void FlipUsefulness();
+  void Apply();
+  bool HasChanges() const { return m_origUseless != m_currUseless; }
+  void Clear() {}
+
+  bool m_origUseless = false;
+  bool m_currUseless = false;
+
+  OnUpdate m_onUpdate;
+};
+
+class ResultsEdits
+{
+public:
+  using Relevance = search::Sample::Result::Relevance;
+
+  struct Entry
+  {
+    enum class Type
+    {
+      Loaded,
+      Created
+    };
+
+    Entry() = default;
+    Entry(std::optional<Relevance> relevance, Type type)
+      : m_currRelevance(relevance)
+      , m_origRelevance(relevance)
+      , m_type(type)
+    {}
+
+    std::optional<Relevance> m_currRelevance = {};
+    std::optional<Relevance> m_origRelevance = {};
+    bool m_deleted = false;
+    Type m_type = Type::Loaded;
+  };
+
+  struct Update
+  {
+    static auto constexpr kInvalidIndex = std::numeric_limits<size_t>::max();
+
+    enum class Type
+    {
+      Single,
+      All,
+      Add,
+      Delete,
+      Resurrect
+    };
+
+    Update() = default;
+    Update(Type type, size_t index) : m_type(type), m_index(index) {}
+
+    static Update MakeAll() { return {}; }
+    static Update MakeSingle(size_t index) { return {Type::Single, index}; }
+    static Update MakeAdd(size_t index) { return {Type::Add, index}; }
+    static Update MakeDelete(size_t index) { return {Type::Delete, index}; }
+    static Update MakeResurrect(size_t index) { return {Type::Resurrect, index}; }
+
+    Type m_type = Type::All;
+    size_t m_index = kInvalidIndex;
+  };
+
+  using OnUpdate = std::function<void(Update const & update)>;
+
+  class Editor
+  {
+  public:
+    Editor(ResultsEdits & parent, size_t index);
+
+    // Sets relevance to |relevance|. Returns true iff |relevance|
+    // differs from the original one.
+    bool Set(Relevance relevance);
+    std::optional<Relevance> const & Get() const;
+    bool HasChanges() const;
+    Entry::Type GetType() const;
+
+  private:
+    ResultsEdits & m_parent;
+    size_t m_index = 0;
+  };
+
+  explicit ResultsEdits(OnUpdate onUpdate) : m_onUpdate(onUpdate) {}
+
+  void Apply();
+  void Reset(std::vector<std::optional<Relevance>> const & relevances);
+
+  // Sets relevance at |index| to |relevance|. Returns true iff
+  // |relevance| differs from the original one.
+  bool SetRelevance(size_t index, Relevance relevance);
+
+  // Sets relevances of all entries to |relevance|.
+  void SetAllRelevances(Relevance relevance);
+
+  // Adds a new entry.
+  void Add(Relevance relevance);
+
+  // Marks entry at |index| as deleted.
+  void Delete(size_t index);
+
+  // Resurrects previously deleted entry at |index|.
+  void Resurrect(size_t index);
+
+  std::vector<Entry> const & GetEntries() const { return m_entries; }
+  Entry & GetEntry(size_t index);
+  Entry const & GetEntry(size_t index) const;
+  size_t NumEntries() const { return m_entries.size(); }
+  std::vector<std::optional<Relevance>> GetRelevances() const;
+
+  Entry const & Get(size_t index) const;
+
+  void Clear();
+  bool HasChanges() const;
+  bool HasChanges(size_t index) const;
+
+private:
+  template <typename Fn>
+  std::invoke_result_t<Fn> WithObserver(Update const & update, Fn && fn)
+  {
+    SCOPE_GUARD(obsCall, ([this, &update]()
+    {
+      if (m_onUpdate)
+        m_onUpdate(update);
+    }));
+    return fn();
+  }
+
+  std::vector<Entry> m_entries;
+
+  size_t m_numEdits = 0;
+
+  OnUpdate m_onUpdate;
+};

libs/search/search_quality/assessment_tool/feature_info_dialog.cpp

@@ -0,0 +1,112 @@
+#include "search/search_quality/assessment_tool/feature_info_dialog.hpp"
+
+#include "indexer/classificator.hpp"
+#include "indexer/map_object.hpp"
+
+#include "coding/string_utf8_multilang.hpp"
+
+#include "geometry/latlon.hpp"
+
+#include "base/assert.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <vector>
+
+#include <QtCore/QString>
+#include <QtGui/QAction>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QPushButton>
+
+using namespace std;
+
+namespace
+{
+QLabel * MakeSelectableLabel(string const & s)
+{
+  auto * result = new QLabel(QString::fromStdString(s));
+  result->setTextInteractionFlags(Qt::TextSelectableByMouse);
+  return result;
+}
+}  // namespace
+
+FeatureInfoDialog::FeatureInfoDialog(QWidget * parent, osm::MapObject const & mapObject,
+                                     search::ReverseGeocoder::Address const & address, string const & locale)
+  : QDialog(parent)
+{
+  auto * layout = new QGridLayout();
+
+  {
+    auto const & id = mapObject.GetID();
+    CHECK(id.IsValid(), ());
+
+    auto * label = new QLabel("id:");
+    auto * content = MakeSelectableLabel(id.GetMwmName() + ", " + strings::to_string(id.m_index));
+
+    AddRow(*layout, label, content);
+  }
+
+  {
+    auto * label = new QLabel("lat lon:");
+    auto const ll = mapObject.GetLatLon();
+    auto const ss = strings::to_string_dac(ll.m_lat, 5) + " " + strings::to_string_dac(ll.m_lon, 5);
+    auto * content = MakeSelectableLabel(ss);
+
+    AddRow(*layout, label, content);
+  }
+
+  {
+    int8_t const localeCode = StringUtf8Multilang::GetLangIndex(locale);
+    vector<int8_t> codes = {{StringUtf8Multilang::kDefaultCode, StringUtf8Multilang::kEnglishCode}};
+    if (localeCode != StringUtf8Multilang::kUnsupportedLanguageCode &&
+        ::find(codes.begin(), codes.end(), localeCode) == codes.end())
+    {
+      codes.push_back(localeCode);
+    }
+
+    for (auto const & code : codes)
+    {
+      string_view name;
+      if (!mapObject.GetNameMultilang().GetString(code, name))
+        continue;
+
+      auto const lang = StringUtf8Multilang::GetLangByCode(code);
+      CHECK(!lang.empty(), ("Can't find lang by code:", code));
+      auto * label = new QLabel(QString::fromStdString(std::string{lang} + ":"));
+      auto * content = MakeSelectableLabel(std::string{name});
+
+      AddRow(*layout, label, content);
+    }
+  }
+
+  {
+    auto const & c = classif();
+
+    vector<string> types;
+    for (auto type : mapObject.GetTypes())
+      types.push_back(c.GetReadableObjectName(type));
+
+    if (!types.empty())
+    {
+      auto * label = new QLabel("types:");
+      auto * content = MakeSelectableLabel(strings::JoinStrings(types, " " /* delimiter */));
+
+      AddRow(*layout, label, content);
+    }
+  }
+
+  {
+    auto * label = new QLabel("address:");
+    auto * content = MakeSelectableLabel(address.FormatAddress());
+
+    AddRow(*layout, label, content);
+  }
+
+  auto * ok = new QPushButton("OK");
+  ok->setDefault(true);
+  connect(ok, &QPushButton::clicked, [&]() { accept(); });
+  AddRow(*layout, ok);
+
+  setLayout(layout);
+}

libs/search/search_quality/assessment_tool/feature_info_dialog.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "search/reverse_geocoder.hpp"
+
+#include <string>
+
+#include <QtWidgets/QDialog>
+#include <QtWidgets/QGridLayout>
+
+namespace osm
+{
+class MapObject;
+}  // namespace osm
+
+class FeatureInfoDialog : public QDialog
+{
+  Q_OBJECT
+
+public:
+  FeatureInfoDialog(QWidget * parent, osm::MapObject const & mapObject,
+                    search::ReverseGeocoder::Address const & address, std::string const & locale);
+
+private:
+  void AddElems(QGridLayout & layout, int row, int col, QWidget * widget) { layout.addWidget(widget, row, col); }
+
+  template <typename... Args>
+  void AddElems(QGridLayout & layout, int row, int col, QWidget * widget, Args *... args)
+  {
+    AddElems(layout, row, col, widget);
+    AddElems(layout, row, col + 1, args...);
+  }
+
+  template <typename... Args>
+  void AddRow(QGridLayout & layout, Args *... args)
+  {
+    AddElems(layout, m_numRows /* row */, 0 /* col */, args...);
+    ++m_numRows;
+  }
+
+  int m_numRows = 0;
+};

libs/search/search_quality/assessment_tool/helpers.cpp

@@ -0,0 +1,13 @@
+#include "search/search_quality/assessment_tool/helpers.hpp"
+
+#include "base/string_utils.hpp"
+
+QString ToQString(strings::UniString const & s)
+{
+  return QString::fromUtf8(strings::ToUtf8(s).c_str());
+}
+
+QString ToQString(std::string const & s)
+{
+  return QString::fromUtf8(s.c_str());
+}

libs/search/search_quality/assessment_tool/helpers.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <string>
+#include <utility>
+
+#include <QtCore/QString>
+
+namespace strings
+{
+class UniString;
+}
+
+QString ToQString(strings::UniString const & s);
+QString ToQString(std::string const & s);
+
+template <typename Layout, typename... Args>
+Layout * BuildLayout(Args &&... args)
+{
+  return new Layout(std::forward<Args>(args)...);
+}
+
+template <typename Layout, typename... Args>
+Layout * BuildLayoutWithoutMargins(Args &&... args)
+{
+  auto * layout = BuildLayout<Layout>(std::forward<Args>(args)...);
+  layout->setContentsMargins(0 /* left */, 0 /* top */, 0 /* right */, 0 /* bottom */);
+  return layout;
+}

libs/search/search_quality/assessment_tool/main_model.cpp

@@ -0,0 +1,387 @@
+#include "main_model.hpp"
+#include "view.hpp"
+
+#include "search/feature_loader.hpp"
+#include "search/search_params.hpp"
+#include "search/search_quality/helpers.hpp"
+#include "search/search_quality/matcher.hpp"
+
+#include "map/framework.hpp"
+
+#include "geometry/algorithm.hpp"
+#include "geometry/mercator.hpp"
+
+#include "platform/platform.hpp"
+
+#include "base/assert.hpp"
+
+#include <algorithm>
+#include <fstream>
+#include <iterator>
+
+using namespace std;
+
+// MainModel ---------------------------------------------------------------------------------------
+MainModel::MainModel(Framework & framework)
+  : m_framework(framework)
+  , m_dataSource(m_framework.GetDataSource())
+  , m_loader(m_dataSource)
+  , m_contexts([this](size_t sampleIndex, ResultsEdits::Update const & update)
+{ OnUpdate(View::ResultType::Found, sampleIndex, update); },
+               [this](size_t sampleIndex, ResultsEdits::Update const & update)
+{ OnUpdate(View::ResultType::NonFound, sampleIndex, update); },
+               [this](size_t sampleIndex) { OnSampleUpdate(sampleIndex); })
+  , m_runner(m_framework, m_dataSource, m_contexts,
+             [this](search::Results const & results) { UpdateViewOnResults(results); }, [this](size_t index)
+{
+  // Only the first parameter matters because we only change SearchStatus.
+  m_view->OnSampleChanged(index, false /* isUseless */, false /* hasEdits */);
+})
+{
+  search::search_quality::CheckLocale();
+}
+
+void MainModel::Open(string const & path)
+{
+  CHECK(m_view, ());
+
+  string contents;
+
+  {
+    ifstream ifs(path);
+    if (!ifs)
+    {
+      m_view->ShowError("Can't open file: " + path);
+      return;
+    }
+    contents.assign(istreambuf_iterator<char>(ifs), istreambuf_iterator<char>());
+  }
+
+  vector<search::Sample> samples;
+  if (!search::Sample::DeserializeFromJSONLines(contents, samples))
+  {
+    m_view->ShowError("Can't parse samples: " + path);
+    return;
+  }
+
+  m_runner.ResetForegroundSearch();
+  m_runner.ResetBackgroundSearch();
+
+  m_view->Clear();
+
+  m_contexts.Resize(samples.size());
+  for (size_t i = 0; i < samples.size(); ++i)
+    m_contexts[i].LoadFromSample(samples[i]);
+
+  m_path = path;
+
+  m_view->SetSamples(ContextList::SamplesSlice(m_contexts));
+  m_selectedSample = kInvalidIndex;
+}
+
+void MainModel::Save()
+{
+  CHECK(HasChanges(), ());
+  SaveAs(m_path);
+}
+
+void MainModel::SaveAs(string const & path)
+{
+  CHECK(HasChanges(), ());
+  CHECK(!path.empty(), ());
+
+  string contents;
+  search::Sample::SerializeToJSONLines(m_contexts.MakeSamples(m_loader), contents);
+
+  {
+    ofstream ofs(path);
+    if (!ofs)
+    {
+      m_view->ShowError("Can't open file: " + path);
+      return;
+    }
+    copy(contents.begin(), contents.end(), ostreambuf_iterator<char>(ofs));
+  }
+
+  m_contexts.ApplyEdits();
+  m_path = path;
+}
+
+void MainModel::InitiateBackgroundSearch(size_t from, size_t to)
+{
+  m_runner.InitiateBackgroundSearch(from, to);
+}
+
+void MainModel::OnSampleSelected(int index)
+{
+  CHECK(m_threadChecker.CalledOnOriginalThread(), ());
+
+  CHECK_GREATER_OR_EQUAL(index, 0, ());
+  CHECK_LESS(static_cast<size_t>(index), m_contexts.Size(), ());
+  CHECK(m_view, ());
+
+  m_selectedSample = index;
+
+  auto & context = m_contexts[index];
+  auto const & sample = context.m_sample;
+
+  m_view->ShowSample(index, sample, sample.m_pos, context.IsUseless(), context.HasChanges());
+
+  m_runner.ResetForegroundSearch();
+
+  if (context.m_initialized)
+  {
+    UpdateViewOnResults(context.m_foundResults);
+    return;
+  }
+
+  InitiateForegroundSearch(index);
+}
+
+void MainModel::OnResultSelected(int index)
+{
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+  auto const & context = m_contexts[m_selectedSample];
+  auto const & foundResults = context.m_foundResults;
+
+  CHECK_GREATER_OR_EQUAL(index, 0, ());
+  CHECK_LESS(static_cast<size_t>(index), foundResults.GetCount(), ());
+  m_view->MoveViewportToResult(foundResults[index]);
+}
+
+void MainModel::OnNonFoundResultSelected(int index)
+{
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+  auto const & context = m_contexts[m_selectedSample];
+  auto const & results = context.m_nonFoundResults;
+
+  CHECK_GREATER_OR_EQUAL(index, 0, ());
+  CHECK_LESS(static_cast<size_t>(index), results.size(), ());
+  m_view->MoveViewportToResult(results[index]);
+}
+
+void MainModel::OnShowViewportClicked()
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  auto const & context = m_contexts[m_selectedSample];
+  m_view->MoveViewportToRect(context.m_sample.m_viewport);
+}
+
+void MainModel::OnShowPositionClicked()
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  static int constexpr kViewportAroundTopResultsSizeM = 100;
+  static double constexpr kViewportAroundTopResultsScale = 1.2;
+  static size_t constexpr kMaxTopResults = 3;
+
+  auto const & context = m_contexts[m_selectedSample];
+
+  vector<m2::PointD> points;
+  if (context.m_sample.m_pos)
+    points.push_back(*context.m_sample.m_pos);
+
+  size_t resultsAdded = 0;
+  for (auto const & result : context.m_foundResults)
+  {
+    if (!result.HasPoint())
+      continue;
+
+    if (resultsAdded == kMaxTopResults)
+      break;
+
+    points.push_back(result.GetFeatureCenter());
+    ++resultsAdded;
+  }
+
+  CHECK(!points.empty(), ());
+  auto boundingBox = m2::ApplyCalculator(points, m2::CalculateBoundingBox());
+  boundingBox.Scale(kViewportAroundTopResultsScale);
+
+  auto const minRect = mercator::RectByCenterXYAndSizeInMeters(boundingBox.Center(), kViewportAroundTopResultsSizeM);
+  m_view->MoveViewportToRect(m2::Add(boundingBox, minRect));
+}
+
+void MainModel::OnMarkAllAsRelevantClicked()
+{
+  OnChangeAllRelevancesClicked(ResultsEdits::Relevance::Relevant);
+}
+
+void MainModel::OnMarkAllAsIrrelevantClicked()
+{
+  OnChangeAllRelevancesClicked(ResultsEdits::Relevance::Irrelevant);
+}
+
+bool MainModel::HasChanges()
+{
+  return m_contexts.HasChanges();
+}
+
+bool MainModel::AlreadyInSamples(FeatureID const & id)
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  bool found = false;
+  ForAnyMatchingEntry(m_contexts[m_selectedSample], id, [&](ResultsEdits & edits, size_t index)
+  {
+    auto const & entry = edits.GetEntry(index);
+    if (!entry.m_deleted)
+      found = true;
+  });
+  return found;
+}
+
+void MainModel::AddNonFoundResult(FeatureID const & id)
+{
+  CHECK(m_selectedSample != kInvalidIndex, ());
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+
+  auto & context = m_contexts[m_selectedSample];
+
+  bool resurrected = false;
+  ForAnyMatchingEntry(context, id, [&](ResultsEdits & edits, size_t index)
+  {
+    auto const & entry = edits.GetEntry(index);
+    CHECK(entry.m_deleted, ());
+    edits.Resurrect(index);
+    resurrected = true;
+  });
+  if (resurrected)
+    return;
+
+  auto ft = m_loader.Load(id);
+  CHECK(ft, ("Can't load feature:", id));
+  auto const result = search::Sample::Result::Build(*ft, search::Sample::Result::Relevance::Vital);
+  context.AddNonFoundResult(result);
+}
+
+void MainModel::FlipSampleUsefulness(int index)
+{
+  CHECK_EQUAL(m_selectedSample, index, ());
+
+  m_contexts[index].m_sampleEdits.FlipUsefulness();
+
+  // Don't bother with resetting search: we cannot tell whether
+  // the sample is useless without its results anyway.
+}
+
+void MainModel::InitiateForegroundSearch(size_t index)
+{
+  auto & context = m_contexts[index];
+  auto const & sample = context.m_sample;
+
+  m_view->ShowSample(index, sample, sample.m_pos, context.IsUseless(), context.HasChanges());
+  m_runner.InitiateForegroundSearch(index);
+  m_view->OnSearchStarted();
+}
+
+void MainModel::OnUpdate(View::ResultType type, size_t sampleIndex, ResultsEdits::Update const & update)
+{
+  using Type = ResultsEdits::Update::Type;
+
+  CHECK_LESS(sampleIndex, m_contexts.Size(), ());
+  auto & context = m_contexts[sampleIndex];
+
+  if (update.m_type == Type::Add)
+  {
+    CHECK_EQUAL(type, View::ResultType::NonFound, ());
+    m_view->ShowNonFoundResults(context.m_nonFoundResults, context.m_nonFoundResultsEdits.GetEntries());
+    m_view->SetResultsEdits(m_selectedSample, context.m_foundResultsEdits, context.m_nonFoundResultsEdits);
+  }
+
+  m_view->OnResultChanged(sampleIndex, type, update);
+  m_view->OnSampleChanged(sampleIndex, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+
+  if (update.m_type == Type::Add || update.m_type == Type::Resurrect || update.m_type == Type::Delete)
+  {
+    CHECK(context.m_initialized, ());
+    CHECK_EQUAL(type, View::ResultType::NonFound, ());
+    m_view->ShowMarks(context);
+  }
+}
+
+void MainModel::OnSampleUpdate(size_t sampleIndex)
+{
+  auto & context = m_contexts[sampleIndex];
+
+  m_view->OnSampleChanged(sampleIndex, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+}
+
+void MainModel::UpdateViewOnResults(search::Results const & results)
+{
+  CHECK(m_threadChecker.CalledOnOriginalThread(), ());
+
+  m_view->AddFoundResults(results);
+
+  if (!results.IsEndedNormal())
+    return;
+
+  auto & context = m_contexts[m_selectedSample];
+
+  m_view->ShowNonFoundResults(context.m_nonFoundResults, context.m_nonFoundResultsEdits.GetEntries());
+  m_view->ShowMarks(context);
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::Found, ResultsEdits::Update::MakeAll());
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::NonFound, ResultsEdits::Update::MakeAll());
+  m_view->OnSampleChanged(m_selectedSample, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+
+  m_view->SetResultsEdits(m_selectedSample, context.m_foundResultsEdits, context.m_nonFoundResultsEdits);
+  m_view->OnSearchCompleted();
+}
+
+void MainModel::OnChangeAllRelevancesClicked(ResultsEdits::Relevance relevance)
+{
+  CHECK_GREATER_OR_EQUAL(m_selectedSample, 0, ());
+  CHECK_LESS(static_cast<size_t>(m_selectedSample), m_contexts.Size(), ());
+  auto & context = m_contexts[m_selectedSample];
+
+  context.m_foundResultsEdits.SetAllRelevances(relevance);
+  context.m_nonFoundResultsEdits.SetAllRelevances(relevance);
+
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::Found, ResultsEdits::Update::MakeAll());
+  m_view->OnResultChanged(m_selectedSample, View::ResultType::NonFound, ResultsEdits::Update::MakeAll());
+  m_view->OnSampleChanged(m_selectedSample, context.IsUseless(), context.HasChanges());
+  m_view->OnSamplesChanged(m_contexts.HasChanges());
+}
+
+template <typename Fn>
+void MainModel::ForAnyMatchingEntry(Context & context, FeatureID const & id, Fn && fn)
+{
+  CHECK(context.m_initialized, ());
+
+  auto const & foundResults = context.m_foundResults;
+  CHECK_EQUAL(foundResults.GetCount(), context.m_foundResultsEdits.NumEntries(), ());
+  for (size_t i = 0; i < foundResults.GetCount(); ++i)
+  {
+    auto const & result = foundResults[i];
+    if (result.GetResultType() != search::Result::Type::Feature)
+      continue;
+    if (result.GetFeatureID() == id)
+      return fn(context.m_foundResultsEdits, i);
+  }
+
+  auto ft = m_loader.Load(id);
+  CHECK(ft, ("Can't load feature:", id));
+  search::Matcher matcher(m_loader);
+
+  auto const & nonFoundResults = context.m_nonFoundResults;
+  CHECK_EQUAL(nonFoundResults.size(), context.m_nonFoundResultsEdits.NumEntries(), ());
+  for (size_t i = 0; i < nonFoundResults.size(); ++i)
+  {
+    auto const & result = context.m_nonFoundResults[i];
+    if (matcher.Matches(context.m_sample.m_query, result, *ft))
+      return fn(context.m_nonFoundResultsEdits, i);
+  }
+}

libs/search/search_quality/assessment_tool/main_model.hpp

@@ -0,0 +1,73 @@
+#pragma once
+#include "context.hpp"
+#include "edits.hpp"
+#include "model.hpp"
+#include "search_request_runner.hpp"
+#include "view.hpp"
+
+#include "search/feature_loader.hpp"
+
+#include "base/thread_checker.hpp"
+
+class Framework;
+class DataSource;
+
+namespace search
+{
+class Results;
+}
+
+class MainModel : public Model
+{
+public:
+  explicit MainModel(Framework & framework);
+
+  // Model overrides:
+  void Open(std::string const & path) override;
+  void Save() override;
+  void SaveAs(std::string const & path) override;
+  void InitiateBackgroundSearch(size_t const from, size_t const to) override;
+
+  void OnSampleSelected(int index) override;
+  void OnResultSelected(int index) override;
+  void OnNonFoundResultSelected(int index) override;
+  void OnShowViewportClicked() override;
+  void OnShowPositionClicked() override;
+  void OnMarkAllAsRelevantClicked() override;
+  void OnMarkAllAsIrrelevantClicked() override;
+  bool HasChanges() override;
+  bool AlreadyInSamples(FeatureID const & id) override;
+  void AddNonFoundResult(FeatureID const & id) override;
+  void FlipSampleUsefulness(int index) override;
+
+private:
+  static int constexpr kInvalidIndex = -1;
+
+  void InitiateForegroundSearch(size_t index);
+
+  void OnUpdate(View::ResultType type, size_t sampleIndex, ResultsEdits::Update const & update);
+  void OnSampleUpdate(size_t sampleIndex);
+
+  void UpdateViewOnResults(search::Results const & results);
+  void ShowMarks(Context const & context);
+
+  void OnChangeAllRelevancesClicked(ResultsEdits::Relevance relevance);
+
+  template <typename Fn>
+  void ForAnyMatchingEntry(Context & context, FeatureID const & id, Fn && fn);
+
+  Framework & m_framework;
+  DataSource const & m_dataSource;
+  search::FeatureLoader m_loader;
+
+  ContextList m_contexts;
+
+  // Path to the last file search samples were loaded from or saved to.
+  std::string m_path;
+
+  int m_selectedSample = kInvalidIndex;
+
+  SearchRequestRunner m_runner;
+
+  ThreadChecker m_threadChecker;
+};

libs/search/search_quality/assessment_tool/main_view.cpp

@@ -0,0 +1,516 @@
+#include "main_view.hpp"
+
+#include "feature_info_dialog.hpp"
+#include "helpers.hpp"
+#include "model.hpp"
+#include "results_view.hpp"
+#include "sample_view.hpp"
+#include "samples_view.hpp"
+
+#include "qt/qt_common/map_widget.hpp"
+#include "qt/qt_common/scale_slider.hpp"
+
+#include "map/framework.hpp"
+#include "map/place_page_info.hpp"
+
+#include "geometry/mercator.hpp"
+
+#include "base/assert.hpp"
+#include "base/checked_cast.hpp"
+#include "base/string_utils.hpp"
+
+#include <limits>
+
+#include <QtCore/Qt>
+#include <QtGui/QCloseEvent>
+#include <QtGui/QIntValidator>
+#include <QtGui/QKeySequence>
+#include <QtWidgets/QApplication>
+#include <QtWidgets/QDialog>
+#include <QtWidgets/QDialogButtonBox>
+#include <QtWidgets/QDockWidget>
+#include <QtWidgets/QFileDialog>
+#include <QtWidgets/QFormLayout>
+#include <QtWidgets/QHBoxLayout>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QLineEdit>
+#include <QtWidgets/QMenuBar>
+#include <QtWidgets/QMessageBox>
+#include <QtWidgets/QToolBar>
+
+using Relevance = search::Sample::Result::Relevance;
+
+namespace
+{
+char const kJSON[] = "JSON Lines files (*.jsonl)";
+}  // namespace
+
+MainView::MainView(Framework & framework, QRect const & screenGeometry) : m_framework(framework)
+{
+  setGeometry(screenGeometry);
+
+  setWindowTitle(tr("Assessment tool"));
+  InitMapWidget();
+  InitDocks();
+  InitMenuBar();
+
+  m_framework.SetPlacePageListeners([this]()
+  {
+    auto const & info = m_framework.GetCurrentPlacePageInfo();
+    auto const & selectedFeature = info.GetID();
+    if (!selectedFeature.IsValid())
+      return;
+    m_selectedFeature = selectedFeature;
+
+    if (m_skipFeatureInfoDialog)
+    {
+      m_skipFeatureInfoDialog = false;
+      return;
+    }
+
+    auto mapObject = m_framework.GetMapObjectByID(selectedFeature);
+    if (!mapObject.GetID().IsValid())
+      return;
+
+    auto const address = m_framework.GetAddressAtPoint(mapObject.GetMercator());
+    FeatureInfoDialog dialog(this /* parent */, mapObject, address, m_sampleLocale);
+    dialog.exec();
+  }, [this]() { m_selectedFeature = FeatureID(); }, {} /* onUpdate */, {} /* onSwitchFullScreenMode */);
+}
+
+MainView::~MainView()
+{
+  if (m_framework.GetDrapeEngine() != nullptr)
+  {
+    m_framework.EnterBackground();
+    m_framework.DestroyDrapeEngine();
+  }
+}
+
+void MainView::SetSamples(ContextList::SamplesSlice const & samples)
+{
+  m_samplesView->SetSamples(samples);
+  m_sampleView->Clear();
+  m_initiateBackgroundSearch->setEnabled(true);
+}
+
+void MainView::OnSearchStarted()
+{
+  m_state = State::Search;
+  m_sampleView->OnSearchStarted();
+}
+
+void MainView::OnSearchCompleted()
+{
+  m_state = State::AfterSearch;
+  m_sampleView->OnSearchCompleted();
+}
+
+void MainView::ShowSample(size_t sampleIndex, search::Sample const & sample, std::optional<m2::PointD> const & position,
+                          bool isUseless, bool hasEdits)
+{
+  m_sampleLocale = sample.m_locale;
+
+  MoveViewportToRect(sample.m_viewport);
+
+  m_sampleView->SetContents(sample, position);
+  m_sampleView->show();
+
+  OnResultChanged(sampleIndex, ResultType::Found, ResultsEdits::Update::MakeAll());
+  OnResultChanged(sampleIndex, ResultType::NonFound, ResultsEdits::Update::MakeAll());
+  OnSampleChanged(sampleIndex, isUseless, hasEdits);
+}
+
+void MainView::AddFoundResults(search::Results const & results)
+{
+  m_sampleView->AddFoundResults(results);
+}
+
+void MainView::ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                                   std::vector<ResultsEdits::Entry> const & entries)
+{
+  m_sampleView->ShowNonFoundResults(results, entries);
+}
+
+void MainView::ShowMarks(Context const & context)
+{
+  m_sampleView->ClearSearchResultMarks();
+  m_sampleView->ShowFoundResultsMarks(context.m_foundResults);
+  m_sampleView->ShowNonFoundResultsMarks(context.m_nonFoundResults, context.m_nonFoundResultsEdits.GetEntries());
+}
+
+void MainView::MoveViewportToResult(search::Result const & result)
+{
+  m_skipFeatureInfoDialog = true;
+  m_framework.SelectSearchResult(result, false /* animation */);
+}
+
+void MainView::MoveViewportToResult(search::Sample::Result const & result)
+{
+  int constexpr kViewportAroundResultSizeM = 100;
+  auto const rect = mercator::RectByCenterXYAndSizeInMeters(result.m_pos, kViewportAroundResultSizeM);
+  MoveViewportToRect(rect);
+}
+
+void MainView::MoveViewportToRect(m2::RectD const & rect)
+{
+  m_framework.ShowRect(rect, -1 /* maxScale */, false /* animation */);
+}
+
+void MainView::OnResultChanged(size_t sampleIndex, ResultType type, ResultsEdits::Update const & update)
+{
+  m_samplesView->OnUpdate(sampleIndex);
+
+  if (!m_samplesView->IsSelected(sampleIndex))
+    return;
+
+  switch (type)
+  {
+  case ResultType::Found: m_sampleView->GetFoundResultsView().Update(update); break;
+  case ResultType::NonFound: m_sampleView->GetNonFoundResultsView().Update(update); break;
+  }
+}
+
+void MainView::OnSampleChanged(size_t sampleIndex, bool isUseless, bool hasEdits)
+{
+  m_samplesView->OnUpdate(sampleIndex);
+  if (!m_samplesView->IsSelected(sampleIndex))
+    return;
+  SetSampleDockTitle(isUseless, hasEdits);
+  m_sampleView->OnUselessnessChanged(isUseless);
+}
+
+void MainView::OnSamplesChanged(bool hasEdits)
+{
+  SetSamplesDockTitle(hasEdits);
+  m_save->setEnabled(hasEdits);
+  m_saveAs->setEnabled(hasEdits);
+}
+
+void MainView::SetResultsEdits(size_t sampleIndex, ResultsEdits & foundResultsEdits,
+                               ResultsEdits & nonFoundResultsEdits)
+{
+  CHECK(m_samplesView->IsSelected(sampleIndex), ());
+  m_sampleView->SetResultsEdits(foundResultsEdits, nonFoundResultsEdits);
+}
+
+void MainView::ShowError(std::string const & msg)
+{
+  QMessageBox box(QMessageBox::Critical /* icon */, tr("Error") /* title */, QString::fromStdString(msg) /* text */,
+                  QMessageBox::Ok /* buttons */, this /* parent */);
+  box.exec();
+}
+
+void MainView::Clear()
+{
+  m_samplesView->Clear();
+  SetSamplesDockTitle(false /* hasEdits */);
+
+  m_sampleView->Clear();
+  SetSampleDockTitle(false /* isUseless */, false /* hasEdits */);
+
+  m_skipFeatureInfoDialog = false;
+  m_sampleLocale.clear();
+}
+
+void MainView::closeEvent(QCloseEvent * event)
+{
+  if (TryToSaveEdits(tr("Save changes before closing?")) == SaveResult::Cancelled)
+    event->ignore();
+  else
+    event->accept();
+}
+
+void MainView::OnSampleSelected(QItemSelection const & current)
+{
+  CHECK(m_model, ());
+  auto const indexes = current.indexes();
+  for (auto const & index : indexes)
+    m_model->OnSampleSelected(index.row());
+}
+
+void MainView::OnResultSelected(QItemSelection const & current)
+{
+  CHECK(m_model, ());
+  auto const indexes = current.indexes();
+  for (auto const & index : indexes)
+    m_model->OnResultSelected(index.row());
+}
+
+void MainView::OnNonFoundResultSelected(QItemSelection const & current)
+{
+  CHECK(m_model, ());
+  auto const indexes = current.indexes();
+  for (auto const & index : indexes)
+    m_model->OnNonFoundResultSelected(index.row());
+}
+
+void MainView::InitMenuBar()
+{
+  auto * bar = menuBar();
+
+  auto * fileMenu = bar->addMenu(tr("&File"));
+
+  {
+    auto * open = new QAction(tr("&Open samples..."), this /* parent */);
+    open->setShortcuts(QKeySequence::Open);
+    open->setStatusTip(tr("Open the file with samples for assessment"));
+    connect(open, &QAction::triggered, this, &MainView::Open);
+    fileMenu->addAction(open);
+  }
+
+  {
+    m_save = new QAction(tr("Save samples"), this /* parent */);
+    m_save->setShortcuts(QKeySequence::Save);
+    m_save->setStatusTip(tr("Save the file with assessed samples"));
+    m_save->setEnabled(false);
+    connect(m_save, &QAction::triggered, this, &MainView::Save);
+    fileMenu->addAction(m_save);
+  }
+
+  {
+    m_saveAs = new QAction(tr("Save samples as..."), this /* parent */);
+    m_saveAs->setShortcuts(QKeySequence::SaveAs);
+    m_saveAs->setStatusTip(tr("Save the file with assessed samples"));
+    m_saveAs->setEnabled(false);
+    connect(m_saveAs, &QAction::triggered, this, &MainView::SaveAs);
+    fileMenu->addAction(m_saveAs);
+  }
+
+  {
+    m_initiateBackgroundSearch = new QAction(tr("Initiate background search"), this /* parent */);
+    m_initiateBackgroundSearch->setShortcut(static_cast<int>(Qt::CTRL) | static_cast<int>(Qt::Key_I));
+    m_initiateBackgroundSearch->setStatusTip(tr("Search in the background for the queries from a selected range"));
+    m_initiateBackgroundSearch->setEnabled(false);
+    connect(m_initiateBackgroundSearch, &QAction::triggered, this, &MainView::InitiateBackgroundSearch);
+    fileMenu->addAction(m_initiateBackgroundSearch);
+  }
+
+  fileMenu->addSeparator();
+
+  {
+    auto * quit = new QAction(tr("&Quit"), this /* parent */);
+    quit->setShortcuts(QKeySequence::Quit);
+    quit->setStatusTip(tr("Exit the tool"));
+    connect(quit, &QAction::triggered, this, &QWidget::close);
+    fileMenu->addAction(quit);
+  }
+
+  auto * viewMenu = bar->addMenu(tr("&View"));
+
+  {
+    CHECK(m_samplesDock != nullptr, ());
+    viewMenu->addAction(m_samplesDock->toggleViewAction());
+  }
+
+  {
+    CHECK(m_sampleDock != nullptr, ());
+    viewMenu->addAction(m_sampleDock->toggleViewAction());
+  }
+}
+
+void MainView::InitMapWidget()
+{
+  auto * widget = new QWidget(this /* parent */);
+  auto * layout = BuildLayoutWithoutMargins<QHBoxLayout>(widget /* parent */);
+  widget->setLayout(layout);
+
+  {
+    auto * mapWidget = new qt::common::MapWidget(m_framework, false /* screenshotMode */, widget /* parent */);
+    connect(mapWidget, &qt::common::MapWidget::OnContextMenuRequested,
+            [this](QPoint const & p) { AddSelectedFeature(p); });
+    auto * toolBar = new QToolBar(widget /* parent */);
+    toolBar->setOrientation(Qt::Vertical);
+    toolBar->setIconSize(QSize(32, 32));
+    qt::common::ScaleSlider::Embed(Qt::Vertical, *toolBar, *mapWidget);
+
+    layout->addWidget(mapWidget);
+    layout->addWidget(toolBar);
+  }
+
+  setCentralWidget(widget);
+}
+
+void MainView::InitDocks()
+{
+  m_samplesView = new SamplesView(this /* parent */);
+
+  {
+    auto * model = m_samplesView->selectionModel();
+    connect(model, &QItemSelectionModel::selectionChanged, this, &MainView::OnSampleSelected);
+  }
+
+  connect(m_samplesView, &SamplesView::FlipSampleUsefulness,
+          [this](int index) { m_model->FlipSampleUsefulness(index); });
+
+  m_samplesDock = CreateDock(*m_samplesView);
+  addDockWidget(Qt::LeftDockWidgetArea, m_samplesDock);
+  SetSamplesDockTitle(false /* hasEdits */);
+
+  m_sampleView = new SampleView(this /* parent */, m_framework);
+
+  connect(m_sampleView, &SampleView::OnShowViewportClicked, [this]() { m_model->OnShowViewportClicked(); });
+  connect(m_sampleView, &SampleView::OnShowPositionClicked, [this]() { m_model->OnShowPositionClicked(); });
+
+  connect(m_sampleView, &SampleView::OnMarkAllAsRelevantClicked, [this]() { m_model->OnMarkAllAsRelevantClicked(); });
+  connect(m_sampleView, &SampleView::OnMarkAllAsIrrelevantClicked,
+          [this]() { m_model->OnMarkAllAsIrrelevantClicked(); });
+
+  {
+    auto const & view = m_sampleView->GetFoundResultsView();
+    connect(&view, &ResultsView::OnResultSelected, [this](int index) { m_model->OnResultSelected(index); });
+  }
+
+  {
+    auto const & view = m_sampleView->GetNonFoundResultsView();
+    connect(&view, &ResultsView::OnResultSelected, [this](int index) { m_model->OnNonFoundResultSelected(index); });
+  }
+
+  m_sampleDock = CreateDock(*m_sampleView);
+  connect(m_sampleDock, &QDockWidget::dockLocationChanged,
+          [this](Qt::DockWidgetArea area) { m_sampleView->OnLocationChanged(area); });
+  addDockWidget(Qt::RightDockWidgetArea, m_sampleDock);
+  SetSampleDockTitle(false /* isUseless */, false /* hasEdits */);
+}
+
+void MainView::Open()
+{
+  CHECK(m_model, ());
+
+  if (TryToSaveEdits(tr("Save changes before opening samples?")) == SaveResult::Cancelled)
+    return;
+
+  auto const name = QFileDialog::getOpenFileName(this /* parent */, tr("Open samples..."), QString() /* dir */, kJSON);
+  auto const file = name.toStdString();
+  if (file.empty())
+    return;
+
+  m_model->Open(file);
+}
+
+void MainView::Save()
+{
+  m_model->Save();
+}
+
+void MainView::SaveAs()
+{
+  auto const name =
+      QFileDialog::getSaveFileName(this /* parent */, tr("Save samples as..."), QString() /* dir */, kJSON);
+  auto const file = name.toStdString();
+  if (!file.empty())
+    m_model->SaveAs(file);
+}
+
+void MainView::InitiateBackgroundSearch()
+{
+  QDialog dialog(this);
+  QFormLayout form(&dialog);
+
+  form.addRow(new QLabel("Queries range"));
+
+  QValidator * validator = new QIntValidator(0, std::numeric_limits<int>::max(), this);
+
+  QLineEdit * lineEditFrom = new QLineEdit(&dialog);
+  form.addRow(new QLabel("First"), lineEditFrom);
+  lineEditFrom->setValidator(validator);
+
+  QLineEdit * lineEditTo = new QLineEdit(&dialog);
+  form.addRow(new QLabel("Last"), lineEditTo);
+  lineEditTo->setValidator(validator);
+
+  QDialogButtonBox buttonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel, Qt::Horizontal, &dialog);
+  form.addRow(&buttonBox);
+
+  connect(&buttonBox, &QDialogButtonBox::accepted, &dialog, &QDialog::accept);
+  connect(&buttonBox, &QDialogButtonBox::rejected, &dialog, &QDialog::reject);
+
+  if (dialog.exec() != QDialog::Accepted)
+    return;
+
+  std::string const strFrom = lineEditFrom->text().toStdString();
+  std::string const strTo = lineEditTo->text().toStdString();
+  uint64_t from = 0;
+  uint64_t to = 0;
+  if (!strings::to_uint64(strFrom, from))
+  {
+    LOG(LERROR, ("Could not parse number from", strFrom));
+    return;
+  }
+  if (!strings::to_uint64(strTo, to))
+  {
+    LOG(LERROR, ("Could not parse number from", strTo));
+    return;
+  }
+
+  m_model->InitiateBackgroundSearch(base::checked_cast<size_t>(from), base::checked_cast<size_t>(to));
+}
+
+void MainView::SetSamplesDockTitle(bool hasEdits)
+{
+  CHECK(m_samplesDock, ());
+  if (hasEdits)
+    m_samplesDock->setWindowTitle(tr("Samples *"));
+  else
+    m_samplesDock->setWindowTitle(tr("Samples"));
+}
+
+void MainView::SetSampleDockTitle(bool isUseless, bool hasEdits)
+{
+  CHECK(m_sampleDock, ());
+  std::string title = "Sample";
+  if (hasEdits)
+    title += " *";
+  if (isUseless)
+    title += " (useless)";
+  m_sampleDock->setWindowTitle(tr(title.data()));
+}
+
+MainView::SaveResult MainView::TryToSaveEdits(QString const & msg)
+{
+  CHECK(m_model, ());
+
+  if (!m_model->HasChanges())
+    return SaveResult::NoEdits;
+
+  QMessageBox box(QMessageBox::Question /* icon */, tr("Save edits?") /* title */, msg /* text */,
+                  QMessageBox::Save | QMessageBox::Discard | QMessageBox::Cancel /* buttons */, this /* parent */);
+  auto const button = box.exec();
+  switch (button)
+  {
+  case QMessageBox::Save: Save(); return SaveResult::Saved;
+  case QMessageBox::Discard: return SaveResult::Discarded;
+  case QMessageBox::Cancel: return SaveResult::Cancelled;
+  }
+
+  CHECK(false, ());
+  return SaveResult::Cancelled;
+}
+
+void MainView::AddSelectedFeature(QPoint const & p)
+{
+  auto const selectedFeature = m_selectedFeature;
+
+  if (!selectedFeature.IsValid())
+    return;
+
+  if (m_state != State::AfterSearch)
+    return;
+
+  if (m_model->AlreadyInSamples(selectedFeature))
+    return;
+
+  QMenu menu;
+  auto const * action = menu.addAction("Add to non-found results");
+  connect(action, &QAction::triggered, [this, selectedFeature]() { m_model->AddNonFoundResult(selectedFeature); });
+  menu.exec(p);
+}
+
+QDockWidget * MainView::CreateDock(QWidget & widget)
+{
+  auto * dock = new QDockWidget(QString(), this /* parent */, Qt::Widget);
+  dock->setFeatures(QDockWidget::DockWidgetClosable | QDockWidget::DockWidgetMovable |
+                    QDockWidget::DockWidgetFloatable);
+  dock->setWidget(&widget);
+  return dock;
+}

libs/search/search_quality/assessment_tool/main_view.hpp

@@ -0,0 +1,128 @@
+#pragma once
+#include "view.hpp"
+
+#include "indexer/feature_decl.hpp"
+
+#include <QtWidgets/QMainWindow>
+
+class Framework;
+class QDockWidget;
+class QItemSelection;
+class SamplesView;
+class SampleView;
+
+namespace qt
+{
+namespace common
+{
+class MapWidget;
+}
+}  // namespace qt
+
+class MainView
+  : public QMainWindow
+  , public View
+{
+  Q_OBJECT
+
+public:
+  explicit MainView(Framework & framework, QRect const & screenGeometry);
+  ~MainView() override;
+
+  // View overrides:
+  void SetSamples(ContextList::SamplesSlice const & samples) override;
+  void OnSearchStarted() override;
+  void OnSearchCompleted() override;
+  void ShowSample(size_t sampleIndex, search::Sample const & sample, std::optional<m2::PointD> const & position,
+                  bool isUseless, bool hasEdits) override;
+
+  void AddFoundResults(search::Results const & results) override;
+  void ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                           std::vector<ResultsEdits::Entry> const & entries) override;
+
+  void ShowMarks(Context const & context) override;
+
+  void MoveViewportToResult(search::Result const & result) override;
+  void MoveViewportToResult(search::Sample::Result const & result) override;
+  void MoveViewportToRect(m2::RectD const & rect) override;
+
+  void OnResultChanged(size_t sampleIndex, ResultType type, ResultsEdits::Update const & update) override;
+  void SetResultsEdits(size_t sampleIndex, ResultsEdits & foundResultsResultsEdits,
+                       ResultsEdits & nonFoundResultsResultsEdits) override;
+  void OnSampleChanged(size_t sampleIndex, bool isUseless, bool hasEdits) override;
+  void OnSamplesChanged(bool hasEdits) override;
+
+  void ShowError(std::string const & msg) override;
+
+  void Clear() override;
+
+protected:
+  // QMainWindow overrides:
+  void closeEvent(QCloseEvent * event) override;
+
+private slots:
+  void OnSampleSelected(QItemSelection const & current);
+  void OnResultSelected(QItemSelection const & current);
+  void OnNonFoundResultSelected(QItemSelection const & current);
+
+private:
+  enum class State
+  {
+    BeforeSearch,
+    Search,
+    AfterSearch
+  };
+
+  friend std::string DebugPrint(State state)
+  {
+    switch (state)
+    {
+    case State::BeforeSearch: return "BeforeSearch";
+    case State::Search: return "Search";
+    case State::AfterSearch: return "AfterSearch";
+    }
+  }
+
+  enum class SaveResult
+  {
+    NoEdits,
+    Saved,
+    Discarded,
+    Cancelled
+  };
+
+  void InitMapWidget();
+  void InitDocks();
+  void InitMenuBar();
+
+  void Open();
+  void Save();
+  void SaveAs();
+  void InitiateBackgroundSearch();
+
+  void SetSamplesDockTitle(bool hasEdits);
+  void SetSampleDockTitle(bool isUseless, bool hasEdits);
+  SaveResult TryToSaveEdits(QString const & msg);
+
+  void AddSelectedFeature(QPoint const & p);
+
+  QDockWidget * CreateDock(QWidget & widget);
+
+  Framework & m_framework;
+
+  SamplesView * m_samplesView = nullptr;
+  QDockWidget * m_samplesDock = nullptr;
+
+  SampleView * m_sampleView = nullptr;
+  QDockWidget * m_sampleDock = nullptr;
+
+  QAction * m_save = nullptr;
+  QAction * m_saveAs = nullptr;
+  QAction * m_initiateBackgroundSearch = nullptr;
+
+  State m_state = State::BeforeSearch;
+  FeatureID m_selectedFeature;
+
+  bool m_skipFeatureInfoDialog = false;
+  std::string m_sampleLocale;
+};

libs/search/search_quality/assessment_tool/model.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <string>
+
+class View;
+struct FeatureID;
+
+class Model
+{
+public:
+  virtual ~Model() = default;
+
+  void SetView(View & view) { m_view = &view; }
+
+  virtual void Open(std::string const & path) = 0;
+  virtual void Save() = 0;
+  virtual void SaveAs(std::string const & path) = 0;
+
+  // Initiates the search in the background on all samples
+  // in the 1-based range [|from|, |to|], both ends included.
+  // Another background search that may currently be running will be cancelled
+  // but the results for already completed requests will not be discarded.
+  //
+  // Does nothing if the range is invalid.
+  virtual void InitiateBackgroundSearch(size_t from, size_t to) = 0;
+
+  virtual void OnSampleSelected(int index) = 0;
+  virtual void OnResultSelected(int index) = 0;
+  virtual void OnNonFoundResultSelected(int index) = 0;
+  virtual void OnShowViewportClicked() = 0;
+  virtual void OnShowPositionClicked() = 0;
+  virtual void OnMarkAllAsRelevantClicked() = 0;
+  virtual void OnMarkAllAsIrrelevantClicked() = 0;
+  virtual bool HasChanges() = 0;
+
+  virtual bool AlreadyInSamples(FeatureID const & id) = 0;
+  virtual void AddNonFoundResult(FeatureID const & id) = 0;
+  virtual void FlipSampleUsefulness(int index) = 0;
+
+protected:
+  View * m_view = nullptr;
+};

libs/search/search_quality/assessment_tool/result_view.cpp

@@ -0,0 +1,185 @@
+#include "search/search_quality/assessment_tool/result_view.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/assessment_tool/helpers.hpp"
+
+#include <memory>
+#include <utility>
+
+#include <QtWidgets/QHBoxLayout>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QRadioButton>
+#include <QtWidgets/QVBoxLayout>
+
+using namespace std;
+
+namespace
+{
+QLabel * CreateLabel(QWidget & parent)
+{
+  QLabel * label = new QLabel(&parent);
+  label->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Minimum);
+  label->setWordWrap(true);
+  return label;
+}
+
+void SetText(QLabel & label, string const & text)
+{
+  if (text.empty())
+  {
+    label.hide();
+    return;
+  }
+
+  label.setText(ToQString(text));
+  label.show();
+}
+
+string GetResultType(search::Sample::Result const & result)
+{
+  return strings::JoinStrings(result.m_types, ", ");
+}
+
+string GetResultType(search::Result const & result)
+{
+  return (result.GetResultType() == search::Result::Type::Feature ? result.GetLocalizedFeatureType() : "");
+}
+}  // namespace
+
+ResultView::ResultView(string const & name, string const & type, string const & address, QWidget & parent)
+  : QWidget(&parent)
+{
+  Init();
+  SetContents(name, type, address);
+  setEnabled(false);
+  setObjectName("result");
+}
+
+ResultView::ResultView(search::Result const & result, QWidget & parent)
+  : ResultView(result.GetString(), GetResultType(result), result.GetAddress(), parent)
+{}
+
+ResultView::ResultView(search::Sample::Result const & result, QWidget & parent)
+  : ResultView(strings::ToUtf8(result.m_name), GetResultType(result), {} /* address */, parent)
+{}
+
+void ResultView::SetEditor(ResultsEdits::Editor && editor)
+{
+  m_editor = make_unique<ResultsEdits::Editor>(std::move(editor));
+
+  UpdateRelevanceRadioButtons();
+
+  setEnabled(true);
+}
+
+void ResultView::Update()
+{
+  if (!m_editor)
+  {
+    setStyleSheet("");
+    return;
+  }
+
+  if (m_editor->GetType() == ResultsEdits::Entry::Type::Created)
+    setStyleSheet("#result {background: rgba(173, 223, 173, 50%)}");
+  else if (m_editor->HasChanges())
+    setStyleSheet("#result {background: rgba(255, 255, 200, 50%)}");
+  else
+    setStyleSheet("");
+
+  UpdateRelevanceRadioButtons();
+}
+
+void ResultView::Init()
+{
+  auto * layout = new QVBoxLayout(this /* parent */);
+  layout->setSizeConstraint(QLayout::SetMinimumSize);
+  setLayout(layout);
+
+  m_name = CreateLabel(*this /* parent */);
+  layout->addWidget(m_name);
+
+  m_type = CreateLabel(*this /* parent */);
+  m_type->setStyleSheet("QLabel { font-size : 8pt }");
+  layout->addWidget(m_type);
+
+  m_address = CreateLabel(*this /* parent */);
+  m_address->setStyleSheet("QLabel { color : green ; font-size : 8pt }");
+  layout->addWidget(m_address);
+
+  {
+    auto * group = new QWidget(this /* parent */);
+    auto * groupLayout = new QHBoxLayout(group /* parent */);
+    group->setLayout(groupLayout);
+
+    m_harmful = CreateRatioButton("Harmful", *groupLayout);
+    m_irrelevant = CreateRatioButton("Irrelevant", *groupLayout);
+    m_relevant = CreateRatioButton("Relevant", *groupLayout);
+    m_vital = CreateRatioButton("Vital", *groupLayout);
+
+    layout->addWidget(group);
+  }
+}
+
+void ResultView::SetContents(string const & name, string const & type, string const & address)
+{
+  SetText(*m_name, name);
+  SetText(*m_type, type);
+  SetText(*m_address, address);
+
+  m_harmful->setChecked(false);
+  m_irrelevant->setChecked(false);
+  m_relevant->setChecked(false);
+  m_vital->setChecked(false);
+}
+
+QRadioButton * ResultView::CreateRatioButton(string const & text, QLayout & layout)
+{
+  QRadioButton * radio = new QRadioButton(QString::fromStdString(text), this /* parent */);
+  layout.addWidget(radio);
+
+  connect(radio, &QRadioButton::toggled, this, &ResultView::OnRelevanceChanged);
+  return radio;
+}
+
+void ResultView::OnRelevanceChanged()
+{
+  if (!m_editor)
+    return;
+
+  auto relevance = Relevance::Irrelevant;
+  if (m_harmful->isChecked())
+    relevance = Relevance::Harmful;
+  else if (m_irrelevant->isChecked())
+    relevance = Relevance::Irrelevant;
+  else if (m_relevant->isChecked())
+    relevance = Relevance::Relevant;
+  else if (m_vital->isChecked())
+    relevance = Relevance::Vital;
+
+  m_editor->Set(relevance);
+}
+
+void ResultView::UpdateRelevanceRadioButtons()
+{
+  if (!m_editor)
+    return;
+
+  m_harmful->setChecked(false);
+  m_irrelevant->setChecked(false);
+  m_relevant->setChecked(false);
+  m_vital->setChecked(false);
+
+  auto const & r = m_editor->Get();
+
+  if (!r)
+    return;
+
+  switch (*r)
+  {
+  case Relevance::Harmful: m_harmful->setChecked(true); break;
+  case Relevance::Irrelevant: m_irrelevant->setChecked(true); break;
+  case Relevance::Relevant: m_relevant->setChecked(true); break;
+  case Relevance::Vital: m_vital->setChecked(true); break;
+  }
+}

libs/search/search_quality/assessment_tool/result_view.hpp

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/edits.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include <memory>
+#include <string>
+
+#include <QtWidgets/QWidget>
+
+class QLabel;
+class QRadioButton;
+
+namespace search
+{
+class Result;
+}
+
+class ResultView : public QWidget
+{
+public:
+  using Relevance = search::Sample::Result::Relevance;
+
+  ResultView(search::Result const & result, QWidget & parent);
+  ResultView(search::Sample::Result const & result, QWidget & parent);
+
+  void SetEditor(ResultsEdits::Editor && editor);
+
+  void Update();
+
+private:
+  ResultView(std::string const & name, std::string const & type, std::string const & address, QWidget & parent);
+
+  void Init();
+  void SetContents(std::string const & name, std::string const & type, std::string const & address);
+
+  QRadioButton * CreateRatioButton(std::string const & label, QLayout & layout);
+  void OnRelevanceChanged();
+  void UpdateRelevanceRadioButtons();
+
+  QLabel * m_name = nullptr;
+  QLabel * m_type = nullptr;
+  QLabel * m_address = nullptr;
+
+  QRadioButton * m_harmful = nullptr;
+  QRadioButton * m_irrelevant = nullptr;
+  QRadioButton * m_relevant = nullptr;
+  QRadioButton * m_vital = nullptr;
+
+  std::unique_ptr<ResultsEdits::Editor> m_editor;
+};

libs/search/search_quality/assessment_tool/results_view.cpp

@@ -0,0 +1,108 @@
+#include "search/search_quality/assessment_tool/results_view.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/assessment_tool/result_view.hpp"
+
+#include "base/assert.hpp"
+
+#include <QtWidgets/QListWidgetItem>
+
+ResultsView::ResultsView(QWidget & parent) : QListWidget(&parent)
+{
+  setAlternatingRowColors(true);
+
+  connect(selectionModel(), &QItemSelectionModel::selectionChanged, [&](QItemSelection const & current)
+  {
+    auto const indexes = current.indexes();
+    for (auto const & index : indexes)
+      emit OnResultSelected(index.row());
+  });
+
+  connect(this, &ResultsView::itemClicked, [&](QListWidgetItem * item)
+  {
+    auto const index = indexFromItem(item);
+    emit OnResultSelected(index.row());
+  });
+}
+
+void ResultsView::Add(search::Result const & result)
+{
+  AddImpl(result, false /* hidden */);
+  if (result.HasPoint())
+    m_hasResultsWithPoints = true;
+}
+
+void ResultsView::Add(search::Sample::Result const & result, ResultsEdits::Entry const & entry)
+{
+  AddImpl(result, entry.m_deleted /* hidden */);
+}
+
+ResultView & ResultsView::Get(size_t i)
+{
+  CHECK_LESS(i, Size(), ());
+  return *m_results[i];
+}
+
+ResultView const & ResultsView::Get(size_t i) const
+{
+  CHECK_LESS(i, Size(), ());
+  return *m_results[i];
+}
+
+void ResultsView::Update(ResultsEdits::Update const & update)
+{
+  switch (update.m_type)
+  {
+  case ResultsEdits::Update::Type::Single:
+  {
+    CHECK_LESS(update.m_index, m_results.size(), ());
+    m_results[update.m_index]->Update();
+    break;
+  }
+  case ResultsEdits::Update::Type::All:
+  {
+    for (auto * result : m_results)
+      result->Update();
+    break;
+  }
+  case ResultsEdits::Update::Type::Add:
+  {
+    CHECK_LESS(update.m_index, m_results.size(), ());
+    m_results[update.m_index]->Update();
+    break;
+  }
+  case ResultsEdits::Update::Type::Delete:
+  {
+    auto const index = update.m_index;
+    CHECK_LESS(index, Size(), ());
+    item(static_cast<int>(index))->setHidden(true);
+    break;
+  }
+  case ResultsEdits::Update::Type::Resurrect:
+    auto const index = update.m_index;
+    CHECK_LESS(index, Size(), ());
+    item(static_cast<int>(index))->setHidden(false);
+    break;
+  };
+}
+
+void ResultsView::Clear()
+{
+  m_results.clear();
+  m_hasResultsWithPoints = false;
+  clear();
+}
+
+template <typename Result>
+void ResultsView::AddImpl(Result const & result, bool hidden)
+{
+  auto * item = new QListWidgetItem(this /* parent */);
+  item->setHidden(hidden);
+  addItem(item);
+
+  auto * view = new ResultView(result, *this /* parent */);
+  item->setSizeHint(view->minimumSizeHint());
+  setItemWidget(item, view);
+
+  m_results.push_back(view);
+}

libs/search/search_quality/assessment_tool/results_view.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/edits.hpp"
+
+#include <cstddef>
+#include <vector>
+
+#include <QtWidgets/QListWidget>
+
+class QWidget;
+class ResultView;
+
+namespace search
+{
+class Result;
+}
+
+class ResultsView : public QListWidget
+{
+  Q_OBJECT
+
+public:
+  explicit ResultsView(QWidget & parent);
+
+  void Add(search::Result const & result);
+  void Add(search::Sample::Result const & result, ResultsEdits::Entry const & entry);
+
+  ResultView & Get(size_t i);
+  ResultView const & Get(size_t i) const;
+  void Update(ResultsEdits::Update const & update);
+
+  size_t Size() const { return m_results.size(); }
+  bool HasResultsWithPoints() const { return m_hasResultsWithPoints; }
+
+  void Clear();
+
+signals:
+  void OnResultSelected(int index);
+
+private:
+  template <typename Result>
+  void AddImpl(Result const & result, bool hidden);
+
+  std::vector<ResultView *> m_results;
+  bool m_hasResultsWithPoints = false;
+};

libs/search/search_quality/assessment_tool/sample_view.cpp

@@ -0,0 +1,387 @@
+#include "sample_view.hpp"
+
+#include "helpers.hpp"
+#include "result_view.hpp"
+#include "results_view.hpp"
+
+#include "qt/qt_common/spinner.hpp"
+
+#include "map/bookmark_manager.hpp"
+#include "map/framework.hpp"
+#include "map/search_mark.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include <QtGui/QStandardItem>
+#include <QtGui/QStandardItemModel>
+#include <QtWidgets/QComboBox>
+#include <QtWidgets/QGroupBox>
+#include <QtWidgets/QHBoxLayout>
+#include <QtWidgets/QHeaderView>
+#include <QtWidgets/QLabel>
+#include <QtWidgets/QLineEdit>
+#include <QtWidgets/QListWidget>
+#include <QtWidgets/QMenu>
+#include <QtWidgets/QPushButton>
+#include <QtWidgets/QVBoxLayout>
+
+namespace
+{
+template <typename Layout>
+Layout * BuildSubLayout(QLayout & mainLayout, QWidget & parent, QWidget ** box)
+{
+  *box = new QWidget(&parent);
+  auto * subLayout = BuildLayoutWithoutMargins<Layout>(*box /* parent */);
+  (*box)->setLayout(subLayout);
+  mainLayout.addWidget(*box);
+  return subLayout;
+}
+
+template <typename Layout>
+Layout * BuildSubLayout(QLayout & mainLayout, QWidget & parent)
+{
+  QWidget * box = nullptr;
+  return BuildSubLayout<Layout>(mainLayout, parent, &box);
+}
+
+void SetVerticalStretch(QWidget & widget, int stretch)
+{
+  auto policy = widget.sizePolicy();
+  policy.setVerticalStretch(stretch);
+  widget.setSizePolicy(policy);
+}
+}  // namespace
+
+SampleView::SampleView(QWidget * parent, Framework & framework) : QWidget(parent), m_framework(framework)
+{
+  m_framework.GetBookmarkManager().GetEditSession().SetIsVisible(UserMark::Type::SEARCH, true);
+  m_framework.GetBookmarkManager().GetEditSession().SetIsVisible(UserMark::Type::COLORED, true);
+
+  auto * mainLayout = BuildLayout<QVBoxLayout>(this /* parent */);
+
+  // When the dock for SampleView is attached to the right side of the
+  // screen, we don't need left margin, because of zoom in/zoom out
+  // slider. In other cases, it's better to keep left margin as is.
+  m_defaultMargins = mainLayout->contentsMargins();
+  m_rightAreaMargins = m_defaultMargins;
+  m_rightAreaMargins.setLeft(0);
+
+  {
+    m_query = new QLabel(this /* parent */);
+    m_query->setToolTip(tr("Query text"));
+    m_query->setWordWrap(true);
+    m_query->hide();
+    mainLayout->addWidget(m_query);
+  }
+
+  {
+    m_langs = new QLabel(this /* parent */);
+    m_langs->setToolTip(tr("Query input language"));
+    m_langs->hide();
+    mainLayout->addWidget(m_langs);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QHBoxLayout>(*mainLayout, *this /* parent */);
+
+    m_showViewport = new QPushButton(tr("Show viewport"), this /* parent */);
+    connect(m_showViewport, &QPushButton::clicked, [this]() { emit OnShowViewportClicked(); });
+    layout->addWidget(m_showViewport);
+
+    m_showPosition = new QPushButton(tr("Show position"), this /* parent */);
+    connect(m_showPosition, &QPushButton::clicked, [this]() { emit OnShowPositionClicked(); });
+    layout->addWidget(m_showPosition);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QVBoxLayout>(*mainLayout, *this /* parent */, &m_relatedQueriesBox);
+    SetVerticalStretch(*m_relatedQueriesBox, 1 /* stretch */);
+
+    layout->addWidget(new QLabel(tr("Related queries")));
+
+    m_relatedQueries = new QListWidget();
+    layout->addWidget(m_relatedQueries);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QHBoxLayout>(*mainLayout, *this /* parent */);
+
+    m_markAllAsRelevant = new QPushButton(tr("Mark all as Relevant"), this /* parent */);
+    connect(m_markAllAsRelevant, &QPushButton::clicked, [this]() { emit OnMarkAllAsRelevantClicked(); });
+    layout->addWidget(m_markAllAsRelevant);
+
+    m_markAllAsIrrelevant = new QPushButton(tr("Mark all as Irrelevant"), this /* parent */);
+    connect(m_markAllAsIrrelevant, &QPushButton::clicked, [this]() { emit OnMarkAllAsIrrelevantClicked(); });
+    layout->addWidget(m_markAllAsIrrelevant);
+  }
+
+  {
+    m_uselessnessLabel = new QLabel(this /* parent */);
+    m_uselessnessLabel->setText(tr("Sample is marked as useless"));
+    m_uselessnessLabel->hide();
+    mainLayout->addWidget(m_uselessnessLabel);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QVBoxLayout>(*mainLayout, *this /* parent */, &m_foundResultsBox);
+    SetVerticalStretch(*m_foundResultsBox, 4 /* stretch */);
+
+    {
+      auto * subLayout = BuildSubLayout<QHBoxLayout>(*layout, *this /* parent */);
+      subLayout->addWidget(new QLabel(tr("Found results")));
+
+      m_spinner = new Spinner();
+      subLayout->addWidget(&m_spinner->AsWidget());
+    }
+
+    m_foundResults = new ResultsView(*this /* parent */);
+    layout->addWidget(m_foundResults);
+  }
+
+  {
+    auto * layout = BuildSubLayout<QVBoxLayout>(*mainLayout, *this /* parent */, &m_nonFoundResultsBox);
+    SetVerticalStretch(*m_nonFoundResultsBox, 2 /* stretch */);
+
+    layout->addWidget(new QLabel(tr("Non found results")));
+
+    m_nonFoundResults = new ResultsView(*this /* parent */);
+    m_nonFoundResults->setContextMenuPolicy(Qt::CustomContextMenu);
+    connect(m_nonFoundResults, &ResultsView::customContextMenuRequested, [&](QPoint pos)
+    {
+      pos = m_nonFoundResults->mapToGlobal(pos);
+
+      auto const items = m_nonFoundResults->selectedItems();
+      for (auto const * item : items)
+      {
+        int const row = m_nonFoundResults->row(item);
+
+        QMenu menu;
+        auto const * action = menu.addAction("Remove result");
+        connect(action, &QAction::triggered, [this, row]() { OnRemoveNonFoundResult(row); });
+
+        menu.exec(pos);
+      }
+    });
+    layout->addWidget(m_nonFoundResults);
+  }
+
+  setLayout(mainLayout);
+
+  Clear();
+}
+
+void SampleView::SetContents(search::Sample const & sample, std::optional<m2::PointD> const & position)
+{
+  if (!sample.m_query.empty())
+  {
+    m_query->setText(ToQString(sample.m_query));
+    m_query->show();
+  }
+  if (!sample.m_locale.empty())
+  {
+    m_langs->setText(ToQString(sample.m_locale));
+    m_langs->show();
+  }
+  m_showViewport->setEnabled(true);
+
+  m_relatedQueries->clear();
+  for (auto const & query : sample.m_relatedQueries)
+    m_relatedQueries->addItem(ToQString(query));
+  if (m_relatedQueries->count() != 0)
+    m_relatedQueriesBox->show();
+
+  ClearAllResults();
+  m_position = position;
+  if (m_position)
+    ShowUserPosition(*m_position);
+  else
+    HideUserPosition();
+}
+
+void SampleView::OnSearchStarted()
+{
+  m_spinner->Show();
+  m_showPosition->setEnabled(false);
+
+  m_markAllAsRelevant->setEnabled(false);
+  m_markAllAsIrrelevant->setEnabled(false);
+}
+
+void SampleView::OnSearchCompleted()
+{
+  m_spinner->Hide();
+  auto const resultsAvailable = m_foundResults->HasResultsWithPoints();
+  if (m_position)
+  {
+    if (resultsAvailable)
+      m_showPosition->setText(tr("Show position and top results"));
+    else
+      m_showPosition->setText(tr("Show position"));
+    m_showPosition->setEnabled(true);
+  }
+  else if (resultsAvailable)
+  {
+    m_showPosition->setText(tr("Show results"));
+    m_showPosition->setEnabled(true);
+  }
+  else
+  {
+    m_showPosition->setEnabled(false);
+  }
+
+  m_markAllAsRelevant->setEnabled(resultsAvailable);
+  m_markAllAsIrrelevant->setEnabled(resultsAvailable);
+}
+
+void SampleView::AddFoundResults(search::Results const & results)
+{
+  /// @todo Should clear previous m_foundResults.
+  for (auto const & res : results)
+    m_foundResults->Add(res);
+}
+
+void SampleView::ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                                     std::vector<ResultsEdits::Entry> const & entries)
+{
+  CHECK_EQUAL(results.size(), entries.size(), ());
+
+  m_nonFoundResults->Clear();
+
+  bool allDeleted = true;
+  for (size_t i = 0; i < results.size(); ++i)
+  {
+    m_nonFoundResults->Add(results[i], entries[i]);
+    if (!entries[i].m_deleted)
+      allDeleted = false;
+  }
+  if (!allDeleted)
+    m_nonFoundResultsBox->show();
+}
+
+void SampleView::ShowFoundResultsMarks(search::Results const & results)
+{
+  /// @todo Should clear previous _found_ results marks, but keep _nonfound_ if any.
+  m_framework.FillSearchResultsMarks(false /* clear */, results);
+}
+
+void SampleView::ShowNonFoundResultsMarks(std::vector<search::Sample::Result> const & results,
+                                          std::vector<ResultsEdits::Entry> const & entries)
+
+{
+  CHECK_EQUAL(results.size(), entries.size(), ());
+
+  auto editSession = m_framework.GetBookmarkManager().GetEditSession();
+  editSession.SetIsVisible(UserMark::Type::SEARCH, true);
+
+  for (size_t i = 0; i < results.size(); ++i)
+  {
+    auto const & result = results[i];
+    auto const & entry = entries[i];
+    if (entry.m_deleted)
+      continue;
+
+    auto * mark = editSession.CreateUserMark<SearchMarkPoint>(result.m_pos);
+    mark->SetNotFoundType();
+  }
+}
+
+void SampleView::ClearSearchResultMarks()
+{
+  m_framework.GetBookmarkManager().GetEditSession().ClearGroup(UserMark::Type::SEARCH);
+}
+
+void SampleView::ClearAllResults()
+{
+  m_foundResults->Clear();
+  m_nonFoundResults->Clear();
+  m_nonFoundResultsBox->hide();
+  ClearSearchResultMarks();
+}
+
+void SampleView::SetResultsEdits(ResultsEdits & resultsResultsEdits, ResultsEdits & nonFoundResultsEdits)
+{
+  SetResultsEdits(*m_foundResults, resultsResultsEdits);
+  SetResultsEdits(*m_nonFoundResults, nonFoundResultsEdits);
+  m_nonFoundResultsEdits = &nonFoundResultsEdits;
+}
+
+void SampleView::OnUselessnessChanged(bool isUseless)
+{
+  if (isUseless)
+  {
+    m_uselessnessLabel->show();
+    m_foundResultsBox->hide();
+    m_nonFoundResultsBox->hide();
+    m_markAllAsRelevant->hide();
+    m_markAllAsIrrelevant->hide();
+  }
+  else
+  {
+    m_uselessnessLabel->hide();
+    m_foundResultsBox->show();
+    m_markAllAsRelevant->show();
+    m_markAllAsIrrelevant->show();
+  }
+}
+
+void SampleView::Clear()
+{
+  m_query->hide();
+  m_langs->hide();
+
+  m_showViewport->setEnabled(false);
+  m_showPosition->setEnabled(false);
+
+  m_markAllAsRelevant->setEnabled(false);
+  m_markAllAsIrrelevant->setEnabled(false);
+
+  m_relatedQueriesBox->hide();
+
+  ClearAllResults();
+  HideUserPosition();
+  m_position = std::nullopt;
+  OnSearchCompleted();
+}
+
+void SampleView::OnLocationChanged(Qt::DockWidgetArea area)
+{
+  if (area == Qt::RightDockWidgetArea)
+    layout()->setContentsMargins(m_rightAreaMargins);
+  else
+    layout()->setContentsMargins(m_defaultMargins);
+}
+
+void SampleView::SetResultsEdits(ResultsView & results, ResultsEdits & edits)
+{
+  size_t const numRelevances = edits.GetRelevances().size();
+  CHECK_EQUAL(results.Size(), numRelevances, ());
+  for (size_t i = 0; i < numRelevances; ++i)
+    results.Get(i).SetEditor(ResultsEdits::Editor(edits, i));
+}
+
+void SampleView::OnRemoveNonFoundResult(int row)
+{
+  m_nonFoundResultsEdits->Delete(row);
+}
+
+void SampleView::ShowUserPosition(m2::PointD const & position)
+{
+  // Clear the old position.
+  HideUserPosition();
+
+  auto es = m_framework.GetBookmarkManager().GetEditSession();
+  auto mark = es.CreateUserMark<ColoredMarkPoint>(position);
+  mark->SetColor(dp::Color(200, 100, 240, 255) /* purple */);
+  mark->SetRadius(8.0f);
+  m_positionMarkId = mark->GetId();
+}
+
+void SampleView::HideUserPosition()
+{
+  if (m_positionMarkId == kml::kInvalidMarkId)
+    return;
+
+  m_framework.GetBookmarkManager().GetEditSession().DeleteUserMark(m_positionMarkId);
+  m_positionMarkId = kml::kInvalidMarkId;
+}

libs/search/search_quality/assessment_tool/sample_view.hpp

@@ -0,0 +1,102 @@
+#pragma once
+#include "edits.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "geometry/point2d.hpp"
+
+#include "kml/type_utils.hpp"
+
+#include <optional>
+
+#include <QtCore/QMargins>
+#include <QtWidgets/QWidget>
+
+class Framework;
+class QLabel;
+class QListWidget;
+class QPushButton;
+class ResultsView;
+class Spinner;
+
+class SampleView : public QWidget
+{
+  Q_OBJECT
+
+public:
+  using Relevance = search::Sample::Result::Relevance;
+
+  SampleView(QWidget * parent, Framework & framework);
+
+  void SetContents(search::Sample const & sample, std::optional<m2::PointD> const & position);
+  void OnSearchStarted();
+  void OnSearchCompleted();
+
+  void AddFoundResults(search::Results const & results);
+  void ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                           std::vector<ResultsEdits::Entry> const & entries);
+
+  void ShowFoundResultsMarks(search::Results const & results);
+  void ShowNonFoundResultsMarks(std::vector<search::Sample::Result> const & results,
+                                std::vector<ResultsEdits::Entry> const & entries);
+  void ClearSearchResultMarks();
+
+  void SetResultsEdits(ResultsEdits & resultsResultsEdits, ResultsEdits & nonFoundResultsResultsEdits);
+
+  void OnUselessnessChanged(bool isUseless);
+
+  void Clear();
+
+  ResultsView & GetFoundResultsView() { return *m_foundResults; }
+  ResultsView & GetNonFoundResultsView() { return *m_nonFoundResults; }
+
+  void OnLocationChanged(Qt::DockWidgetArea area);
+
+signals:
+  void OnShowViewportClicked();
+  void OnShowPositionClicked();
+  void OnMarkAllAsRelevantClicked();
+  void OnMarkAllAsIrrelevantClicked();
+
+private:
+  void ClearAllResults();
+  void SetResultsEdits(ResultsView & results, ResultsEdits & edits);
+  void OnRemoveNonFoundResult(int row);
+
+  void ShowUserPosition(m2::PointD const & position);
+  void HideUserPosition();
+
+  Framework & m_framework;
+
+  Spinner * m_spinner = nullptr;
+
+  QLabel * m_query = nullptr;
+  QLabel * m_langs = nullptr;
+
+  QListWidget * m_relatedQueries = nullptr;
+  QWidget * m_relatedQueriesBox = nullptr;
+
+  QPushButton * m_showViewport = nullptr;
+  QPushButton * m_showPosition = nullptr;
+
+  QPushButton * m_markAllAsRelevant = nullptr;
+  QPushButton * m_markAllAsIrrelevant = nullptr;
+
+  QLabel * m_uselessnessLabel = nullptr;
+
+  ResultsView * m_foundResults = nullptr;
+  QWidget * m_foundResultsBox = nullptr;
+
+  ResultsView * m_nonFoundResults = nullptr;
+  QWidget * m_nonFoundResultsBox = nullptr;
+
+  ResultsEdits * m_nonFoundResultsEdits = nullptr;
+
+  QMargins m_rightAreaMargins;
+  QMargins m_defaultMargins;
+
+  kml::MarkId m_positionMarkId = kml::kInvalidMarkId;
+
+  std::optional<m2::PointD> m_position;
+};

libs/search/search_quality/assessment_tool/samples_view.cpp

@@ -0,0 +1,77 @@
+#include "search/search_quality/assessment_tool/samples_view.hpp"
+
+#include "search/search_quality/assessment_tool/helpers.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+
+#include <string>
+
+#include <QtGui/QAction>
+#include <QtGui/QContextMenuEvent>
+#include <QtGui/QStandardItem>
+#include <QtWidgets/QHeaderView>
+#include <QtWidgets/QMenu>
+
+// SamplesView::Model ------------------------------------------------------------------------------
+SamplesView::Model::Model(QWidget * parent) : QStandardItemModel(0 /* rows */, 1 /* columns */, parent) {}
+
+QVariant SamplesView::Model::data(QModelIndex const & index, int role) const
+{
+  auto const row = index.row();
+
+  if (role == Qt::DisplayRole && m_samples.IsValid())
+    return QString::fromStdString(m_samples.GetLabel(row));
+  if (role == Qt::BackgroundRole && m_samples.IsValid())
+  {
+    if (m_samples.IsChanged(row))
+      return QBrush(QColor(0xFF, 0xFF, 0xC8));
+
+    if (m_samples.GetSearchState(row) == Context::SearchState::InQueue)
+      return QBrush(QColor(0xFF, 0xCC, 0x66));
+
+    if (m_samples.GetSearchState(row) == Context::SearchState::Completed)
+      return QBrush(QColor(0xCA, 0xFE, 0xDB));
+
+    return QBrush(Qt::transparent);
+  }
+  return QStandardItemModel::data(index, role);
+}
+
+// SamplesView -------------------------------------------------------------------------------------
+SamplesView::SamplesView(QWidget * parent) : QTableView(parent)
+{
+  setEditTriggers(QAbstractItemView::NoEditTriggers);
+  setSelectionMode(QAbstractItemView::SingleSelection);
+
+  {
+    auto * header = horizontalHeader();
+    header->setStretchLastSection(true /* stretch */);
+    header->hide();
+  }
+
+  m_model = new Model(this /* parent */);
+  // TODO: Do not invoke virtual functions from constructor.
+  setModel(m_model);
+}
+
+bool SamplesView::IsSelected(size_t index) const
+{
+  return selectionModel()->isRowSelected(base::checked_cast<int>(index), QModelIndex());
+}
+
+void SamplesView::contextMenuEvent(QContextMenuEvent * event)
+{
+  QModelIndex modelIndex = selectionModel()->currentIndex();
+  if (!modelIndex.isValid())
+    return;
+
+  int const index = modelIndex.row();
+  bool const isUseless = m_model->SampleIsUseless(index);
+
+  QMenu menu(this);
+  auto const text = std::string(isUseless ? "unmark" : "mark") + " sample as useless";
+  auto const * action = menu.addAction(text.c_str());
+  connect(action, &QAction::triggered, [this, index]() { emit FlipSampleUsefulness(index); });
+  menu.exec(event->globalPos());
+}

libs/search/search_quality/assessment_tool/samples_view.hpp

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/context.hpp"
+
+#include "base/checked_cast.hpp"
+
+#include <cstddef>
+#include <vector>
+
+#include <QtGui/QStandardItemModel>
+#include <QtWidgets/QMainWindow>
+#include <QtWidgets/QTableView>
+
+class SamplesView : public QTableView
+{
+  Q_OBJECT
+
+public:
+  explicit SamplesView(QWidget * parent);
+
+  void SetSamples(ContextList::SamplesSlice const & samples) { m_model->SetSamples(samples); }
+  bool IsSelected(size_t index) const;
+  void OnUpdate(size_t index) { m_model->OnUpdate(index); }
+  void Clear() { m_model->SetSamples(ContextList::SamplesSlice{}); }
+
+  // QMainWindow overrides:
+  void contextMenuEvent(QContextMenuEvent * event) override;
+
+signals:
+  void FlipSampleUsefulness(int index);
+
+private:
+  class Model : public QStandardItemModel
+  {
+  public:
+    explicit Model(QWidget * parent);
+
+    void SetSamples(ContextList::SamplesSlice const & samples)
+    {
+      removeRows(0, rowCount());
+      m_samples = samples;
+      if (m_samples.IsValid())
+        insertRows(0, base::checked_cast<int>(m_samples.Size()));
+    }
+
+    void OnUpdate(size_t index)
+    {
+      auto const ix = createIndex(base::checked_cast<int>(index) /* row */, 0 /* column */);
+
+      // Need to refresh view when some item is updated.
+      emit dataChanged(ix, ix);
+    }
+
+    bool SampleIsUseless(int index) const { return m_samples.IsUseless(index); }
+
+    // QStandardItemModel overrides:
+    QVariant data(QModelIndex const & index, int role = Qt::DisplayRole) const override;
+
+  private:
+    ContextList::SamplesSlice m_samples;
+  };
+
+  Model * m_model = nullptr;
+};

libs/search/search_quality/assessment_tool/search_request_runner.cpp

@@ -0,0 +1,253 @@
+#include "search/search_quality/assessment_tool/search_request_runner.hpp"
+
+#include "search/feature_loader.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <utility>
+
+using namespace std;
+
+SearchRequestRunner::SearchRequestRunner(Framework & framework, DataSource const & dataSource, ContextList & contexts,
+                                         UpdateViewOnResults && updateViewOnResults,
+                                         UpdateSampleSearchState && updateSampleSearchState)
+  : m_framework(framework)
+  , m_dataSource(dataSource)
+  , m_contexts(contexts)
+  , m_updateViewOnResults(std::move(updateViewOnResults))
+  , m_updateSampleSearchState(std::move(updateSampleSearchState))
+{}
+
+void SearchRequestRunner::InitiateForegroundSearch(size_t index)
+{
+  RunRequest(index, false /* background */, m_foregroundTimestamp);
+}
+
+void SearchRequestRunner::InitiateBackgroundSearch(size_t from, size_t to)
+{
+  // 1 <= from <= to <= m_contexts.Size().
+  if (from < 1 || from > to || to > m_contexts.Size())
+  {
+    LOG(LINFO, ("Could not initiate search in the range", from, to, "Total samples:", m_contexts.Size()));
+    return;
+  }
+
+  ResetBackgroundSearch();
+
+  // Convert to 0-based.
+  --from;
+  --to;
+  m_backgroundFirstIndex = from;
+  m_backgroundLastIndex = to;
+  m_backgroundNumProcessed = 0;
+
+  for (size_t index = from; index <= to; ++index)
+  {
+    if (m_contexts[index].m_searchState == Context::SearchState::Untouched)
+    {
+      m_contexts[index].m_searchState = Context::SearchState::InQueue;
+      m_backgroundQueue.push(index);
+      m_updateSampleSearchState(index);
+    }
+    else
+    {
+      CHECK(m_contexts[index].m_searchState == Context::SearchState::Completed, ());
+      ++m_backgroundNumProcessed;
+      LOG(LINFO, ("Using results from an earlier search for request number", index + 1));
+      if (m_backgroundNumProcessed == to - from + 1)
+        PrintBackgroundSearchStats();
+    }
+  }
+
+  size_t const numThreads = m_framework.GetSearchAPI().GetEngine().GetNumThreads();
+  for (size_t i = 0; i < numThreads; ++i)
+    RunNextBackgroundRequest(m_backgroundTimestamp);
+}
+
+void SearchRequestRunner::ResetForegroundSearch()
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  ++m_foregroundTimestamp;
+  if (auto handle = m_foregroundQueryHandle.lock())
+    handle->Cancel();
+}
+
+void SearchRequestRunner::ResetBackgroundSearch()
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  ++m_backgroundTimestamp;
+
+  queue<size_t>().swap(m_backgroundQueue);
+  m_backgroundNumProcessed = 0;
+
+  bool cancelledAny = false;
+  for (auto const & entry : m_backgroundQueryHandles)
+  {
+    auto handle = entry.second.lock();
+    if (handle)
+    {
+      handle->Cancel();
+      cancelledAny = true;
+    }
+  }
+  m_backgroundQueryHandles.clear();
+
+  if (cancelledAny)
+  {
+    for (size_t index = m_backgroundFirstIndex; index <= m_backgroundLastIndex; ++index)
+    {
+      if (m_contexts[index].m_searchState == Context::SearchState::InQueue)
+      {
+        m_contexts[index].m_searchState = Context::SearchState::Untouched;
+        m_updateSampleSearchState(index);
+      }
+    }
+  }
+
+  m_backgroundFirstIndex = kInvalidIndex;
+  m_backgroundLastIndex = kInvalidIndex;
+}
+
+void SearchRequestRunner::RunNextBackgroundRequest(size_t timestamp)
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  if (m_backgroundQueue.empty())
+    return;
+
+  size_t index = m_backgroundQueue.front();
+  m_backgroundQueue.pop();
+
+  RunRequest(index, true /* background */, timestamp);
+}
+
+void SearchRequestRunner::RunRequest(size_t index, bool background, size_t timestamp)
+{
+  CHECK_THREAD_CHECKER(m_threadChecker, ());
+
+  auto const & context = m_contexts[index];
+  auto const & sample = context.m_sample;
+
+  search::SearchParams params;
+  sample.FillSearchParams(params);
+  params.m_onResults = [=, this](search::Results const & results)
+  {
+    vector<optional<ResultsEdits::Relevance>> relevances;
+    vector<size_t> goldenMatching;
+    vector<size_t> actualMatching;
+
+    if (results.IsEndedNormal())
+    {
+      // Can't use MainModel's m_loader here due to thread-safety issues.
+      search::FeatureLoader loader(m_dataSource);
+      search::Matcher matcher(loader);
+
+      vector<search::Result> const actual(results.begin(), results.end());
+      matcher.Match(sample, actual, goldenMatching, actualMatching);
+      relevances.resize(actual.size());
+      for (size_t i = 0; i < goldenMatching.size(); ++i)
+      {
+        auto const j = goldenMatching[i];
+        if (j != search::Matcher::kInvalidId)
+        {
+          CHECK_LESS(j, relevances.size(), ());
+          relevances[j] = sample.m_results[i].m_relevance;
+        }
+      }
+
+      LOG(LINFO, ("Request number", index + 1, "has been processed in the", background ? "background" : "foreground"));
+    }
+
+    GetPlatform().RunTask(Platform::Thread::Gui,
+                          [this, background, timestamp, index, results, relevances, goldenMatching, actualMatching]
+    {
+      size_t const latestTimestamp = background ? m_backgroundTimestamp : m_foregroundTimestamp;
+      if (timestamp != latestTimestamp)
+        return;
+
+      auto & context = m_contexts[index];
+
+      context.m_foundResults = results;
+
+      if (results.IsEndMarker())
+      {
+        if (results.IsEndedNormal())
+          context.m_searchState = Context::SearchState::Completed;
+        else
+          context.m_searchState = Context::SearchState::Untouched;
+        m_updateSampleSearchState(index);
+      }
+
+      if (results.IsEndedNormal())
+      {
+        if (!context.m_initialized)
+        {
+          context.m_foundResultsEdits.Reset(relevances);
+          context.m_goldenMatching = goldenMatching;
+          context.m_actualMatching = actualMatching;
+
+          {
+            vector<optional<ResultsEdits::Relevance>> relevances;
+
+            auto & nonFound = context.m_nonFoundResults;
+            CHECK(nonFound.empty(), ());
+            for (size_t i = 0; i < context.m_goldenMatching.size(); ++i)
+            {
+              auto const j = context.m_goldenMatching[i];
+              if (j != search::Matcher::kInvalidId)
+                continue;
+              nonFound.push_back(context.m_sample.m_results[i]);
+              relevances.emplace_back(nonFound.back().m_relevance);
+            }
+            context.m_nonFoundResultsEdits.Reset(relevances);
+          }
+
+          context.m_initialized = true;
+        }
+
+        if (background)
+        {
+          ++m_backgroundNumProcessed;
+          m_backgroundQueryHandles.erase(index);
+          if (m_backgroundNumProcessed == m_backgroundLastIndex - m_backgroundFirstIndex + 1)
+            PrintBackgroundSearchStats();
+          else
+            RunNextBackgroundRequest(timestamp);
+        }
+      }
+
+      if (!background)
+        m_updateViewOnResults(results);
+    });
+  };
+
+  auto & engine = m_framework.GetSearchAPI().GetEngine();
+  if (background)
+    m_backgroundQueryHandles[index] = engine.Search(std::move(params));
+  else
+    m_foregroundQueryHandle = engine.Search(std::move(params));
+}
+
+void SearchRequestRunner::PrintBackgroundSearchStats() const
+{
+  LOG(LINFO, ("All requests from", m_backgroundFirstIndex + 1, "to", m_backgroundLastIndex + 1, "have been processed"));
+
+  vector<size_t> vitals;
+  vitals.reserve(m_backgroundLastIndex - m_backgroundFirstIndex + 1);
+  for (size_t index = m_backgroundFirstIndex; index <= m_backgroundLastIndex; ++index)
+  {
+    auto const & entries = m_contexts[index].m_foundResultsEdits.GetEntries();
+    bool const foundVital = any_of(entries.begin(), entries.end(), [](ResultsEdits::Entry const & e)
+    { return e.m_currRelevance == search::Sample::Result::Relevance::Vital; });
+    if (foundVital)
+      vitals.emplace_back(index + 1);
+  }
+
+  LOG(LINFO, ("Vital results found:", vitals.size(), "out of", m_backgroundLastIndex - m_backgroundFirstIndex + 1));
+  LOG(LINFO, ("Vital results found for these queries (1-based):", vitals));
+}

libs/search/search_quality/assessment_tool/search_request_runner.hpp

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "search/search_quality/assessment_tool/context.hpp"
+
+#include "map/framework.hpp"
+
+#include "base/thread_checker.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <map>
+#include <memory>
+#include <queue>
+#include <vector>
+
+// A proxy for SearchAPI/SearchEngine.
+// This class updates the Model's |m_contexts| directly (from the main thread) and updates
+// the View via the |m_updateViewOnResults| and |m_updateSampleSearchState| callbacks.
+class SearchRequestRunner
+{
+public:
+  using UpdateViewOnResults = std::function<void(search::Results const & results)>;
+  using UpdateSampleSearchState = std::function<void(size_t index)>;
+
+  SearchRequestRunner(Framework & framework, DataSource const & dataSource, ContextList & contexts,
+                      UpdateViewOnResults && updateViewOnResults, UpdateSampleSearchState && updateSampleSearchState);
+
+  void InitiateForegroundSearch(size_t index);
+
+  void InitiateBackgroundSearch(size_t from, size_t to);
+
+  void ResetForegroundSearch();
+
+  void ResetBackgroundSearch();
+
+private:
+  static size_t constexpr kInvalidIndex = std::numeric_limits<size_t>::max();
+
+  // Tries to run the unprocessed request with the smallest index, if there is one.
+  void RunNextBackgroundRequest(size_t timestamp);
+
+  void RunRequest(size_t index, bool background, size_t timestamp);
+
+  void PrintBackgroundSearchStats() const;
+
+  Framework & m_framework;
+
+  DataSource const & m_dataSource;
+
+  ContextList & m_contexts;
+
+  UpdateViewOnResults m_updateViewOnResults;
+  UpdateSampleSearchState m_updateSampleSearchState;
+
+  std::weak_ptr<search::ProcessorHandle> m_foregroundQueryHandle;
+  std::map<size_t, std::weak_ptr<search::ProcessorHandle>> m_backgroundQueryHandles;
+
+  size_t m_foregroundTimestamp = 0;
+  size_t m_backgroundTimestamp = 0;
+
+  size_t m_backgroundFirstIndex = kInvalidIndex;
+  size_t m_backgroundLastIndex = kInvalidIndex;
+  std::queue<size_t> m_backgroundQueue;
+  size_t m_backgroundNumProcessed = 0;
+
+  ThreadChecker m_threadChecker;
+};

libs/search/search_quality/assessment_tool/view.hpp

@@ -0,0 +1,69 @@
+#pragma once
+#include "context.hpp"
+#include "edits.hpp"
+
+#include "search/result.hpp"
+#include "search/search_quality/sample.hpp"
+
+#include "geometry/point2d.hpp"
+#include "geometry/rect2d.hpp"
+
+#include <cstddef>
+#include <optional>
+#include <string>
+#include <vector>
+
+class ResultsEdits;
+class Model;
+
+class View
+{
+public:
+  enum class ResultType
+  {
+    Found,
+    NonFound
+  };
+
+  virtual ~View() = default;
+
+  void SetModel(Model & model) { m_model = &model; }
+
+  virtual void SetSamples(ContextList::SamplesSlice const & samples) = 0;
+  virtual void OnSearchStarted() = 0;
+  virtual void OnSearchCompleted() = 0;
+  virtual void ShowSample(size_t index, search::Sample const & sample, std::optional<m2::PointD> const & position,
+                          bool isUseless, bool hasEdits) = 0;
+
+  virtual void AddFoundResults(search::Results const & results) = 0;
+  virtual void ShowNonFoundResults(std::vector<search::Sample::Result> const & results,
+                                   std::vector<ResultsEdits::Entry> const & entries) = 0;
+
+  virtual void ShowMarks(Context const & context) = 0;
+
+  virtual void MoveViewportToResult(search::Result const & result) = 0;
+  virtual void MoveViewportToResult(search::Sample::Result const & result) = 0;
+  virtual void MoveViewportToRect(m2::RectD const & rect) = 0;
+
+  virtual void OnResultChanged(size_t sampleIndex, ResultType type, ResultsEdits::Update const & update) = 0;
+  virtual void SetResultsEdits(size_t index, ResultsEdits & foundResultsEdits, ResultsEdits & nonFoundResultsEdits) = 0;
+  virtual void OnSampleChanged(size_t sampleIndex, bool isUseless, bool hasEdits) = 0;
+  virtual void OnSamplesChanged(bool hasEdits) = 0;
+
+  virtual void ShowError(std::string const & msg) = 0;
+
+  virtual void Clear() = 0;
+
+protected:
+  Model * m_model = nullptr;
+};
+
+inline std::string DebugPrint(View::ResultType type)
+{
+  switch (type)
+  {
+  case View::ResultType::Found: return "Found";
+  case View::ResultType::NonFound: return "NonFound";
+  }
+  return "Unknown";
+}