Repo created

libs/mwm_diff/CMakeLists.txt

@@ -0,0 +1,17 @@
+project(mwm_diff)
+
+set(SRC
+  diff.cpp
+  diff.hpp
+)
+
+omim_add_library(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME}
+  bsdiff
+  coding
+)
+
+omim_add_tool_subdirectory(mwm_diff_tool)
+
+omim_add_test_subdirectory(mwm_diff_tests)

libs/mwm_diff/diff.cpp

@@ -0,0 +1,169 @@
+#include "mwm_diff/diff.hpp"
+
+#include "coding/buffered_file_writer.hpp"
+#include "coding/file_reader.hpp"
+#include "coding/file_writer.hpp"
+#include "coding/reader.hpp"
+#include "coding/write_to_sink.hpp"
+#include "coding/writer.hpp"
+#include "coding/zlib.hpp"
+
+#include "base/assert.hpp"
+#include "base/cancellable.hpp"
+#include "base/checked_cast.hpp"
+#include "base/logging.hpp"
+
+#include <cstdint>
+#include <iterator>
+#include <vector>
+
+#include "3party/bsdiff-courgette/bsdiff/bsdiff.h"
+
+namespace
+{
+enum Version
+{
+  // Format Version 0: bsdiff+gzip.
+  VERSION_V0 = 0,
+  VERSION_LATEST = VERSION_V0
+};
+
+bool MakeDiffVersion0(FileReader & oldReader, FileReader & newReader, FileWriter & diffFileWriter)
+{
+  std::vector<uint8_t> diffBuf;
+  MemWriter<std::vector<uint8_t>> diffMemWriter(diffBuf);
+
+  auto const status = bsdiff::CreateBinaryPatch(oldReader, newReader, diffMemWriter);
+
+  if (status != bsdiff::BSDiffStatus::OK)
+  {
+    LOG(LERROR, ("Could not create patch with bsdiff:", status));
+    return false;
+  }
+
+  using Deflate = coding::ZLib::Deflate;
+  Deflate deflate(Deflate::Format::ZLib, Deflate::Level::BestCompression);
+
+  std::vector<uint8_t> deflatedDiffBuf;
+  deflate(diffBuf.data(), diffBuf.size(), back_inserter(deflatedDiffBuf));
+
+  // A basic header that holds only version.
+  WriteToSink(diffFileWriter, static_cast<uint32_t>(VERSION_V0));
+  diffFileWriter.Write(deflatedDiffBuf.data(), deflatedDiffBuf.size());
+
+  return true;
+}
+
+generator::mwm_diff::DiffApplicationResult ApplyDiffVersion0(FileReader & oldReader, FileWriter & newWriter,
+                                                             ReaderSource<FileReader> & diffFileSource,
+                                                             base::Cancellable const & cancellable)
+{
+  using generator::mwm_diff::DiffApplicationResult;
+
+  std::vector<uint8_t> deflatedDiff(base::checked_cast<size_t>(diffFileSource.Size()));
+  diffFileSource.Read(deflatedDiff.data(), deflatedDiff.size());
+
+  using Inflate = coding::ZLib::Inflate;
+  Inflate inflate(Inflate::Format::ZLib);
+  std::vector<uint8_t> diffBuf;
+  inflate(deflatedDiff.data(), deflatedDiff.size(), back_inserter(diffBuf));
+
+  // Our bsdiff assumes that both the old mwm and the diff files are correct and
+  // does no checks when using its readers.
+  // Yet sometimes we observe corrupted files in the logs, and to avoid
+  // crashes from such files the exception-throwing version of MemReader is used here.
+  // |oldReader| is a FileReader so it throws exceptions too but we
+  // are more confident in the uncorrupted status of the old file because
+  // its checksum is compared to the one stored in the diff file.
+  MemReaderWithExceptions diffMemReader(diffBuf.data(), diffBuf.size());
+
+  auto const status = bsdiff::ApplyBinaryPatch(oldReader, newWriter, diffMemReader, cancellable);
+
+  if (status == bsdiff::BSDiffStatus::CANCELLED)
+  {
+    LOG(LDEBUG, ("Diff application has been cancelled"));
+    return DiffApplicationResult::Cancelled;
+  }
+
+  if (status == bsdiff::BSDiffStatus::OK)
+    return DiffApplicationResult::Ok;
+
+  LOG(LERROR, ("Could not apply patch with bsdiff:", status));
+  return DiffApplicationResult::Failed;
+}
+}  // namespace
+
+namespace generator
+{
+namespace mwm_diff
+{
+bool MakeDiff(std::string const & oldMwmPath, std::string const & newMwmPath, std::string const & diffPath)
+{
+  try
+  {
+    FileReader oldReader(oldMwmPath);
+    FileReader newReader(newMwmPath);
+    FileWriter diffFileWriter(diffPath);
+
+    switch (VERSION_LATEST)
+    {
+    case VERSION_V0: return MakeDiffVersion0(oldReader, newReader, diffFileWriter);
+    default: LOG(LERROR, ("Making mwm diffs with diff format version", VERSION_LATEST, "is not implemented"));
+    }
+  }
+  catch (Reader::Exception const & e)
+  {
+    LOG(LERROR, ("Could not open file when creating a patch:", e.Msg()));
+    return false;
+  }
+  catch (Writer::Exception const & e)
+  {
+    LOG(LERROR, ("Could not open file when creating a patch:", e.Msg()));
+    return false;
+  }
+
+  return false;
+}
+
+DiffApplicationResult ApplyDiff(std::string const & oldMwmPath, std::string const & newMwmPath,
+                                std::string const & diffPath, base::Cancellable const & cancellable)
+{
+  try
+  {
+    FileReader oldReader(oldMwmPath);
+    BufferedFileWriter newWriter(newMwmPath);
+    FileReader diffFileReader(diffPath);
+
+    ReaderSource<FileReader> diffFileSource(diffFileReader);
+    auto const version = ReadPrimitiveFromSource<uint32_t>(diffFileSource);
+
+    switch (version)
+    {
+    case VERSION_V0: return ApplyDiffVersion0(oldReader, newWriter, diffFileSource, cancellable);
+    default: LOG(LERROR, ("Unknown version format of mwm diff:", version)); return DiffApplicationResult::Failed;
+    }
+  }
+  catch (Reader::Exception const & e)
+  {
+    LOG(LERROR, ("Could not open file for reading when applying a patch:", e.Msg()));
+  }
+  catch (Writer::Exception const & e)
+  {
+    LOG(LERROR, ("Could not open file for writing when applying a patch:", e.Msg()));
+  }
+
+  return cancellable.IsCancelled() ? DiffApplicationResult::Cancelled : DiffApplicationResult::Failed;
+}
+
+std::string DebugPrint(DiffApplicationResult const & result)
+{
+  switch (result)
+  {
+  case DiffApplicationResult::Ok: return "Ok";
+  case DiffApplicationResult::Failed: return "Failed";
+  case DiffApplicationResult::Cancelled: return "Cancelled";
+  }
+  UNREACHABLE();
+}
+}  // namespace mwm_diff
+}  // namespace generator

libs/mwm_diff/diff.hpp

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <string>
+
+namespace base
+{
+class Cancellable;
+}
+
+namespace generator
+{
+namespace mwm_diff
+{
+enum class DiffApplicationResult
+{
+  Ok,
+  Failed,
+  Cancelled,
+};
+
+// Makes a diff that, when applied to the mwm at |oldMwmPath|, will
+// result in the mwm at |newMwmPath|. The diff is stored at |diffPath|.
+// It is assumed that the files at |oldMwmPath| and |newMwmPath| are valid mwms.
+// Returns true on success and false on failure.
+bool MakeDiff(std::string const & oldMwmPath, std::string const & newMwmPath, std::string const & diffPath);
+
+// Applies the diff at |diffPath| to the mwm at |oldMwmPath|. The resulting
+// mwm is stored at |newMwmPath|.
+// It is assumed that the file at |oldMwmPath| is a valid mwm and the file
+// at |diffPath| is a valid mwmdiff.
+// The application process can be stopped via |cancellable| in which case
+// it is up to the caller to clean the partially written file at |diffPath|.
+DiffApplicationResult ApplyDiff(std::string const & oldMwmPath, std::string const & newMwmPath,
+                                std::string const & diffPath, base::Cancellable const & cancellable);
+
+std::string DebugPrint(DiffApplicationResult const & result);
+}  // namespace mwm_diff
+}  // namespace generator

libs/mwm_diff/mwm_diff_tests/CMakeLists.txt

@@ -0,0 +1,9 @@
+project(mwm_diff_tests)
+
+set(SRC
+  diff_tests.cpp
+)
+
+omim_add_test(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME} mwm_diff)

libs/mwm_diff/mwm_diff_tests/diff_tests.cpp

@@ -0,0 +1,87 @@
+#include "testing/testing.hpp"
+
+#include "mwm_diff/diff.hpp"
+
+#include "platform/platform.hpp"
+
+#include "coding/file_writer.hpp"
+#include "coding/internal/file_data.hpp"
+
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+#include "base/scope_guard.hpp"
+
+#include <vector>
+
+namespace generator::diff_tests
+{
+using namespace mwm_diff;
+using std::string, std::vector;
+
+UNIT_TEST(IncrementalUpdates_Smoke)
+{
+  base::ScopedLogAbortLevelChanger ignoreLogError(base::LogLevel::LCRITICAL);
+
+  string const oldMwmPath = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass.mwm");
+  string const newMwmPath1 = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass-new1.mwm");
+  string const newMwmPath2 = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass-new2.mwm");
+  string const diffPath = base::JoinPath(GetPlatform().WritableDir(), "minsk-pass.mwmdiff");
+
+  SCOPE_GUARD(cleanup, [&]
+  {
+    FileWriter::DeleteFileX(newMwmPath1);
+    FileWriter::DeleteFileX(newMwmPath2);
+    FileWriter::DeleteFileX(diffPath);
+  });
+
+  {
+    // Create an empty file.
+    FileWriter writer(newMwmPath1);
+  }
+
+  base::Cancellable cancellable;
+  TEST(MakeDiff(oldMwmPath, newMwmPath1, diffPath), ());
+  TEST_EQUAL(ApplyDiff(oldMwmPath, newMwmPath2, diffPath, cancellable), DiffApplicationResult::Ok, ());
+
+  {
+    // Alter the old mwm slightly.
+    vector<uint8_t> oldMwmContents = base::ReadFile(oldMwmPath);
+    size_t const sz = oldMwmContents.size();
+    for (size_t i = 3 * sz / 10; i < 4 * sz / 10; i++)
+      oldMwmContents[i] += static_cast<uint8_t>(i);
+
+    FileWriter writer(newMwmPath1);
+    writer.Write(oldMwmContents.data(), oldMwmContents.size());
+  }
+
+  TEST(MakeDiff(oldMwmPath, newMwmPath1, diffPath), ());
+  TEST_EQUAL(ApplyDiff(oldMwmPath, newMwmPath2, diffPath, cancellable), DiffApplicationResult::Ok, ());
+
+  TEST(base::IsEqualFiles(newMwmPath1, newMwmPath2), ());
+
+  cancellable.Cancel();
+  TEST_EQUAL(ApplyDiff(oldMwmPath, newMwmPath2, diffPath, cancellable), DiffApplicationResult::Cancelled, ());
+  cancellable.Reset();
+
+  {
+    // Corrupt the diff file contents.
+    vector<uint8_t> diffContents = base::ReadFile(diffPath);
+
+    // Leave the version bits intact.
+    for (size_t i = 4; i < diffContents.size(); i += 2)
+      diffContents[i] ^= 255;
+
+    FileWriter writer(diffPath);
+    writer.Write(diffContents.data(), diffContents.size());
+  }
+
+  TEST_EQUAL(ApplyDiff(oldMwmPath, newMwmPath2, diffPath, cancellable), DiffApplicationResult::Failed, ());
+
+  {
+    // Reset the diff file contents.
+    FileWriter writer(diffPath);
+  }
+
+  TEST_EQUAL(ApplyDiff(oldMwmPath, newMwmPath2, diffPath, cancellable), DiffApplicationResult::Failed, ());
+}
+}  // namespace generator::diff_tests

libs/mwm_diff/mwm_diff_tool/CMakeLists.txt

@@ -0,0 +1,7 @@
+project(mwm_diff_tool)
+
+set(SRC mwm_diff_tool.cpp)
+
+omim_add_executable(${PROJECT_NAME} ${SRC})
+
+target_link_libraries(${PROJECT_NAME} mwm_diff)

libs/mwm_diff/mwm_diff_tool/mwm_diff_tool.cpp

@@ -0,0 +1,46 @@
+#include "mwm_diff/diff.hpp"
+
+#include "base/cancellable.hpp"
+
+#include <cstring>
+#include <iostream>
+
+int main(int argc, char ** argv)
+{
+  auto const ShowUsage = [argv]()
+  {
+    std::cout << "Usage: " << argv[0]
+              << " make|apply olderMWMPath newerMWMPath diffPath\n"
+                 "make\n"
+                 "  Creates the diff between newer and older MWMs at `diffPath`\n"
+                 "apply\n"
+                 "  Applies the diff at `diffPath` to the mwm at `olderMWMPath` and stores result at `newerMWMPath`.\n"
+                 "WARNING: THERE IS NO MWM VALIDITY CHECK!\n";
+  };
+
+  if (argc < 5)
+  {
+    ShowUsage();
+    return -1;
+  }
+
+  auto const IsEqualUsage = [argv](char const * s) { return 0 == std::strcmp(argv[1], s); };
+  char const *olderMWMPath{argv[2]}, *newerMWMPath{argv[3]}, *diffPath{argv[4]};
+
+  if (IsEqualUsage("make"))
+  {
+    if (generator::mwm_diff::MakeDiff(olderMWMPath, newerMWMPath, diffPath))
+      return 0;
+  }
+  else if (IsEqualUsage("apply"))
+  {
+    base::Cancellable cancellable;
+    auto const res = generator::mwm_diff::ApplyDiff(olderMWMPath, newerMWMPath, diffPath, cancellable);
+    if (res == generator::mwm_diff::DiffApplicationResult::Ok)
+      return 0;
+  }
+  else
+    ShowUsage();
+
+  return -1;  // Failed, Cancelled.
+}