Repo created

libs/coding/internal/file64_api.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "std/target_os.hpp"
+
+#if defined(OMIM_OS_WINDOWS_NATIVE)
+#define fseek64 _fseeki64
+#define ftell64 _ftelli64
+
+#elif defined(OMIM_OS_WINDOWS_MINGW)
+#define fseek64 fseeko64
+#define ftell64 ftello64
+
+#else
+// POSIX standart.
+#include <sys/types.h>
+
+// TODO: Always assert for 8 bytes after increasing min Android API to 24+.
+// See more details here: https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md
+#if defined(OMIM_OS_ANDROID) && (defined(__arm__) || defined(__i386__))
+static_assert(sizeof(off_t) == 4, "32-bit Android NDK < API 24 has only 32-bit file operations support");
+#else
+static_assert(sizeof(off_t) == 8, "FileReader and FileWriter require 64-bit file operations");
+#endif
+#define fseek64 fseeko
+#define ftell64 ftello
+
+#endif
+
+#include <cstdio>

libs/coding/internal/file_data.cpp

@@ -0,0 +1,339 @@
+#include "coding/internal/file_data.hpp"
+
+#include "coding/constants.hpp"
+#include "coding/internal/file64_api.hpp"
+#include "coding/reader.hpp"  // For Reader exceptions.
+#include "coding/writer.hpp"  // For Writer exceptions.
+
+#include "base/exception.hpp"
+#include "base/logging.hpp"
+#include "base/string_utils.hpp"
+
+#include "std/target_os.hpp"
+
+#include <algorithm>
+#include <cerrno>
+#include <cstring>
+#include <exception>
+#include <fstream>
+#include <vector>
+
+#ifdef OMIM_OS_WINDOWS
+#include <io.h>
+#else
+#include <unistd.h>  // ftruncate
+#endif
+
+namespace base
+{
+using namespace std;
+
+std::ostream & operator<<(std::ostream & stream, FileData::Op op)
+{
+  switch (op)
+  {
+  case FileData::Op::READ: stream << "READ"; break;
+  case FileData::Op::WRITE_TRUNCATE: stream << "WRITE_TRUNCATE"; break;
+  case FileData::Op::WRITE_EXISTING: stream << "WRITE_EXISTING"; break;
+  case FileData::Op::APPEND: stream << "APPEND"; break;
+  }
+  return stream;
+}
+
+FileData::FileData(string const & fileName, Op op) : m_FileName(fileName), m_Op(op)
+{
+  char const * const modes[] = {"rb", "wb", "r+b", "ab"};
+
+  m_File = fopen(fileName.c_str(), modes[static_cast<int>(op)]);
+  if (m_File)
+  {
+#if defined(_MSC_VER)
+    // Move file pointer to the end of the file to make it consistent with other platforms
+    if (op == Op::APPEND)
+      fseek64(m_File, 0, SEEK_END);
+#endif
+    return;
+  }
+
+  if (op == Op::WRITE_EXISTING)
+  {
+    // Special case, since "r+b" fails if file doesn't exist.
+    m_File = fopen(fileName.c_str(), "wb");
+    if (m_File)
+      return;
+  }
+
+  // if we're here - something bad is happened
+  if (m_Op != Op::READ)
+    MYTHROW(Writer::OpenException, (GetErrorProlog()));
+  else
+    MYTHROW(Reader::OpenException, (GetErrorProlog()));
+}
+
+FileData::~FileData()
+{
+  if (m_File)
+  {
+    if (fclose(m_File))
+      LOG(LWARNING, ("Error closing file", GetErrorProlog()));
+  }
+}
+
+string FileData::GetErrorProlog() const
+{
+  std::ostringstream stream;
+  stream << m_FileName << "; " << m_Op << "; " << strerror(errno);
+  return stream.str();
+}
+
+static int64_t constexpr INVALID_POS = -1;
+
+uint64_t FileData::Size() const
+{
+  int64_t const pos = ftell64(m_File);
+  if (pos == INVALID_POS)
+    MYTHROW(Reader::SizeException, (GetErrorProlog(), pos));
+
+  if (fseek64(m_File, 0, SEEK_END))
+    MYTHROW(Reader::SizeException, (GetErrorProlog()));
+
+  int64_t const size = ftell64(m_File);
+  if (size == INVALID_POS)
+    MYTHROW(Reader::SizeException, (GetErrorProlog(), size));
+
+  if (fseek64(m_File, static_cast<off_t>(pos), SEEK_SET))
+    MYTHROW(Reader::SizeException, (GetErrorProlog(), pos));
+
+  ASSERT_GREATER_OR_EQUAL(size, 0, ());
+  return static_cast<uint64_t>(size);
+}
+
+void FileData::Read(uint64_t pos, void * p, size_t size)
+{
+  if (fseek64(m_File, static_cast<off_t>(pos), SEEK_SET))
+    MYTHROW(Reader::ReadException, (GetErrorProlog(), pos));
+
+  size_t const bytesRead = fread(p, 1, size, m_File);
+  if (bytesRead != size || ferror(m_File))
+    MYTHROW(Reader::ReadException, (GetErrorProlog(), bytesRead, pos, size));
+}
+
+uint64_t FileData::Pos() const
+{
+  int64_t const pos = ftell64(m_File);
+  if (pos == INVALID_POS)
+    MYTHROW(Writer::PosException, (GetErrorProlog(), pos));
+
+  ASSERT_GREATER_OR_EQUAL(pos, 0, ());
+  return static_cast<uint64_t>(pos);
+}
+
+void FileData::Seek(uint64_t pos)
+{
+  ASSERT_NOT_EQUAL(m_Op, Op::APPEND, (m_FileName, m_Op, pos));
+  if (fseek64(m_File, static_cast<off_t>(pos), SEEK_SET))
+    MYTHROW(Writer::SeekException, (GetErrorProlog(), pos));
+}
+
+void FileData::Write(void const * p, size_t size)
+{
+  size_t const bytesWritten = fwrite(p, 1, size, m_File);
+  if (bytesWritten != size || ferror(m_File))
+    MYTHROW(Writer::WriteException, (GetErrorProlog(), bytesWritten, size));
+}
+
+void FileData::Flush()
+{
+  if (fflush(m_File))
+    MYTHROW(Writer::WriteException, (GetErrorProlog()));
+}
+
+void FileData::Truncate(uint64_t sz)
+{
+#ifdef OMIM_OS_WINDOWS
+  int const res = _chsize(fileno(m_File), sz);
+#else
+  int const res = ftruncate(fileno(m_File), static_cast<off_t>(sz));
+#endif
+
+  if (res)
+    MYTHROW(Writer::WriteException, (GetErrorProlog(), sz));
+}
+
+bool GetFileSize(string const & fName, uint64_t & sz)
+{
+  try
+  {
+    typedef FileData fdata_t;
+    fdata_t f(fName, fdata_t::Op::READ);
+    sz = f.Size();
+    return true;
+  }
+  catch (RootException const &)
+  {
+    // supress all exceptions here
+    return false;
+  }
+}
+
+namespace
+{
+
+bool CheckFileOperationResult(int res, string const & fName)
+{
+  if (!res)
+    return true;
+
+  LOG(LWARNING, ("File operation error for file:", fName, "-", strerror(errno)));
+
+  // additional check if file really was removed correctly
+  uint64_t dummy;
+  if (GetFileSize(fName, dummy))
+    LOG(LERROR, ("File exists but can't be deleted. Sharing violation?", fName));
+
+  return false;
+}
+
+bool IsEOF(ifstream & fs)
+{
+  return fs.peek() == ifstream::traits_type::eof();
+}
+
+}  // namespace
+
+bool DeleteFileX(string const & fName)
+{
+  int res = remove(fName.c_str());
+  return CheckFileOperationResult(res, fName);
+}
+
+bool RenameFileX(string const & fOld, string const & fNew)
+{
+  int res = rename(fOld.c_str(), fNew.c_str());
+  return CheckFileOperationResult(res, fOld);
+}
+
+bool MoveFileX(string const & fOld, string const & fNew)
+{
+  // Try to rename the file first.
+  int res = rename(fOld.c_str(), fNew.c_str());
+  if (res == 0)
+    return true;
+
+  // Otherwise perform the full move.
+  if (!CopyFileX(fOld, fNew))
+  {
+    (void)DeleteFileX(fNew);
+    return false;
+  }
+  (void)DeleteFileX(fOld);
+  return true;
+}
+
+bool WriteToTempAndRenameToFile(string const & dest, function<bool(string const &)> const & write, string const & tmp)
+{
+  string const tmpFileName = tmp.empty() ? dest + ".tmp" + strings::to_string(this_thread::get_id()) : tmp;
+  if (!write(tmpFileName))
+  {
+    LOG(LERROR, ("Can't write to", tmpFileName));
+    DeleteFileX(tmpFileName);
+    return false;
+  }
+  if (!RenameFileX(tmpFileName, dest))
+  {
+    LOG(LERROR, ("Can't rename file", tmpFileName, "to", dest));
+    DeleteFileX(tmpFileName);
+    return false;
+  }
+  return true;
+}
+
+void AppendFileToFile(string const & fromFilename, string const & toFilename)
+{
+  ifstream from;
+  from.exceptions(fstream::failbit | fstream::badbit);
+  from.open(fromFilename, ios::binary);
+
+  ofstream to;
+  to.exceptions(fstream::badbit);
+  to.open(toFilename, ios::binary | ios::app);
+
+  auto * buffer = from.rdbuf();
+  if (!IsEOF(from))
+    to << buffer;
+}
+
+bool CopyFileX(string const & fOld, string const & fNew)
+{
+  ifstream ifs;
+  ofstream ofs;
+  ifs.exceptions(ifstream::failbit | ifstream::badbit);
+  ofs.exceptions(ifstream::failbit | ifstream::badbit);
+
+  try
+  {
+    ifs.open(fOld.c_str());
+    ofs.open(fNew.c_str());
+
+    // If source file is empty - make empty dest file without any errors.
+    if (IsEOF(ifs))
+      return true;
+
+    ofs << ifs.rdbuf();
+    ofs.flush();
+    return true;
+  }
+  catch (system_error const &)
+  {
+    LOG(LWARNING, ("Failed to copy file from", fOld, "to", fNew, ":", strerror(errno)));
+  }
+  catch (exception const &)
+  {
+    LOG(LERROR, ("Unknown error when coping files:", fOld, "to", fNew, strerror(errno)));
+  }
+
+  // Don't care about possible error here ..
+  (void)DeleteFileX(fNew);
+  return false;
+}
+
+bool IsEqualFiles(string const & firstFile, string const & secondFile)
+{
+  FileData first(firstFile, FileData::Op::READ);
+  FileData second(secondFile, FileData::Op::READ);
+  if (first.Size() != second.Size())
+    return false;
+
+  size_t constexpr bufSize = READ_FILE_BUFFER_SIZE;
+  vector<char> buf1, buf2;
+  buf1.resize(bufSize);
+  buf2.resize(bufSize);
+  size_t const fileSize = static_cast<size_t>(first.Size());
+  size_t currSize = 0;
+
+  while (currSize < fileSize)
+  {
+    size_t const toRead = min(bufSize, fileSize - currSize);
+
+    first.Read(currSize, &buf1[0], toRead);
+    second.Read(currSize, &buf2[0], toRead);
+
+    if (buf1 != buf2)
+      return false;
+
+    currSize += toRead;
+  }
+
+  return true;
+}
+
+std::vector<uint8_t> ReadFile(std::string const & filePath)
+{
+  FileData file(filePath, FileData::Op::READ);
+  uint64_t const sz = file.Size();
+  std::vector<uint8_t> contents(sz);
+  file.Read(0, contents.data(), sz);
+  return contents;
+}
+
+}  // namespace base

libs/coding/internal/file_data.hpp

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "base/macros.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <string>
+
+namespace base
+{
+class FileData
+{
+public:
+  /// @note Do not change order (@see FileData::FileData).
+  enum class Op
+  {
+    READ = 0,
+    WRITE_TRUNCATE,
+    WRITE_EXISTING,
+    APPEND
+  };
+
+  FileData(std::string const & fileName, Op op);
+  ~FileData();
+
+  uint64_t Size() const;
+  uint64_t Pos() const;
+
+  void Seek(uint64_t pos);
+
+  void Read(uint64_t pos, void * p, size_t size);
+  void Write(void const * p, size_t size);
+
+  void Flush();
+  void Truncate(uint64_t sz);
+
+  std::string const & GetName() const { return m_FileName; }
+
+private:
+  FILE * m_File;
+  std::string const m_FileName;
+  Op const m_Op;
+
+  std::string GetErrorProlog() const;
+
+  DISALLOW_COPY(FileData);
+};
+
+bool GetFileSize(std::string const & fName, uint64_t & sz);
+bool DeleteFileX(std::string const & fName);
+bool RenameFileX(std::string const & fOld, std::string const & fNew);
+
+/// Write to temp file and rename it to dest. Delete temp on failure.
+/// @param write function that writes to file with a given name, returns true on success.
+bool WriteToTempAndRenameToFile(std::string const & dest, std::function<bool(std::string const &)> const & write,
+                                std::string const & tmp = "");
+
+void AppendFileToFile(std::string const & fromFilename, std::string const & toFilename);
+
+/// @return false if copy fails. DOES NOT THROWS exceptions
+bool CopyFileX(std::string const & fOld, std::string const & fNew);
+/// @return false if moving fails. DOES NOT THROW exceptions
+bool MoveFileX(std::string const & fOld, std::string const & fNew);
+bool IsEqualFiles(std::string const & firstFile, std::string const & secondFile);
+
+std::vector<uint8_t> ReadFile(std::string const & filePath);
+
+}  // namespace base

libs/coding/internal/xmlparser.hpp

@@ -0,0 +1,162 @@
+#pragma once
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-parameter"
+#endif
+
+#include <memory>
+#include <sstream>
+#include <string>
+
+#ifndef XML_STATIC
+#define XML_STATIC
+#endif
+#include <expat.h>
+
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+
+/// Dispatcher's methods Push, Pop and AddAttr can conveniently take different parameters:
+/// 1. char const * (no any overhead, is called by the Expat)
+/// 2. std::string or std::string const & (temporary std::string is created from char const *)
+/// 3. std::string_view (created from char const *)
+///
+/// CharData accepts std::string const & or std::string & to modify the data before consumption.
+template <typename DispatcherT>
+class XmlParser
+{
+public:
+  explicit XmlParser(DispatcherT & dispatcher, bool enableCharHandler = false)
+    : m_depth(0)
+    , m_restrictDepth(static_cast<size_t>(-1))
+    , m_dispatcher(dispatcher)
+    , m_enableCharHandler(enableCharHandler)
+    , m_parser(std::unique_ptr<XML_ParserStruct, decltype(&XML_ParserFree)>(XML_ParserCreate(nullptr /* encoding */),
+                                                                            &XML_ParserFree))
+  {
+    CHECK(m_parser, ());
+    OnPostCreate();
+  }
+
+  static void StartElementHandler(void * userData, XML_Char const * name, XML_Char const ** attrs)
+  {
+    CHECK(userData, (name));
+    auto * xmlParser = static_cast<XmlParser *>(userData);
+    xmlParser->OnStartElement(name, attrs);
+  }
+
+  static void EndElementHandler(void * userData, XML_Char const * name)
+  {
+    CHECK(userData, (name));
+    auto * xmlParser = static_cast<XmlParser *>(userData);
+    xmlParser->OnEndElement(name);
+  }
+
+  static void CharacterDataHandler(void * userData, XML_Char const * data, int length)
+  {
+    CHECK(userData, (data));
+    auto * xmlParser = static_cast<XmlParser *>(userData);
+    xmlParser->OnCharacterData(data, length);
+  }
+
+  void * GetBuffer(int len)
+  {
+    CHECK(m_parser, ());
+    return XML_GetBuffer(m_parser.get(), len);
+  }
+
+  XML_Status ParseBuffer(int len, int isFinal)
+  {
+    CHECK(m_parser, ());
+    return XML_ParseBuffer(m_parser.get(), len, isFinal);
+  }
+
+  void OnPostCreate()
+  {
+    CHECK(m_parser, ());
+    // Enable all the event routines we want
+    XML_SetStartElementHandler(m_parser.get(), StartElementHandler);
+    XML_SetEndElementHandler(m_parser.get(), EndElementHandler);
+    if (m_enableCharHandler)
+      XML_SetCharacterDataHandler(m_parser.get(), CharacterDataHandler);
+
+    XML_SetUserData(m_parser.get(), static_cast<void *>(this));
+  }
+
+  using StringPtrT = XML_Char const *;
+
+  // Start element handler
+  void OnStartElement(StringPtrT name, StringPtrT * attrs)
+  {
+    CheckCharData();
+
+    ++m_depth;
+    if (m_depth >= m_restrictDepth)
+      return;
+
+    if (!m_dispatcher.Push(name))
+    {
+      m_restrictDepth = m_depth;
+      return;
+    }
+
+    for (size_t i = 0; attrs[2 * i]; ++i)
+      m_dispatcher.AddAttr(attrs[2 * i], attrs[2 * i + 1]);
+  }
+
+  // End element handler
+  void OnEndElement(StringPtrT name)
+  {
+    CheckCharData();
+
+    --m_depth;
+    if (m_depth >= m_restrictDepth)
+      return;
+
+    if (m_restrictDepth != size_t(-1))
+      m_restrictDepth = static_cast<size_t>(-1);
+    else
+      m_dispatcher.Pop(name);
+  }
+
+  void OnCharacterData(XML_Char const * data, int length)
+  {
+    // Accumulate character data - it can be passed by parts
+    // (when reading from fixed length buffer).
+    m_charData.append(data, length);
+  }
+
+  std::string GetErrorMessage()
+  {
+    if (XML_GetErrorCode(m_parser.get()) == XML_ERROR_NONE)
+      return {};
+
+    std::stringstream s;
+    s << "XML parse error at line " << XML_GetCurrentLineNumber(m_parser.get()) << " and byte "
+      << XML_GetCurrentByteIndex(m_parser.get());
+    return s.str();
+  }
+
+private:
+  size_t m_depth;
+  size_t m_restrictDepth;
+  DispatcherT & m_dispatcher;
+
+  std::string m_charData;
+  bool m_enableCharHandler;
+  std::unique_ptr<XML_ParserStruct, decltype(&XML_ParserFree)> m_parser;
+
+  void CheckCharData()
+  {
+    if (m_enableCharHandler && !m_charData.empty())
+    {
+      m_dispatcher.CharData(m_charData);
+      m_charData.clear();
+    }
+  }
+};