Repo created

This commit is contained in:
Fr4nz D13trich 2025-11-22 13:58:55 +01:00
parent 4af19165ec
commit 68073add76
12458 changed files with 12350765 additions and 2 deletions

View file

@ -0,0 +1,14 @@
if (NOT PYBINDINGS_VERSION)
execute_process(
COMMAND grep -h -E "propVersionName|CURRENT_PROJECT_VERSION" xcode/common.xcconfig android/gradle.properties
COMMAND cut -d = -f 2
COMMAND awk "{print $1}"
COMMAND sort -Vr
COMMAND head -1
WORKING_DIRECTORY "${OMIM_ROOT}"
OUTPUT_VARIABLE PYBINDINGS_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE
)
endif()
configure_file(module_version.hpp.in module_version.hpp)

View file

View file

@ -0,0 +1,2 @@
// Bindings build for Python version @PYTHON_VERSION@
#define PYBINDINGS_VERSION "@PYBINDINGS_VERSION@"

29
libs/pyhelpers/pair.hpp Normal file
View file

@ -0,0 +1,29 @@
#pragma once
#include <utility>
#include <boost/python.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
namespace
{
using namespace boost::python;
// Converts a std::pair instance to a Python tuple.
template <typename T1, typename T2>
struct pair_to_tuple
{
static PyObject * convert(std::pair<T1, T2> const & p)
{
return incref(boost::python::make_tuple(p.first, p.second).ptr());
}
static PyTypeObject const * get_pytype() { return &PyTuple_Type; }
};
template <typename T1, typename T2>
struct pair_to_python_converter
{
pair_to_python_converter() { to_python_converter<std::pair<T1, T2>, pair_to_tuple<T1, T2>, true>(); }
};
} // namespace

566
libs/pyhelpers/setup.py Normal file
View file

@ -0,0 +1,566 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import inspect
import linecache
import multiprocessing
import os
import re
import sys
from contextlib import contextmanager
from distutils import log
from distutils.command.bdist import bdist
from distutils.command.build import build
from distutils.core import Command
from distutils.dir_util import mkpath
from distutils.file_util import copy_file
from distutils.spawn import spawn, find_executable
from distutils.sysconfig import (
get_config_var,
get_python_inc,
get_python_version,
)
from distutils.version import LooseVersion
from inspect import (
getsourcefile,
getfile,
getmodule,
ismodule,
isclass,
ismethod,
isfunction,
istraceback,
isframe,
iscode,
)
from setuptools import dist, setup
from setuptools.command.build_ext import build_ext
from setuptools.command.egg_info import egg_info, manifest_maker
from setuptools.command.install import install
from setuptools.extension import Extension
try:
# for pip >= 10
from pip._internal.req import parse_requirements
except ImportError:
# for pip <= 9.0.3
from pip.req import parse_requirements
# Monkey-patching to disable checking package names
dist.check_packages = lambda dist, attr, value: None
# Patch from https://github.com/ipython/ipython/issues/1456/
def findsource(object):
"""Return the entire source file and starting line number for an object.
The argument may be a module, class, method, function, traceback, frame,
or code object. The source code is returned as a list of all the lines
in the file and the line number indexes a line in that list. An IOError
is raised if the source code cannot be retrieved.
FIXED version with which we monkeypatch the stdlib to work around a bug."""
file = getsourcefile(object) or getfile(object)
# If the object is a frame, then trying to get the globals dict from its
# module won't work. Instead, the frame object itself has the globals
# dictionary.
globals_dict = None
if inspect.isframe(object):
# XXX: can this ever be false?
globals_dict = object.f_globals
else:
module = getmodule(object, file)
if module:
globals_dict = module.__dict__
lines = linecache.getlines(file, globals_dict)
if not lines:
raise IOError('could not get source code')
if ismodule(object):
return lines, 0
if isclass(object):
name = object.__name__
pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
# make some effort to find the best matching class definition:
# use the one with the least indentation, which is the one
# that's most probably not inside a function definition.
candidates = []
for i in range(len(lines)):
match = pat.match(lines[i])
if match:
# if it's at toplevel, it's already the best one
if lines[i][0] == 'c':
return lines, i
# else add whitespace to candidate list
candidates.append((match.group(1), i))
if candidates:
# this will sort by whitespace, and by line number,
# less whitespace first
candidates.sort()
return lines, candidates[0][1]
else:
raise IOError('could not find class definition')
if ismethod(object):
object = object.im_func
if isfunction(object):
object = object.func_code
if istraceback(object):
object = object.tb_frame
if isframe(object):
object = object.f_code
if iscode(object):
if not hasattr(object, 'co_firstlineno'):
raise IOError('could not find function definition')
pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
pmatch = pat.match
# fperez - fix: sometimes, co_firstlineno can give a number larger than
# the length of lines, which causes an error. Safeguard against that.
lnum = min(object.co_firstlineno, len(lines)) - 1
while lnum > 0:
if pmatch(lines[lnum]):
break
lnum -= 1
return lines, lnum
raise IOError('could not find code object')
# Monkeypatch inspect to apply our bugfix.
# This code only works with Python >= 2.5
inspect.findsource = findsource
PYHELPERS_DIR = os.path.abspath(os.path.dirname(__file__))
OMIM_ROOT = os.path.dirname(PYHELPERS_DIR)
BOOST_ROOT = os.path.join(OMIM_ROOT, '3party', 'boost')
BOOST_LIBRARYDIR = os.path.join(BOOST_ROOT, 'stage', 'lib')
ORIGINAL_CWD = os.getcwd()
@contextmanager
def chdir(target_dir):
saved_cwd = os.getcwd()
os.chdir(target_dir)
try:
yield
finally:
os.chdir(saved_cwd)
class BuildCommand(build, object):
user_options = build_ext.user_options + [
('omim-builddir=', None, 'Path to omim build directory'),
]
def initialize_options(self):
super(BuildCommand, self).initialize_options()
self.omim_builddir = os.path.join(OMIM_ROOT, 'build')
def finalize_options(self):
if os.path.isabs(self.omim_builddir):
self.omim_builddir = os.path.abspath(self.omim_builddir)
else:
self.omim_builddir = os.path.abspath(
os.path.join(ORIGINAL_CWD, self.omim_builddir)
)
self.build_base = os.path.relpath(
os.path.join(
self.omim_builddir,
'{}-builddir'.format(self.distribution.get_name()),
)
)
super(BuildCommand, self).finalize_options()
class BdistCommand(bdist, object):
user_options = build_ext.user_options + [
('omim-builddir=', None, 'Path to omim build directory'),
]
def initialize_options(self):
super(BdistCommand, self).initialize_options()
self.omim_builddir = None
def finalize_options(self):
self.set_undefined_options(
'build', ('omim_builddir', 'omim_builddir'),
)
self.dist_dir = os.path.join(
self.omim_builddir, '{}-dist'.format(self.distribution.get_name())
)
super(BdistCommand, self).finalize_options()
class ManifestMaker(manifest_maker, object):
def add_defaults(self):
super(ManifestMaker, self).add_defaults()
# Our README.md is for entire omim project, no point including it
# into python package, so remove it.
self.filelist.exclude('README.*')
class EggInfoCommand(egg_info, object):
user_options = build_ext.user_options + [
('omim-builddir=', None, 'Path to omim build directory'),
]
def initialize_options(self):
super(EggInfoCommand, self).initialize_options()
self.omim_builddir = None
def finalize_options(self):
self.set_undefined_options(
'build', ('omim_builddir', 'omim_builddir'),
)
self.egg_base = os.path.relpath(
os.path.join(
self.omim_builddir,
'{}-egg-info'.format(self.distribution.get_name()),
)
)
mkpath(self.egg_base)
super(EggInfoCommand, self).finalize_options()
def find_sources(self):
"""
Copied from setuptools.command.egg_info method to override
internal manifest_maker with our subclass
Generate SOURCES.txt manifest file
"""
manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt')
mm = ManifestMaker(self.distribution)
mm.manifest = manifest_filename
mm.run()
self.filelist = mm.filelist
class InstallCommand(install, object):
user_options = install.user_options + [
('omim-builddir=', None, 'Path to omim build directory'),
]
def initialize_options(self):
super(InstallCommand, self).initialize_options()
self.omim_builddir = None
def finalize_options(self):
super(InstallCommand, self).finalize_options()
self.set_undefined_options(
'build', ('omim_builddir', 'omim_builddir'),
)
class BuildBoostPythonCommand(Command, object):
user_options = [
(
'force',
'f',
'forcibly build boost_python library (ignore existence)',
),
('omim-builddir=', None, 'Path to omim build directory'),
]
boolean_options = ['force']
def initialize_options(self):
self.force = None
self.omim_builddir = None
def finalize_options(self):
self.set_undefined_options(
'build', ('force', 'force'), ('omim_builddir', 'omim_builddir'),
)
def get_boost_python_libname(self):
return 'boost_python{}{}'.format(
sys.version_info.major, sys.version_info.minor
)
def get_boost_config_path(self):
return os.path.join(
self.omim_builddir,
'python{}-config.jam'.format(get_python_version()),
)
def configure_omim(self):
with chdir(OMIM_ROOT):
spawn(['./configure.sh'])
def create_boost_config(self):
mkpath(self.omim_builddir)
with open(self.get_boost_config_path(), 'w') as f:
f.write(
'using python : {} : {} : {} ;\n'.format(
get_python_version(),
sys.executable,
get_python_inc(),
)
)
def get_boost_python_builddir(self):
return os.path.join(
self.omim_builddir,
'boost-build-python{}'.format(get_python_version()),
)
def clean(self):
with chdir(BOOST_ROOT):
spawn(
[
'./b2',
'--user-config={}'.format(self.get_boost_config_path()),
'--with-python',
'python={}'.format(get_python_version()),
'--build-dir={}'.format(self.get_boost_python_builddir()),
'--clean',
]
)
def build(self):
if os.path.exists(self.get_boost_python_builddir()):
self.clean()
with chdir(BOOST_ROOT):
spawn(
[
'./b2',
'--user-config={}'.format(self.get_boost_config_path()),
'--with-python',
'python={}'.format(get_python_version()),
'--build-dir={}'.format(self.get_boost_python_builddir()),
'cxxflags="-fPIC"',
]
)
def run(self):
lib_path = os.path.join(
BOOST_LIBRARYDIR, 'lib{}.a'.format(self.get_boost_python_libname())
)
if os.path.exists(lib_path) and not self.force:
log.info(
'Boost_python library `{}` for current '
'python version already present, skipping build'.format(
lib_path
)
)
return
self.configure_omim()
self.create_boost_config()
self.build()
class BuildOmimBindingCommand(build_ext, object):
user_options = build_ext.user_options + [
('omim-builddir=', None, 'Path to omim build directory'),
]
def initialize_options(self):
super(BuildOmimBindingCommand, self).initialize_options()
self.omim_builddir = None
def finalize_options(self):
super(BuildOmimBindingCommand, self).finalize_options()
self.set_undefined_options(
'build', ('omim_builddir', 'omim_builddir'),
)
def cmake_pybindings(self):
# On some linux systems the cmake we need is called cmake3
# So we must prefer it
for cmake in ['cmake3', 'cmake']:
if find_executable(cmake):
break
mkpath(self.omim_builddir)
with chdir(self.omim_builddir):
spawn(
[
cmake,
'-DSKIP_QT_GUI=1',
'-DPYBINDINGS=ON',
'-DPYBINDINGS_VERSION={}'.format(get_version()),
'-DPYTHON_EXECUTABLE={}'.format(sys.executable),
'-DPYTHON_INCLUDE_DIR={}'.format(get_python_inc()),
OMIM_ROOT,
]
)
def build_extension(self, ext):
with chdir(self.omim_builddir):
spawn(
[
'make',
'-j',
str(max(1, multiprocessing.cpu_count() // 2)),
ext.name,
]
)
mkpath(self.build_lib)
copy_file(
os.path.join(self.omim_builddir, '{}.so'.format(ext.name)),
self.get_ext_fullpath(ext.name),
)
def run(self):
self.run_command('build_boost_python')
self.cmake_pybindings()
super(BuildOmimBindingCommand, self).run()
VERSIONS_LOCATIONS = {
'xcode/common.xcconfig': 'CURRENT_PROJECT_VERSION',
'android/gradle.properties': 'propVersionName',
}
PYBINDINGS = {
'pygen': {
'path': 'generator/pygen',
'py_modules': ['example',],
'install_requires': ['omim-data-essential'],
'description': 'Binding for working with generation data',
},
'pykmlib': {
'path': 'kml/pykmlib',
'install_requires': ['omim-data-essential'],
'description': 'Binding for working with maps.me KML files',
},
'pymwm_diff': {
'path': 'generator/mwm_diff/pymwm_diff',
'description': 'Binding for working with mwm diffs',
},
'pysearch': {
'path': 'search/pysearch',
'description': 'Binding to access maps.me search engine',
'install_requires': ['omim-data-essential'],
},
'pytracking': {
'path': 'tracking/pytracking',
'description': 'Binding for working with user tracks',
},
'pytraffic': {
'path': 'traffic/pytraffic',
'description': (
'Binding for generation traffic data for maps.me application'
),
'install_requires': ['omim-data-essential'],
},
}
def get_version():
versions = []
for path, varname in VERSIONS_LOCATIONS.items():
with open(os.path.join(OMIM_ROOT, os.path.normpath(path))) as f:
for line in f:
match = re.search(
r'^\s*{}\s*=\s*(?P<version>.*)'.format(varname),
line.strip(),
)
if match:
versions.append(LooseVersion(match.group('version')))
break
code_version = max(versions)
env_version_addendum = os.environ.get('OMIM_SCM_VERSION', '')
return "{}{}".format(code_version, env_version_addendum)
def transform_omim_requirement(requirement, omim_package_version):
if requirement.startswith("omim"):
index = len(requirement) - 1
for i, ch in enumerate(requirement):
if not (ch.isalnum() or ch in "-_"):
index = i
break
requirement_without_version = requirement[0: index + 1]
requirement = "{}=={}".format(
requirement_without_version, omim_package_version
)
return requirement
def get_requirements(path="", omim_package_version=get_version()):
requirements = []
fpath = os.path.join(path, "requirements.txt")
for d in parse_requirements(fpath, session="s"):
try:
req_with_version = d.requirement
except AttributeError:
req_with_version = str(d.req)
requirements.append(
transform_omim_requirement(req_with_version, omim_package_version)
)
return requirements
def setup_omim_pybinding(
name,
version=None,
author='CoMaps',
author_email='info@comaps.app',
url='https://codeberg.org/comaps/comaps',
license='Apache-2.0',
supported_pythons=('2', '2.7', '3', '3.5', '3.6', '3.7', '3.8', '3.9'),
):
if version is None:
version = str(get_version())
install_requires = [
transform_omim_requirement(req, version)
for req in PYBINDINGS[name].get('install_requires', [])
]
setup(
name='omim-{}'.format(name),
version=version,
description=PYBINDINGS[name]['description'],
author=author,
author_email=author_email,
url=url,
license=license,
packages=PYBINDINGS[name].get('packages', []),
package_dir=PYBINDINGS[name].get('package_dir', {}),
py_modules=PYBINDINGS[name].get('py_modules', []),
package_data=PYBINDINGS[name].get('package_data', {}),
install_requires=install_requires,
ext_modules=[Extension(name, [])],
cmdclass={
'bdist': BdistCommand,
'build': BuildCommand,
'build_boost_python': BuildBoostPythonCommand,
'build_ext': BuildOmimBindingCommand,
'egg_info': EggInfoCommand,
'install': InstallCommand,
},
classifiers=[
# Trove classifiers
# Full list:
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python',
'Programming Language :: Python :: Implementation :: CPython',
]
+ [
'Programming Language :: Python :: {}'.format(supported_python)
for supported_python in supported_pythons
],
)
if __name__ == '__main__':
log.set_threshold(log.INFO)
for binding in PYBINDINGS.keys():
log.info('Run {}:'.format(binding))
path = os.path.join(
OMIM_ROOT, os.path.normpath(PYBINDINGS[binding]['path'])
)
with chdir(path):
setup_omim_pybinding(binding)

View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
PROJECT=$1
VERSION=$2
RELEASE=$3
if [ -z "$PROJECT" ]; then
echo "Specify project as first parameter"
exit 1
fi
if [ -z "$VERSION" ]; then
echo "Specify version as second parameter"
exit 1
fi
if [ -z "$RELEASE" ]; then
RELEASE=1
fi
basedir=$(dirname "$0")
PROJECT=$PROJECT VERSION=$VERSION RELEASE=$RELEASE rpmbuild -ba python35-mapsme-modules.spec
if [ $? -ne 0 ]; then
echo "Build failed!"
exit
fi
rsync -av $HOME/rpmbuild/RPMS/x86_64/python35-$PROJECT-$VERSION-$RELEASE.portal.el6.x86_64.rpm mapsme-team@pkg.corp.mail.ru::c6-mapsme-x64
echo "c6-mapsme-x64" | nc pkg.corp.mail.ru 12222 | grep -v '^* c'
echo
echo "$PROJECT packages version $VERSION-$RELEASE build done, ready to deploy"

View file

@ -0,0 +1,103 @@
%global __arch_install_post QA_SKIP_RPATHS=1 /usr/lib/rpm/check-rpaths /usr/lib/rpm/check-buildroot
%define py_version 3.5.1
%define py_release 1.portal
%define py_prefix /usr/local/python35
%global __python %{py_prefix}/bin/python3.5
%define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; import sys; sys.stdout.write(get_python_lib())")
%define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; import sys; sys.stdout.write(get_python_lib(1))")
%define pybindir %(%{__python} -c "import sys; print '%s/bin' % sys.exec_prefix")
#%{expand: %%define pyver %(%{__python} -c 'import sys;print(sys.version[0:5])')}
%define __os_install_post \
/usr/lib/rpm/redhat/brp-compress \
%{!?__debug_package:/usr/lib/rpm/redhat/brp-strip %{__strip}} \
/usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
/usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} \
/usr/lib/rpm/brp-python-bytecompile %{__python} \
/usr/lib/rpm/redhat/brp-python-hardlink \
%{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \
%{nil}
%define project %(echo $PROJECT)
%define version %(echo $VERSION)
%define release %(echo $RELEASE)
%define tag py-modules-%{version}
Name: python35-mapsme-modules
Version: %{version}
Release: %{release}.portal%{dist}
Summary: Python maps.me modules
License: Apache Public License 2.0
Vendor: Mail.Ru Group
Group: Development/Languages/Python
URL: https://codeberg.org/comaps/comaps
Source: omim-py-modules-%{version}.tar.gz
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
Prefix: %{_prefix}
BuildRequires: python35 >= %{py_version}
# BuildRequires: python35-setuptools
BuildRequires: cmake3
BuildRequires: boost_prefix-devel >= 1.54.0-3
BuildRequires: redhat-rpm-config
Requires: python35 >= %{py_version}
Requires: python35-%{project} = %{version}-%{release}
%description
Python maps.me modules metapackage. Installs all included modules
%package -n python35-%{project}
Summary: %{project} python module from maps.me localads
Group: Development/Languages/Python
%description -n python35-%{project}
Separate %{project} module for python35 from maps.me localads
%prep
%{__rm} -rf %{_builddir}/%{name}-%{version}
if [ -e %{S:0} ]; then
%{__tar} xzf %{S:0}
%{__chmod} -Rf a+rX,u+w,g-w,o-w %{_builddir}/%{name}-%{version}
else
git clone --depth=1 https://codeberg.org/comaps/comaps.git %{_builddir}/%{name}-%{version}/omim
pushd %{_builddir}/%{name}-%{version}/omim
git fetch origin tag %{tag} --depth=1
git checkout %{tag}
git submodule update --init --checkout
# pack source to save it in src rpm
popd
%{__tar} czf %{S:0} %{name}-%{version}
fi
%setup -D -T
%build
. /opt/rh/devtoolset-3/enable
cd omim
echo git@github.com:mapsme/omim-private.git | ./configure.sh
cd ..
%{__mkdir_p} build && cd build
# TODO(mgergio, yershov): Why should we stills specify PYTHON_LIBRARY and
# PYTHON_INCLUDE_DIR manually?
%{__cmake3} -DPYTHON_LIBRARY=/usr/local/python35/lib/libpython3.so -DPYTHON_INCLUDE_DIR=/usr/local/python35/include/python3.5m/ -DBOOST_INCLUDEDIR=/usr/local/boost_1.54.0/include/ -DPYBINDINGS=ON -DSKIP_QT_GUI=ON ../omim
%{__make} %{?_smp_mflags} %{project}
%install
%{__install} -m 755 -D %{_builddir}/%{name}-%{version}/build/%{project}.so %{buildroot}/%{python_sitelib}/%{project}.so
%clean
rm -rf %{buildroot}
%files -n python35-%{project}
%defattr(-,root,root)
%{python_sitelib}/%{project}.so
%changelog
* Thu Aug 03 2017 Sergey Yershov <yershov@corp.mail.ru> 0.2a-1
- Adopted to build any of available bingings
* Wed Apr 26 2017 Magidovich Sergey <s.magidovich@corp.mail.ru> - 0.1b-1
- Initiated build

View file

@ -0,0 +1,25 @@
#pragma once
#include <vector>
#include <boost/python.hpp>
#include <boost/python/stl_iterator.hpp>
namespace pyhelpers
{
template <typename T>
std::vector<T> PythonListToStdVector(boost::python::object const & iterable)
{
return std::vector<T>(boost::python::stl_input_iterator<T>(iterable), boost::python::stl_input_iterator<T>());
}
// For this to work one should define
// class_<std::vector<YourClass>>("YourClassList")
// .def(vector_indexing_suite<std::vector<YourClass>>());
template <typename T>
boost::python::list StdVectorToPythonList(std::vector<T> const & v)
{
boost::python::object get_iter = boost::python::iterator<std::vector<T>>();
return boost::python::list(get_iter(v));
}
} // namespace pyhelpers

View file

@ -0,0 +1,56 @@
#pragma once
#include <vector>
// These headers are necessary for cross-python compilation.
// Python3 does not have PyString_* methods. One should use PyBytes_* instead.
// bytesobject.h contains a mapping from PyBytes_* to PyString_*.
// See https://docs.python.org/2/howto/cporting.html for more.
#include "Python.h"
#include "bytesobject.h"
#include <boost/python.hpp>
#include <boost/python/suite/indexing/map_indexing_suite.hpp>
namespace
{
using namespace boost::python;
// Converts a vector<uint8_t> to Python2 str or Python3 bytes.
struct vector_uint8t_to_str
{
static PyObject * convert(std::vector<uint8_t> const & v)
{
auto bytes = PyBytes_FromStringAndSize(reinterpret_cast<char const *>(v.data()), v.size());
Py_INCREF(bytes);
return bytes;
}
};
// Converts a vector<uint8_t> from Python2 str or Python3 bytes.
struct vector_uint8t_from_python_str
{
vector_uint8t_from_python_str()
{
converter::registry::push_back(&convertible, &construct, type_id<std::vector<uint8_t>>());
}
static void * convertible(PyObject * obj_ptr)
{
if (!PyBytes_Check(obj_ptr))
return nullptr;
return obj_ptr;
}
static void construct(PyObject * obj_ptr, converter::rvalue_from_python_stage1_data * data)
{
char const * value = PyBytes_AsString(obj_ptr);
if (value == nullptr)
throw_error_already_set();
void * storage = ((converter::rvalue_from_python_storage<std::vector<uint8_t>> *)data)->storage.bytes;
new (storage) std::vector<uint8_t>(value, value + PyBytes_Size(obj_ptr));
data->convertible = storage;
}
};
} // namespace