Repo created

tools/unix/maps/Dockerfile

@@ -0,0 +1,68 @@
+# See run-docker.sh for cloning, building, and running the maps generator Docker routine
+FROM debian:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+RUN apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    cargo \
+    clang \
+    cmake \
+    curl \
+    git \
+    jq \
+    libc++-dev \
+    libfreetype-dev \
+    libgeos-dev \
+    libgl1-mesa-dev \
+    libglvnd-dev \
+    libharfbuzz-dev \
+    libicu-dev \
+    libqt6positioning6 \
+    libqt6positioning6-plugins \
+    libqt6svg6-dev \
+    libsqlite3-dev \
+    libxcursor-dev \
+    libxi-dev \
+    libxinerama-dev \
+    libxrandr-dev \
+    ninja-build \
+    nodejs \
+    openssh-client \
+    osmctools \
+    osmium-tool \
+    pyosmium \
+    python3 \
+    python3-pip \
+    python3-venv \
+    python-is-python3 \
+    python3-dev \
+    qt6-base-dev \
+    qt6-positioning-dev \
+    rclone \
+    rustc \
+    sshpass \
+    vim-tiny \
+    wget \
+    wget2 \
+    zlib1g-dev \
+    && rm -rf /var/cache/apt/* /var/lib/apt/lists/*;
+
+# Allow pip to install system-wide in the container
+RUN pip3 install "protobuf<4" --break-system-packages
+
+# Mount a volume to store the large input, output, and temp files here
+RUN mkdir /mnt/4tbexternal
+# And a volume to store >10gb files for the planet output here
+RUN mkdir /home/planet
+
+WORKDIR ~
+
+# The actual map generation process will run something like this,
+# but ~/comaps isn't mounted nor provided in the docker image.
+#CMD ~/comaps/tools/unix/maps/docker_maps_generator.sh
+
+CMD /bin/bash

tools/unix/maps/docker_maps_generator.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "<$(date +%T)> Starting..."
+
+# Prepare paths
+# Most other paths in /mnt/4tbexternal or /home/planet are already created by Dockerfile or CI/CD.
+#
+mkdir -p /root/.config/CoMaps # Odd mkdir permission errors in generator_tool in Docker without these
+chmod -R 777 /root/.config
+mkdir -p /home/planet/postcodes/gb-postcode-data/
+mkdir -p /home/planet/postcodes/us-postcodes/
+mkdir -p /home/planet/SRTM-patched-europe/
+mkdir -p /home/planet/subway
+
+echo "<$(date +%T)> Running ./configure.sh ..."
+cd ~/comaps
+export SKIP_MAP_DOWNLOAD=1 SKIP_GENERATE_SYMBOLS=1
+./configure.sh
+
+echo "<$(date +%T)> Compiling tools..."
+cd ~/comaps
+./tools/unix/build_omim.sh -p ~ -R generator_tool
+./tools/unix/build_omim.sh -p ~ -R world_roads_builder_tool
+./tools/unix/build_omim.sh -p ~ -R mwm_diff_tool
+cd tools/python/maps_generator
+python3 -m venv /tmp/venv
+/tmp/venv/bin/pip3 install -r requirements_dev.txt
+
+echo "<$(date +%T)> Copying map generator INI..."
+cp var/etc/map_generator.ini.prod var/etc/map_generator.ini
+
+
+cd ~/comaps/tools/python
+if [ $MWMCONTINUE -gt 0 ]; then
+
+echo "<$(date +%T)> Continuing from preexisting generator run..."
+/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs" --continue
+
+else
+
+if [[ -n $MWMCOUNTRIES ]]; then
+
+echo "<$(date +%T)> Generating only specific maps [$MWMCOUNTRIES]..."
+/tmp/venv/bin/python -m maps_generator --countries=$MWMCOUNTRIES --skip="MwmDiffs"
+
+else
+
+echo "<$(date +%T)> Generating maps..."
+/tmp/venv/bin/python -m maps_generator --skip="MwmDiffs"
+
+fi
+fi
+
+echo "<$(date +%T)> DONE"

tools/unix/maps/generate_subways.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -e -u -o pipefail
+
+# Generate subways.transit.json file consumed by the maps generator.
+# Inputs:
+# - OSM planet in pbf format
+# - csv table of subway networks
+#   (auto-downloaded from https://docs.google.com/spreadsheets/d/1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k)
+# Output:
+# - subways.transit.json
+
+source "$(dirname "$0")/helper_settings.sh"
+source "$REPO_PATH/tools/unix/helper_python.sh"
+
+# Parameters for the process_subways.sh script:
+export PLANET="$PLANET_PBF"
+export SKIP_PLANET_UPDATE="1"
+# http(s) or "file://" URL to a CSV file with a list of subway networks.
+# Auto-downloaded from https://docs.google.com/spreadsheets/d/1SEW1-NiNOnA2qDwievcxYV1FOaQl1mb1fdeyqAxHu3k
+# If unavailable then replace with a local file.
+# TODO: keep the downloaded csv file from the latest run.
+#export CITIES_INFO_URL=""
+export TMPDIR="$BUILD_PATH/subway"
+# The output file, which needs post-processing by transit_graph_generator.py
+export MAPSME="$SUBWAYS_PATH/subway.json"
+
+# Produce additional files needed for https://cdn.organicmaps.app/subway/
+export HTML_DIR="$SUBWAYS_VALIDATOR_PATH"
+export DUMP="$SUBWAYS_VALIDATOR_PATH"
+export GEOJSON="$SUBWAYS_VALIDATOR_PATH"
+export DUMP_CITY_LIST="$SUBWAYS_VALIDATOR_PATH/cities.txt"
+
+# cd to subways repo so relative paths work in the script
+PREVDIR=$(pwd)
+cd "$SUBWAYS_REPO_PATH"
+echo "Running process_subways.sh:"
+./scripts/process_subways.sh 2>&1 | tee "$SUBWAYS_LOG"
+cd "$PREVDIR"
+
+# Make render.html available for map visualization on the web
+cp -r "$SUBWAYS_REPO_PATH"/render/* "$SUBWAYS_VALIDATOR_PATH/"
+
+TRANSIT_TOOL_PATH="$REPO_PATH/tools/python/transit"
+SUBWAYS_GRAPH_FILE="$SUBWAYS_PATH/subways.transit.json"
+
+activate_venv_at_path "$TRANSIT_TOOL_PATH"
+"$PYTHON" "$TRANSIT_TOOL_PATH/transit_graph_generator.py" "$MAPSME" "$SUBWAYS_GRAPH_FILE" 2>&1 | tee -a "$SUBWAYS_LOG"
+deactivate
+
+echo "Generated subways transit graph file:"
+echo "$SUBWAYS_GRAPH_FILE"
+echo "Finished"

tools/unix/maps/helper_settings.sh

@@ -0,0 +1,25 @@
+SETTINGS_FILE="${SETTINGS_FILE:-$(cd "$(dirname "$0")"; pwd -P)/settings.sh}"
+if [ -f "$SETTINGS_FILE" ]; then
+  echo "Using settings from $SETTINGS_FILE"
+  source "$SETTINGS_FILE"
+else
+  echo "Creating a template settings file $SETTINGS_FILE"
+  cat << EOF > "$SETTINGS_FILE"
+# Customize the default settings here.
+# (check the defaults in settings_default.sh)
+
+# The default maps workspace base is ../maps relative to the repo.
+# All source and output and intermediate files will be organized there in corresponding subdirs.
+# E.g. set it to the user's home directory:
+# BASE_PATH="$HOME"
+EOF
+fi
+
+source "$(dirname "$0")/settings_default.sh"
+
+mkdir -p "$BASE_PATH"
+mkdir -p "$BUILD_PATH"
+mkdir -p "$DATA_PATH"
+
+mkdir -p "$PLANET_PATH"
+mkdir -p "$SUBWAYS_PATH"

tools/unix/maps/run-docker.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Run the maps generator via Docker manually without CI
+# See .forgejo/workflows/map-generator.yml for steps to run before the main mapgen process,
+# e.g. clone the repos, get/update planet file, generate isolines etc.
+
+# To build the docker container:
+# cd /mnt/4tbexternal/comaps/tools/unix/maps
+# docker build . -t maps_generator
+#
+# To push for ci/cd, tag for codeberg:
+#   docker login codeberg.org
+#   docker tag maps_generator codeberg.org/comaps/maps_generator:latest
+#   docker push codeberg.org/comaps/maps_generator:latest
+# You can also tag and push the image Id for posterity: codeberg.org/comaps/maps_generator:1234abcd
+# If you get a Dockerfile not found error especially on an XFS partition, try copying Dockerfile to an ext4 partition first.
+# Or use docker via apt instead of snap.
+
+# We assume that the following will be cloned into the container itself at runtime:
+# ~/comaps (comaps main app repo)
+# ~/subways (repo for processing OSM subway/transit info)
+# ~/wikiparser (repo for processing Wikipedia data)
+
+# We also assume a number of files/folders/repos are pre-set-up before mounting via volumes below:
+# /mnt/4tbexternal (base folder for directory traversal)
+# /mnt/4tbexternal/osm-maps (folder for holding generated map data output)
+# /home/planet (folder for holding required input dumps)
+
+docker run \
+  --ulimit nofile=262144:262144 \
+  -v /mnt/4tbexternal/:/mnt/4tbexternal/ \
+  -v /mnt/4tbexternal/osm-planet:/home/planet \
+  -it codeberg.org/comaps/maps_generator:latest \
+  ~/comaps/tools/unix/maps/docker_maps_generator.sh

tools/unix/maps/settings.sh.prod

@@ -0,0 +1,26 @@
+## NOTE: edit the settings.sh file to customize/override the defaults.
+
+# Absolutize & normalize paths.
+REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}"
+
+#TODO: is base path used?
+BASE_PATH="${BASE_PATH:-$REPO_PATH/..}"
+# Source map data and processed outputs e.g. wiki articles
+DATA_PATH="${DATA_PATH:-/home/planet}"
+# Temporary files
+BUILD_PATH="${BUILD_PATH:-$DATA_PATH/build}"
+# Other code repositories, e.g. subways, wikiparser..
+CODE_PATH="${CODE_PATH:-~}"
+
+# OSM planet source files
+
+PLANET_PATH="${PLANET_PATH:-$DATA_PATH/planet}"
+PLANET_PBF="${PLANET_PBF:-$PLANET_PATH/planet-latest.osm.pbf}"
+PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}"
+
+# Subways
+
+SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-/root/subways}"
+SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subway}"
+SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subway.log}"
+SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"

tools/unix/maps/settings_default.sh

@@ -0,0 +1,25 @@
+# NOTE: edit the settings.sh file to customize/override the defaults.
+
+# Absolutize & normalize paths.
+REPO_PATH="${REPO_PATH:-$(cd "$(dirname "$0")/../../.."; pwd -P)}"
+
+BASE_PATH="${BASE_PATH:-$REPO_PATH/../maps}"
+# Temporary files
+BUILD_PATH="${BUILD_PATH:-$BASE_PATH/build}"
+# Other code repositories, e.g. subways, wikiparser..
+CODE_PATH="${CODE_PATH:-$REPO_PATH/..}"
+# Source map data and processed outputs e.g. wiki articles
+DATA_PATH="${DATA_PATH:-$BASE_PATH/data}"
+
+# OSM planet source files
+
+PLANET_PATH="${PLANET_PATH:-$DATA_PATH/planet}"
+PLANET_PBF="${PLANET_PBF:-$PLANET_PATH/planet-latest.osm.pbf}"
+PLANET_O5M="${PLANET_O5M:-$PLANET_PATH/planet-latest.o5m}"
+
+# Subways
+
+SUBWAYS_REPO_PATH="${SUBWAYS_REPO_PATH:-$CODE_PATH/subways}"
+SUBWAYS_PATH="${SUBWAYS_PATH:-$DATA_PATH/subway}"
+SUBWAYS_LOG="${SUBWAYS_LOG:-$SUBWAYS_PATH/subway.log}"
+SUBWAYS_VALIDATOR_PATH="${SUBWAYS_VALIDATOR_PATH:-$SUBWAYS_PATH/validator}"

tools/unix/maps/upload_to_cdn.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Upload new maps version to all CDN nodes (in parallel) and remove old versions.
+
+# Use following commands for deleting older maps:
+#
+# ru1 - keep max 3 maps versions
+# First list all maps versions on the server
+#   sudo rclone lsd ru1:comaps-maps/maps
+# Delete the old version
+#   sudo rclone purge -v ru1:comaps-maps/maps/250713/
+#
+# fi1 - max 3 versions
+#   sudo rclone lsd fi1:/var/www/html/maps
+#   sudo rclone purge -v fi1:/var/www/html/maps/250713/
+#
+# de1 - max 6 versions
+#    sudo rclone lsd de1:/var/www/html/comaps-cdn/maps
+#    sudo rclone purge -v de1:/var/www/html/comaps-cdn/maps/250713/
+#
+# us2 - all versions, don't delete
+#    sudo rclone lsd us2:comaps-map-files/maps
+
+set -e -u
+
+if [ $# -eq 0 ]; then
+  echo "Usage: upload_to_cdn.sh MAPS_PATH"
+  echo "e.g. sudo upload_to_cdn.sh osm-maps/2025_09_06__09_48_08/250906"
+  echo "uploads are run in parallel to us2,ru1,fi1,de1 servers,"
+  echo "subsequent runs will update only missing/differing files,"
+  echo "so its fine to run second time to ensure there were no incomplete transfers"
+  echo "or to run on an unfinished generation first and then again after its fully finished."
+  echo "(sudo is needed to access rclone.conf with servers credentials)"
+  exit 1
+fi
+
+MAPS=$(basename $1)
+DIR=$(dirname $1)/$MAPS
+
+echo "Uploading maps folder $DIR to $MAPS"
+
+# Remove old versions before uploading new ones
+echo "Checking for old versions to remove..."
+
+# ru1 - keep max 3 versions
+echo "Cleaning ru1 (keeping 3 newest versions)..."
+OLD_VERSIONS_RU1=$(rclone lsd ru1:comaps-maps/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_RU1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting ru1:comaps-maps/maps/$version/"
+    rclone purge -v ru1:comaps-maps/maps/$version/
+  fi
+done
+
+# fi1 - keep max 3 versions
+echo "Cleaning fi1 (keeping 3 newest versions)..."
+OLD_VERSIONS_FI1=$(rclone lsd fi1:/var/www/html/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_FI1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting fi1:/var/www/html/maps/$version/"
+    rclone purge -v fi1:/var/www/html/maps/$version/
+  fi
+done
+
+# de1 - keep max 6 versions
+echo "Cleaning de1 (keeping 6 newest versions)..."
+OLD_VERSIONS_DE1=$(rclone lsd de1:/var/www/html/comaps-cdn/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
+for version in $OLD_VERSIONS_DE1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting de1:/var/www/html/comaps-cdn/maps/$version/"
+    rclone purge -v de1:/var/www/html/comaps-cdn/maps/$version/
+  fi
+done
+
+# fr1 - keep max 6 versions
+echo "Cleaning fr1 (keeping 6 newest versions)..."
+OLD_VERSIONS_FR1=$(rclone lsd fr1:/data/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
+for version in $OLD_VERSIONS_FR1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting fr1:/data/maps/$version/"
+    rclone purge -v fr1:/data/maps/$version/
+  fi
+done
+
+# us2 - keep all versions (no cleanup)
+echo "Skipping us2 cleanup (keeping all versions)"
+
+echo "Old version cleanup complete"
+
+echo "Uploading to us2"
+# An explicit mwm/txt filter is used to skip temp files when run for an unfinished generation
+rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS &
+
+echo "Uploading to ru1"
+rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS &
+
+echo "Uploading to fi1"
+rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS &
+
+echo "Uploading to de1"
+rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS &
+
+echo "Uploading to fr1"
+rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS &
+
+# us1 is not used for maps atm
+# rclone lsd us1:/home/dh_zzxxrk/cdn-us-1.comaps.app/maps
+
+wait
+
+echo "Running once more without parallelization to output status:"
+
+echo "us2 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS
+
+echo "ru1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS
+
+echo "fi1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS
+
+echo "de1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS
+
+echo "fr1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS
+
+echo "Upload complete"