co-maps/tools/unix/maps/upload_to_cdn.sh

#!/usr/bin/env bash

# Upload new maps version to all CDN nodes (in parallel) and remove old versions.

# Use following commands for deleting older maps:
#
# ru1 - keep max 3 maps versions
# First list all maps versions on the server
#   sudo rclone lsd ru1:comaps-maps/maps
# Delete the old version
#   sudo rclone purge -v ru1:comaps-maps/maps/250713/
#
# fi1 - max 3 versions
#   sudo rclone lsd fi1:/var/www/html/maps
#   sudo rclone purge -v fi1:/var/www/html/maps/250713/
#
# de1 - max 6 versions
#    sudo rclone lsd de1:/var/www/html/comaps-cdn/maps
#    sudo rclone purge -v de1:/var/www/html/comaps-cdn/maps/250713/
#
# us2 - all versions, don't delete
#    sudo rclone lsd us2:comaps-map-files/maps

set -e -u

if [ $# -eq 0 ]; then
  echo "Usage: upload_to_cdn.sh MAPS_PATH"
  echo "e.g. sudo upload_to_cdn.sh osm-maps/2025_09_06__09_48_08/250906"
  echo "uploads are run in parallel to us2,ru1,fi1,de1 servers,"
  echo "subsequent runs will update only missing/differing files,"
  echo "so its fine to run second time to ensure there were no incomplete transfers"
  echo "or to run on an unfinished generation first and then again after its fully finished."
  echo "(sudo is needed to access rclone.conf with servers credentials)"
  exit 1
fi

MAPS=$(basename $1)
DIR=$(dirname $1)/$MAPS

echo "Uploading maps folder $DIR to $MAPS"

# Remove old versions before uploading new ones
echo "Checking for old versions to remove..."

# ru1 - keep max 3 versions
echo "Cleaning ru1 (keeping 3 newest versions)..."
OLD_VERSIONS_RU1=$(rclone lsd ru1:comaps-maps/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
for version in $OLD_VERSIONS_RU1; do
  if [ $version -gt 250101 ]; then
    echo "  Deleting ru1:comaps-maps/maps/$version/"
    rclone purge -v ru1:comaps-maps/maps/$version/
  fi
done

# fi1 - keep max 3 versions
echo "Cleaning fi1 (keeping 3 newest versions)..."
OLD_VERSIONS_FI1=$(rclone lsd fi1:/var/www/html/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
for version in $OLD_VERSIONS_FI1; do
  if [ $version -gt 250101 ]; then
    echo "  Deleting fi1:/var/www/html/maps/$version/"
    rclone purge -v fi1:/var/www/html/maps/$version/
  fi
done

# de1 - keep max 6 versions
echo "Cleaning de1 (keeping 6 newest versions)..."
OLD_VERSIONS_DE1=$(rclone lsd de1:/var/www/html/comaps-cdn/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
for version in $OLD_VERSIONS_DE1; do
  if [ $version -gt 250101 ]; then
    echo "  Deleting de1:/var/www/html/comaps-cdn/maps/$version/"
    rclone purge -v de1:/var/www/html/comaps-cdn/maps/$version/
  fi
done

# fr1 - keep max 6 versions
echo "Cleaning fr1 (keeping 6 newest versions)..."
OLD_VERSIONS_FR1=$(rclone lsd fr1:/data/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
for version in $OLD_VERSIONS_FR1; do
  if [ $version -gt 250101 ]; then
    echo "  Deleting fr1:/data/maps/$version/"
    rclone purge -v fr1:/data/maps/$version/
  fi
done

# us2 - keep all versions (no cleanup)
echo "Skipping us2 cleanup (keeping all versions)"

echo "Old version cleanup complete"

echo "Uploading to us2"
# An explicit mwm/txt filter is used to skip temp files when run for an unfinished generation
rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS &

echo "Uploading to ru1"
rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS &

echo "Uploading to fi1"
rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS &

echo "Uploading to de1"
rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS &

echo "Uploading to fr1"
rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS &

# us1 is not used for maps atm
# rclone lsd us1:/home/dh_zzxxrk/cdn-us-1.comaps.app/maps

wait

echo "Running once more without parallelization to output status:"

echo "us2 status:"
rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS

echo "ru1 status:"
rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS

echo "fi1 status:"
rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS

echo "de1 status:"
rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS

echo "fr1 status:"
rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS

echo "Upload complete"