Repo created

tools/unix/maps/upload_to_cdn.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+# Upload new maps version to all CDN nodes (in parallel) and remove old versions.
+
+# Use following commands for deleting older maps:
+#
+# ru1 - keep max 3 maps versions
+# First list all maps versions on the server
+#   sudo rclone lsd ru1:comaps-maps/maps
+# Delete the old version
+#   sudo rclone purge -v ru1:comaps-maps/maps/250713/
+#
+# fi1 - max 3 versions
+#   sudo rclone lsd fi1:/var/www/html/maps
+#   sudo rclone purge -v fi1:/var/www/html/maps/250713/
+#
+# de1 - max 6 versions
+#    sudo rclone lsd de1:/var/www/html/comaps-cdn/maps
+#    sudo rclone purge -v de1:/var/www/html/comaps-cdn/maps/250713/
+#
+# us2 - all versions, don't delete
+#    sudo rclone lsd us2:comaps-map-files/maps
+
+set -e -u
+
+if [ $# -eq 0 ]; then
+  echo "Usage: upload_to_cdn.sh MAPS_PATH"
+  echo "e.g. sudo upload_to_cdn.sh osm-maps/2025_09_06__09_48_08/250906"
+  echo "uploads are run in parallel to us2,ru1,fi1,de1 servers,"
+  echo "subsequent runs will update only missing/differing files,"
+  echo "so its fine to run second time to ensure there were no incomplete transfers"
+  echo "or to run on an unfinished generation first and then again after its fully finished."
+  echo "(sudo is needed to access rclone.conf with servers credentials)"
+  exit 1
+fi
+
+MAPS=$(basename $1)
+DIR=$(dirname $1)/$MAPS
+
+echo "Uploading maps folder $DIR to $MAPS"
+
+# Remove old versions before uploading new ones
+echo "Checking for old versions to remove..."
+
+# ru1 - keep max 3 versions
+echo "Cleaning ru1 (keeping 3 newest versions)..."
+OLD_VERSIONS_RU1=$(rclone lsd ru1:comaps-maps/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_RU1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting ru1:comaps-maps/maps/$version/"
+    rclone purge -v ru1:comaps-maps/maps/$version/
+  fi
+done
+
+# fi1 - keep max 3 versions
+echo "Cleaning fi1 (keeping 3 newest versions)..."
+OLD_VERSIONS_FI1=$(rclone lsd fi1:/var/www/html/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
+for version in $OLD_VERSIONS_FI1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting fi1:/var/www/html/maps/$version/"
+    rclone purge -v fi1:/var/www/html/maps/$version/
+  fi
+done
+
+# de1 - keep max 6 versions
+echo "Cleaning de1 (keeping 6 newest versions)..."
+OLD_VERSIONS_DE1=$(rclone lsd de1:/var/www/html/comaps-cdn/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
+for version in $OLD_VERSIONS_DE1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting de1:/var/www/html/comaps-cdn/maps/$version/"
+    rclone purge -v de1:/var/www/html/comaps-cdn/maps/$version/
+  fi
+done
+
+# fr1 - keep max 6 versions
+echo "Cleaning fr1 (keeping 6 newest versions)..."
+OLD_VERSIONS_FR1=$(rclone lsd fr1:/data/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
+for version in $OLD_VERSIONS_FR1; do
+  if [ $version -gt 250101 ]; then
+    echo "  Deleting fr1:/data/maps/$version/"
+    rclone purge -v fr1:/data/maps/$version/
+  fi
+done
+
+# us2 - keep all versions (no cleanup)
+echo "Skipping us2 cleanup (keeping all versions)"
+
+echo "Old version cleanup complete"
+
+echo "Uploading to us2"
+# An explicit mwm/txt filter is used to skip temp files when run for an unfinished generation
+rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS &
+
+echo "Uploading to ru1"
+rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS &
+
+echo "Uploading to fi1"
+rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS &
+
+echo "Uploading to de1"
+rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS &
+
+echo "Uploading to fr1"
+rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS &
+
+# us1 is not used for maps atm
+# rclone lsd us1:/home/dh_zzxxrk/cdn-us-1.comaps.app/maps
+
+wait
+
+echo "Running once more without parallelization to output status:"
+
+echo "us2 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS
+
+echo "ru1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS
+
+echo "fi1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS
+
+echo "de1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS
+
+echo "fr1 status:"
+rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS
+
+echo "Upload complete"