128 lines
4.2 KiB
Bash
Executable file
128 lines
4.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
# Upload new maps version to all CDN nodes (in parallel) and remove old versions.
|
|
|
|
# Use following commands for deleting older maps:
|
|
#
|
|
# ru1 - keep max 3 maps versions
|
|
# First list all maps versions on the server
|
|
# sudo rclone lsd ru1:comaps-maps/maps
|
|
# Delete the old version
|
|
# sudo rclone purge -v ru1:comaps-maps/maps/250713/
|
|
#
|
|
# fi1 - max 3 versions
|
|
# sudo rclone lsd fi1:/var/www/html/maps
|
|
# sudo rclone purge -v fi1:/var/www/html/maps/250713/
|
|
#
|
|
# de1 - max 6 versions
|
|
# sudo rclone lsd de1:/var/www/html/comaps-cdn/maps
|
|
# sudo rclone purge -v de1:/var/www/html/comaps-cdn/maps/250713/
|
|
#
|
|
# us2 - all versions, don't delete
|
|
# sudo rclone lsd us2:comaps-map-files/maps
|
|
|
|
set -e -u
|
|
|
|
if [ $# -eq 0 ]; then
|
|
echo "Usage: upload_to_cdn.sh MAPS_PATH"
|
|
echo "e.g. sudo upload_to_cdn.sh osm-maps/2025_09_06__09_48_08/250906"
|
|
echo "uploads are run in parallel to us2,ru1,fi1,de1 servers,"
|
|
echo "subsequent runs will update only missing/differing files,"
|
|
echo "so its fine to run second time to ensure there were no incomplete transfers"
|
|
echo "or to run on an unfinished generation first and then again after its fully finished."
|
|
echo "(sudo is needed to access rclone.conf with servers credentials)"
|
|
exit 1
|
|
fi
|
|
|
|
MAPS=$(basename $1)
|
|
DIR=$(dirname $1)/$MAPS
|
|
|
|
echo "Uploading maps folder $DIR to $MAPS"
|
|
|
|
# Remove old versions before uploading new ones
|
|
echo "Checking for old versions to remove..."
|
|
|
|
# ru1 - keep max 3 versions
|
|
echo "Cleaning ru1 (keeping 3 newest versions)..."
|
|
OLD_VERSIONS_RU1=$(rclone lsd ru1:comaps-maps/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
|
|
for version in $OLD_VERSIONS_RU1; do
|
|
if [ $version -gt 250101 ]; then
|
|
echo " Deleting ru1:comaps-maps/maps/$version/"
|
|
rclone purge -v ru1:comaps-maps/maps/$version/
|
|
fi
|
|
done
|
|
|
|
# fi1 - keep max 3 versions
|
|
echo "Cleaning fi1 (keeping 3 newest versions)..."
|
|
OLD_VERSIONS_FI1=$(rclone lsd fi1:/var/www/html/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +4)
|
|
for version in $OLD_VERSIONS_FI1; do
|
|
if [ $version -gt 250101 ]; then
|
|
echo " Deleting fi1:/var/www/html/maps/$version/"
|
|
rclone purge -v fi1:/var/www/html/maps/$version/
|
|
fi
|
|
done
|
|
|
|
# de1 - keep max 6 versions
|
|
echo "Cleaning de1 (keeping 6 newest versions)..."
|
|
OLD_VERSIONS_DE1=$(rclone lsd de1:/var/www/html/comaps-cdn/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
|
|
for version in $OLD_VERSIONS_DE1; do
|
|
if [ $version -gt 250101 ]; then
|
|
echo " Deleting de1:/var/www/html/comaps-cdn/maps/$version/"
|
|
rclone purge -v de1:/var/www/html/comaps-cdn/maps/$version/
|
|
fi
|
|
done
|
|
|
|
# fr1 - keep max 6 versions
|
|
echo "Cleaning fr1 (keeping 6 newest versions)..."
|
|
OLD_VERSIONS_FR1=$(rclone lsd fr1:/data/maps --max-depth 1 | awk '{print $5}' | sort -r | tail -n +7)
|
|
for version in $OLD_VERSIONS_FR1; do
|
|
if [ $version -gt 250101 ]; then
|
|
echo " Deleting fr1:/data/maps/$version/"
|
|
rclone purge -v fr1:/data/maps/$version/
|
|
fi
|
|
done
|
|
|
|
# us2 - keep all versions (no cleanup)
|
|
echo "Skipping us2 cleanup (keeping all versions)"
|
|
|
|
echo "Old version cleanup complete"
|
|
|
|
echo "Uploading to us2"
|
|
# An explicit mwm/txt filter is used to skip temp files when run for an unfinished generation
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS &
|
|
|
|
echo "Uploading to ru1"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS &
|
|
|
|
echo "Uploading to fi1"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS &
|
|
|
|
echo "Uploading to de1"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS &
|
|
|
|
echo "Uploading to fr1"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS &
|
|
|
|
# us1 is not used for maps atm
|
|
# rclone lsd us1:/home/dh_zzxxrk/cdn-us-1.comaps.app/maps
|
|
|
|
wait
|
|
|
|
echo "Running once more without parallelization to output status:"
|
|
|
|
echo "us2 status:"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR us2:comaps-map-files/maps/$MAPS
|
|
|
|
echo "ru1 status:"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR ru1:comaps-maps/maps/$MAPS
|
|
|
|
echo "fi1 status:"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR fi1:/var/www/html/maps/$MAPS
|
|
|
|
echo "de1 status:"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR de1:/var/www/html/comaps-cdn/maps/$MAPS
|
|
|
|
echo "fr1 status:"
|
|
rclone copy -v --include "*.{mwm,txt}" $DIR fr1:/data/maps/$MAPS
|
|
|
|
echo "Upload complete"
|