#!/usr/bin/env bash

# Use rclone to sync the last two good Wikimedia XML data dumps.

set -euxo pipefail

if [ -z "$1" ]; then
    echo "Please supply a mirror URL"
    exit 1
fi
MIRROR_URL="$1"

RCLONE_REMOTE=$(basename "$MIRROR_URL")
NTFY_IP=$(sudo tailscale status | grep "dietpi" | awk '{print $1}')

wget "${MIRROR_URL}/rsync-filelist-last-2-good.txt" \
    -O /home/jas/rsync-filelist-last-2-good.txt

grep "enwiki" /home/jas/rsync-filelist-last-2-good.txt |
    grep -v "tenwiki" |
    tee /home/jas/rsync-filelist-last-2-good-en.txt

rm -fv /home/jas/rsync-filelist-last-2-good.txt

rclone sync \
    --http-no-head \
    --transfers 8 \
    --include-from /home/jas/rsync-filelist-last-2-good-en.txt "${RCLONE_REMOTE}:" \
    /naspool/archives/wikimedia-xmldatadumps-en

rm -fv /home/jas/rsync-filelist-last-2-good-en.txt

curl \
    -H prio:default \
    -H tags:incoming_envelope \
    -d "Syncing of wikimedia xml datadumps succeeded" \
    "http://${NTFY_IP}:8080/wikimedia_xmldatadumps_en"

exit 0