Assorted shell and Python scripts
hyperreal.bsky.moonshadow.dev
1#!/usr/bin/env bash
2
3# Use rclone to sync the last two good Wikimedia XML data dumps.
4
5set -euxo pipefail
6
7if [ -z "$1" ]; then
8 echo "Please supply a mirror URL"
9 exit 1
10fi
11MIRROR_URL="$1"
12
13RCLONE_REMOTE=$(basename "$MIRROR_URL")
14NTFY_IP=$(sudo tailscale status | grep "dietpi" | awk '{print $1}')
15
16wget "${MIRROR_URL}/rsync-filelist-last-2-good.txt" \
17 -O /home/jas/rsync-filelist-last-2-good.txt
18
19grep "enwiki" /home/jas/rsync-filelist-last-2-good.txt |
20 grep -v "tenwiki" |
21 tee /home/jas/rsync-filelist-last-2-good-en.txt
22
23rm -fv /home/jas/rsync-filelist-last-2-good.txt
24
25rclone sync \
26 --http-no-head \
27 --transfers 8 \
28 --include-from /home/jas/rsync-filelist-last-2-good-en.txt "${RCLONE_REMOTE}:" \
29 /naspool/archives/wikimedia-xmldatadumps-en
30
31rm -fv /home/jas/rsync-filelist-last-2-good-en.txt
32
33curl \
34 -H prio:default \
35 -H tags:incoming_envelope \
36 -d "Syncing of wikimedia xml datadumps succeeded" \
37 "http://${NTFY_IP}:8080/wikimedia_xmldatadumps_en"
38
39exit 0