feat: backup using restic entire app_data to backblaze and storj

Changed files
+121 -38
butane
configs
containers
scripts
+34 -34
.terraform.lock.hcl
···
# Manual edits may be lost in future updates.
provider "registry.opentofu.org/bpg/proxmox" {
-
version = "0.73.1"
-
constraints = "0.73.1"
+
version = "0.76.0"
+
constraints = "0.76.0"
hashes = [
-
"h1:5cdvbQero7ByLSPRBlfNpjV4hjPvNtya9tSZuwgIhRw=",
-
"zh:094ee6bcd664890918e30ff602d9042a94d6c9e7e687ecd76ccaf39f6019d32f",
-
"zh:09fd81a2088e280a4947ce7aa6de97612129befccedcbe12ba74ffb8f77c547b",
-
"zh:488c284e1aed1ef9ed6587956b84ad96433cbecb6c373998a594febac206df5f",
-
"zh:52668ff9af14662f06fc0bdaeb633fe76d6c4bfbe0024b447d32ca3424b2c136",
-
"zh:548e7889b0d32d11a4c18aa4c5444291086010eef62d952f84740669788d8bc8",
-
"zh:5fcaf8ea3e2f0657947394993a5e1f41f4e21f0676a408624c9fc44489e5d59f",
-
"zh:67949764ca14d110aa94ea92d54e9e337f382932454e28d0ded389bd65d8d0fd",
-
"zh:7d8263ecdd98f558e88afa95628b3c4e0b8480aa77b470bbc5695fd8e5206109",
-
"zh:95d9615af70eb73eeb0f1d895baf8c8d9f9c71e7f40143dcb0bf1aa4c792fde4",
-
"zh:af8c3d8289cfd02ba7e2fcf73e325d7950cba113775139b3123d2cdd806c3cdd",
-
"zh:ba5b81aeab7a9e84bcbffea4b372252088add1ad6af0a34f0702c0fc0abdf04b",
-
"zh:d94842ce37447c2cf8981ad602ab4d2f410ee5e724a78611472c87f791f4458d",
-
"zh:da2d5adc66d7b305ca114ceeecf5133f32211640c9870c4b5e7c7782982a40e3",
+
"h1:eDxlV3RBgpcLegK4ld73R7yZNwMIeuPg6IazvNRV5xs=",
+
"zh:0fd9529786f688855103a945d70d6ae212281ab08651266bb40a5e9945c4e4d3",
+
"zh:1babbffb85e406c4eeb0821c140f0eb4b580f01a14471a5c9344b9cab9888bc4",
+
"zh:2a052111b42c9adcef471edb5c72268801a2560e25ec75398958854c7cfb1445",
+
"zh:369fe361179f29aadf6c4ccef99329baf7c5ce81ebebab411b361ed4e156bb7a",
+
"zh:4168d53d2d9312ffbbb9f2d2c73a198aca90e77741ade744cc7667ce94bbf00b",
+
"zh:506ae6f2bc49f7762bf54b05a51ac6efb4c22cd696ee5063fe329e35e8819673",
+
"zh:55faa7276c3a600a54e70331afdd9d6a35484b0b1dba871dfa0a2d08a6ba4d51",
+
"zh:57f1bc30e3db042115912d0c2cc3d0fa40a09ed59f34d4340152d8af94e65d74",
+
"zh:72a4ab4a5f04d079426070dbc9eca0f71d29ed080e40ff3f08a072bf84776267",
+
"zh:7854c38e3362aa02c3a454147b9e06c59ec81617c2f524b52654580bd8dae580",
+
"zh:a8c4f28083ff0277ef7d68bb6b4effe4400244e0a18c8d8aab759c9743edc66f",
+
"zh:c5480c547c08d6a9506b2444bf2bb347475f0f8de99698ce56e49b4cbc46acb7",
+
"zh:e7c5b359f584adb16dd534fff108c9403862a9f86806fd7ec3e27136fcd9da5e",
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
-
"zh:fb762b9c687fb143116bda5f8a29ca8d61d65a2fcf1bed772828d1b3ff91db5e",
+
"zh:f6ce33ea77e5a7b380cebf3b8636c5371aab212d4604ccc73e86faa32efe4ba5",
]
}
···
}
provider "registry.opentofu.org/maxlaverse/bitwarden" {
-
version = "0.13.5"
-
constraints = "0.13.5"
+
version = "0.13.6"
+
constraints = "0.13.6"
hashes = [
-
"h1:OaA1Rzyg2FOVQuIDXUxBKkYqwLyYj6i3GKw/VPZUh4g=",
-
"zh:1863940c132f8d033b1907e1991b5c806a3d7e07b4f3c5a8cef0c3e1a9084b06",
-
"zh:3abe09ac78f9e4c79b2f0dc2a401891339b5b9c9db419b2c5fb6f8245cd144f6",
-
"zh:3e86564db23c7cebddf2dd594fd8cf85e8a6b915a46bc6f2ea92171365d41e5c",
-
"zh:44c5059426c294478f79fedf36e20e321b3f774d54de7fc40f7c4734192cebb9",
-
"zh:7404950f4e0d69d7a1e4264e06598f3d236cb2020921fabb7145e0fa5b06afed",
-
"zh:86d9ed1bb69fdcf1c995153da85f5f9ae0b1e6c79ba98c56017f98c550683212",
-
"zh:985b6de9b01b704bb15940b85e1ba622d11bbe7380cfa138bac7009d103de92f",
-
"zh:ac86dc26400d92cba2e80b0595218575e1b6d58a3f82f7094fb56252c772289c",
-
"zh:c199dbcc6b82e5015391404b23f8bda8f1a4260762d32b592e3a0da81b4f4add",
-
"zh:c9208ee8392b0ebf5facce583cb172be0d365340c8253402ad9075ae742d18d7",
-
"zh:d36551c78e2c8e0c4a9ea441140edb90c84728b05c4056a6ef09b24303609a6b",
-
"zh:d6fd8590c87817b2094ecce9e3b6b8e4158f54b28d88b2cdb663a131a0bcc950",
-
"zh:e04127aa44d583b05d225aea98805d91fb71f8ddc4cb7500ecc8bf18e5f57174",
+
"h1:g7YFxAH3wK82tX9nNQpKlZFTqD2pvjhN3yPe2dAcX6w=",
+
"zh:0548cf354bd65bb146e63db28fe3c6ab9339118f0255271886d8088325c7a044",
+
"zh:1eb2f8b07da17acb0cf9fde60b15facfe599faaf912b0a540ad18629e6fd76eb",
+
"zh:2f58bcce32cb8082f0627f7ca12925d73e0c9a2fe9501a6060dfa90472a5ddc3",
+
"zh:2fea04c23e1333a7682538033dc87be2df7ed55226c3972e62600e434ea1e3de",
+
"zh:34cf36acad9a731bdce793995ffd539b3578a4e15cf04e59faf8de884e585ec0",
+
"zh:3516e5d815938b87749bdb427971955b875e1e0dd051f180988a8731cc0ca2ec",
+
"zh:578b199ac64e8ecef439a15c4d0b5be42fee3872706ca1789842e3471cb59b0c",
+
"zh:6ac028d60fb9fd9dc8115888786224f38f59807d0971ed6e8ac3882eb5dbf680",
+
"zh:743f1fd4933d9bfae0256921326649d5139a7023040d86c36cc48a1a4a8f04fc",
+
"zh:893bafe00218cd5352c176c54cef573d709705f8d569ae49fd0e7eaf0c8f7d44",
+
"zh:a45c56bb6cde3070ca1e3d2f500811f585f9c2cd1cc337c2c1e50e6283cdc86f",
+
"zh:c39941a88c19ff15a55fcf88cdb98fbc917355db7e7100f6c9beff8016d15a1d",
+
"zh:dc5e6fbb1a06134233ebc42bfbb1a7979aac5625c186f8ead5d28295cfbd0cb1",
"zh:f809ab383cca0a5f83072981c64208cbd7fa67e986a86ee02dd2c82333221e32",
-
"zh:fcb4384d6614c5d39146188491fbab1f455bebc29a4a518ecc3559a6b544a0dc",
+
"zh:ffec18112906046d45ffd1bb0fbd8387e99e4b8834c215ad0d5790bc71df450f",
]
}
+65 -1
butane/fcos.yml.tftpl
···
[Service]
Type=oneshot
RemainAfterExit=yes
-
ExecStart=/usr/bin/rpm-ostree install --allow-inactive --assumeyes --reboot qemu-guest-agent intel-gpu-tools
+
ExecStart=/usr/bin/rpm-ostree install --allow-inactive --assumeyes --reboot qemu-guest-agent restic intel-gpu-tools
ExecStart=/bin/touch /var/lib/%N.stamp
[Install]
···
- name: nftables.service
enabled: true
+
+
- name: restic-backblaze.service
+
contents: |
+
[Unit]
+
Description=Backup Docker data to Backblaze
+
Wants=network-online.target
+
After=network-online.target iscsi.service
+
+
[Service]
+
Type=oneshot
+
LoadCredential=restic-b2-account-id
+
LoadCredential=restic-b2-account-key
+
LoadCredential=restic-password
+
Environment=RESTIC_PASSWORD_FILE=%d/restic-password
+
ExecStart=/bin/bash -c "export B2_ACCOUNT_ID=$(cat $CREDENTIALS_DIRECTORY/restic-b2-account-id); export B2_ACCOUNT_KEY=$(cat $CREDENTIALS_DIRECTORY/restic-b2-account-key); restic -r b2:krasovsky-homelab:app-data --verbose backup /var/mnt/docker/app_data"
+
+
[Install]
+
WantedBy=multi-user.target
+
+
- name: restic-backblaze.timer
+
enabled: true
+
contents: |
+
[Unit]
+
Description=Daily backup to Backblaze
+
+
[Timer]
+
OnCalendar=daily
+
RandomizedDelaySec=3600
+
Unit=restic-backblaze.service
+
+
[Install]
+
WantedBy=timers.target
+
+
- name: restic-storj.service
+
contents: |
+
[Unit]
+
Description=Backup Docker data to Storj
+
Wants=network-online.target
+
After=network-online.target iscsi.service
+
+
[Service]
+
Type=oneshot
+
LoadCredential=restic-aws-access-key-id
+
LoadCredential=restic-aws-secret-access-key
+
LoadCredential=restic-password
+
Environment=RESTIC_PASSWORD_FILE=%d/restic-password
+
ExecStart=/bin/bash -c "export AWS_ACCESS_KEY_ID=$(cat $CREDENTIALS_DIRECTORY/restic-aws-access-key-id); export AWS_SECRET_ACCESS_KEY=$(cat $CREDENTIALS_DIRECTORY/restic-aws-secret-access-key); restic -r s3:https://gateway.eu1.storjshare.io/homelab-backup/app-data --verbose backup /var/mnt/docker/app_data"
+
+
[Install]
+
WantedBy=multi-user.target
+
+
- name: restic-storj.timer
+
enabled: true
+
contents: |
+
[Unit]
+
Description=Daily backup to Storj
+
+
[Timer]
+
OnCalendar=daily
+
RandomizedDelaySec=3600
+
Unit=restic-storj.service
+
+
[Install]
+
WantedBy=timers.target
kernel_arguments:
should_exist:
+4
configs/containers/systemd/open-webui/open-webui-pipelines.container.tftpl
···
# Unfortunately open-webui cannot work under non-root user
+
Secret=open-webui-anthropic-api-key,type=env,target=ANTHROPIC_API_KEY
+
Secret=open-webui-google-api-key,type=env,target=GOOGLE_API_KEY
+
Secret=open-webui-openai-api-key,type=env,target=OPENAI_API_KEY
+
Label="glance.parent=open-webui"
Label="glance.name=Pipelines"
Label="glance.hide=false"
+2 -2
main.tf
···
required_providers {
proxmox = {
source = "bpg/proxmox"
-
version = "0.73.1"
+
version = "0.76.0"
}
bitwarden = {
source = "maxlaverse/bitwarden"
-
version = "0.13.5"
+
version = "0.13.6"
}
ct = {
source = "poseidon/ct"
+8 -1
scripts/init_fcos.sh.tftpl
···
#!/bin/bash
set -e
-
echo "Importing secrets..."
+
echo "Importing Podman and Restic secrets..."
# Bitwarden Secrets Manager CLI requires to save state in order to work correctly, but
# Fedora CoreOS has strict SELinux policies, so we need to make proper adjustments.
%{ for name, id in secrets ~}
+
%{ if !startswith(name, "restic_") ~}
podman run --rm -it -v /var/home/core:/home/app --user 1000:1000 --uidmap +1000:@1000:1 --security-opt=label=disable \
docker.io/bitwarden/bws secret get --color=no --access-token=${bws_access_token} ${id} | jq -r .value | tr -d '\n' | \
podman secret create --replace ${replace(name, "_", "-")} - # I prefer '-' divider for everything related to podman
+
%{ else ~}
+
podman run --rm -it -v /var/home/core:/home/app --user 1000:1000 --uidmap +1000:@1000:1 --security-opt=label=disable \
+
docker.io/bitwarden/bws secret get --color=no --access-token=${bws_access_token} ${id} | jq -r .value | tr -d '\n' | \
+
sudo dd status=none of=/etc/credstore/${replace(name, "_", "-")}
+
sudo chmod 0600 /etc/credstore/${replace(name, "_", "-")}
+
%{ endif ~}
%{ endfor ~}
echo "Starting Quadlets..."
+8
variables.tf
···
actual_budget_openid_client_secret = "5754702b-d9d5-4127-b5ab-b29200abdd6a"
open_webui_oauth_client_secret = "b595040b-a23a-44af-8bff-b29200ad6258"
open_webui_google_drive_api_key = "d24bf77b-622d-4ef0-88ae-b2b200d67ee1"
+
open_webui_anthropic_api_key = "104a349b-a4be-4d3a-9c0b-b2c700e64c9a"
+
open_webui_google_api_key = "3016f3ef-c14c-4f4c-8439-b2c700e62f21"
+
open_webui_openai_api_key = "24fd45e2-0fd3-42cd-8fd5-b2c700e66731"
karakeep_oauth_client_secret = "784d379b-bcaf-424f-bc77-b29500ff1be6"
karakeep_openai_api_key = "98f5ccdf-d4b1-4883-b4e3-b295010ba589"
meili_master_key = "a67874c5-95c2-4f7a-b335-b295010010e0"
···
glance_github_token = "de3353d8-09d9-4063-b513-b2a3008cc2c9"
tangled_knot_server_secret = "a58caac0-1c07-4152-89e6-b2a900c8fe8f"
forward_info_bot_telegram_token = "f8eda775-f945-4eb8-b48a-b2b80092cf54"
+
restic_aws_access_key_id = "2743cf63-05ae-45b4-997f-b2c700dfabef"
+
restic_aws_secret_access_key = "134279a9-b3ee-4309-ae9e-b2c700dfe86c"
+
restic_b2_account_id = "3e058bd3-e13d-4b6a-9d48-b2c700e00d62"
+
restic_b2_account_key = "ddc2f07b-47ca-49b2-ae41-b2c700e02f01"
+
restic_password = "52ce5eb2-98ae-4243-ba08-b2c700e04b7e"
}
}