A tool for backing up ATProto related data to S3
willdot.net
1package main
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "io"
8 "log/slog"
9 "os"
10 "path/filepath"
11
12 "github.com/bugsnag/bugsnag-go/v2"
13 "github.com/minio/minio-go/v7"
14)
15
16func backupTangledKnot(ctx context.Context, minioClient *minio.Client, bucketName string) {
17 if os.Getenv("BACKUP_TANGLED_KNOT") != "true" {
18 return
19 }
20
21 backupKnotDB(ctx, minioClient, bucketName)
22}
23
24func backupKnotDB(ctx context.Context, minioClient *minio.Client, bucketName string) {
25 dir := os.Getenv("TANGLED_KNOT_DATABASE_DIRECTORY")
26
27 pipeReader, pipeWriter := io.Pipe()
28 defer pipeReader.Close()
29
30 go compress(dir, pipeWriter)
31
32 _, err := minioClient.PutObject(ctx, bucketName, "knot-db.zip", pipeReader, -1, minio.PutObjectOptions{})
33 if err != nil {
34 slog.Error("stream knot DB to bucket: %w")
35 bugsnag.Notify(err)
36 }
37}
38
39func backupKnotRepos(ctx context.Context, minioClient *minio.Client, bucketName string) {
40 dir := os.Getenv("TANGLED_KNOT_REPOSITORY_DIRECTORY")
41
42 pipeReader, pipeWriter := io.Pipe()
43 defer pipeReader.Close()
44
45 go compress(dir, pipeWriter)
46
47 _, err := minioClient.PutObject(ctx, bucketName, "knot-repos.zip", pipeReader, -1, minio.PutObjectOptions{})
48 if err != nil {
49 slog.Error("stream knot repos to bucket: %w")
50 bugsnag.Notify(err)
51 }
52}
53
54func compress(src string, writer io.WriteCloser) error {
55 zipWriter := gzip.NewWriter(writer)
56 tarWriter := tar.NewWriter(zipWriter)
57
58 defer writer.Close()
59 defer zipWriter.Close()
60 defer tarWriter.Close()
61
62 filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
63 header, err := tar.FileInfoHeader(fi, file)
64 if err != nil {
65 return err
66 }
67
68 // must provide real name
69 // (see https://golang.org/src/archive/tar/common.go?#L626)
70 header.Name = filepath.ToSlash(file)
71
72 if err := tarWriter.WriteHeader(header); err != nil {
73 return err
74 }
75 // if not a dir, write file content
76 if !fi.IsDir() {
77 data, err := os.Open(file)
78 if err != nil {
79 return err
80 }
81 if _, err := io.Copy(tarWriter, data); err != nil {
82 return err
83 }
84 }
85 return nil
86 })
87
88 // produce tar
89 if err := tarWriter.Close(); err != nil {
90 return err
91 }
92 // produce gzip
93 if err := zipWriter.Close(); err != nil {
94 return err
95 }
96 //
97 return nil
98}