A tool for backing up ATProto related data to S3
willdot.net
1package main
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "io"
8 "log/slog"
9 "os"
10 "path/filepath"
11
12 "github.com/bugsnag/bugsnag-go/v2"
13 "github.com/minio/minio-go/v7"
14)
15
16func backupTangledKnot(ctx context.Context, minioClient *minio.Client, bucketName string) {
17 backupKnotDB(ctx, minioClient, bucketName)
18 backupKnotRepos(ctx, minioClient, bucketName)
19}
20
21func backupKnotDB(ctx context.Context, minioClient *minio.Client, bucketName string) {
22 dir := os.Getenv("TANGLED_KNOT_DATABASE_DIRECTORY")
23 if dir == "" {
24 slog.Info("TANGLED_KNOT_DATABASE_DIRECTORY env not set - skipping knot DB backup")
25 }
26
27 pipeReader, pipeWriter := io.Pipe()
28 defer pipeReader.Close()
29
30 go compress(dir, pipeWriter)
31
32 _, err := minioClient.PutObject(ctx, bucketName, "knot-db.zip", pipeReader, -1, minio.PutObjectOptions{})
33 if err != nil {
34 slog.Error("stream knot DB to bucket: %w")
35 bugsnag.Notify(err)
36 }
37}
38
39func backupKnotRepos(ctx context.Context, minioClient *minio.Client, bucketName string) {
40 dir := os.Getenv("TANGLED_KNOT_REPOSITORY_DIRECTORY")
41 if dir == "" {
42 slog.Info("TANGLED_KNOT_REPOSITORY_DIRECTORY env not set - skipping knot repo backup")
43 }
44
45 pipeReader, pipeWriter := io.Pipe()
46 defer pipeReader.Close()
47
48 go compress(dir, pipeWriter)
49
50 _, err := minioClient.PutObject(ctx, bucketName, "knot-repos.zip", pipeReader, -1, minio.PutObjectOptions{})
51 if err != nil {
52 slog.Error("stream knot repos to bucket: %w")
53 bugsnag.Notify(err)
54 }
55}
56
57func compress(src string, writer io.WriteCloser) error {
58 zipWriter := gzip.NewWriter(writer)
59 tarWriter := tar.NewWriter(zipWriter)
60
61 defer writer.Close()
62 defer zipWriter.Close()
63 defer tarWriter.Close()
64
65 filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
66 header, err := tar.FileInfoHeader(fi, file)
67 if err != nil {
68 return err
69 }
70
71 // must provide real name
72 // (see https://golang.org/src/archive/tar/common.go?#L626)
73 header.Name = filepath.ToSlash(file)
74
75 if err := tarWriter.WriteHeader(header); err != nil {
76 return err
77 }
78 // if not a dir, write file content
79 if !fi.IsDir() {
80 data, err := os.Open(file)
81 if err != nil {
82 return err
83 }
84 if _, err := io.Copy(tarWriter, data); err != nil {
85 return err
86 }
87 }
88 return nil
89 })
90
91 // produce tar
92 if err := tarWriter.Close(); err != nil {
93 return err
94 }
95 // produce gzip
96 if err := zipWriter.Close(); err != nil {
97 return err
98 }
99 //
100 return nil
101}