commit 2b30b6c99e7f11790eec9ef961985764c26fcc10 · willdot.net/back-at-it

.env.example

···

       1
       1
       +
       ENDPOINT="S3-endpoint"

     

       2
       2
       +
       ACCESS_ID="S3-ID"

     

       3
       3
       +
       SECRET_ACCESS_KEY="S3-secret"

     

       4
       4
       +
       BUCKET_NAME="my-super-duper-bucket"

     

       5
       5
       +
       DID="the-did-to-backup"

     

       6
       6
       +
       PDS_HOST="https://your-pds.com"

.gitignore

+27

go.mod

···

       1
       1
       +
       module tangled.sh/willdot.net/backatit

     

       2
       2
       +
       

     

       3
       3
       +
       go 1.25.0

     

       4
       4
       +
       

     

       5
       5
       +
       require (

     

       6
       6
       +
       	github.com/joho/godotenv v1.5.1

     

       7
       7
       +
       	github.com/minio/minio-go/v7 v7.0.95

     

       8
       8
       +
       )

     

       9
       9
       +
       

     

       10
       10
       +
       require (

     

       11
       11
       +
       	github.com/dustin/go-humanize v1.0.1 // indirect

     

       12
       12
       +
       	github.com/go-ini/ini v1.67.0 // indirect

     

       13
       13
       +
       	github.com/goccy/go-json v0.10.5 // indirect

     

       14
       14
       +
       	github.com/google/uuid v1.6.0 // indirect

     

       15
       15
       +
       	github.com/klauspost/compress v1.18.0 // indirect

     

       16
       16
       +
       	github.com/klauspost/cpuid/v2 v2.2.11 // indirect

     

       17
       17
       +
       	github.com/minio/crc64nvme v1.0.2 // indirect

     

       18
       18
       +
       	github.com/minio/md5-simd v1.1.2 // indirect

     

       19
       19
       +
       	github.com/philhofer/fwd v1.2.0 // indirect

     

       20
       20
       +
       	github.com/rs/xid v1.6.0 // indirect

     

       21
       21
       +
       	github.com/stretchr/testify v1.10.0 // indirect

     

       22
       22
       +
       	github.com/tinylib/msgp v1.3.0 // indirect

     

       23
       23
       +
       	golang.org/x/crypto v0.39.0 // indirect

     

       24
       24
       +
       	golang.org/x/net v0.41.0 // indirect

     

       25
       25
       +
       	golang.org/x/sys v0.33.0 // indirect

     

       26
       26
       +
       	golang.org/x/text v0.26.0 // indirect

     

       27
       27
       +
       )

+43

go.sum

···

       1
       1
       +
       github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

     

       2
       2
       +
       github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

     

       3
       3
       +
       github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

     

       4
       4
       +
       github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=

     

       5
       5
       +
       github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=

     

       6
       6
       +
       github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=

     

       7
       7
       +
       github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=

     

       8
       8
       +
       github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=

     

       9
       9
       +
       github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=

     

       10
       10
       +
       github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

     

       11
       11
       +
       github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=

     

       12
       12
       +
       github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=

     

       13
       13
       +
       github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=

     

       14
       14
       +
       github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=

     

       15
       15
       +
       github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=

     

       16
       16
       +
       github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=

     

       17
       17
       +
       github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=

     

       18
       18
       +
       github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=

     

       19
       19
       +
       github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=

     

       20
       20
       +
       github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=

     

       21
       21
       +
       github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=

     

       22
       22
       +
       github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=

     

       23
       23
       +
       github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=

     

       24
       24
       +
       github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=

     

       25
       25
       +
       github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=

     

       26
       26
       +
       github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

     

       27
       27
       +
       github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

     

       28
       28
       +
       github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=

     

       29
       29
       +
       github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=

     

       30
       30
       +
       github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=

     

       31
       31
       +
       github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=

     

       32
       32
       +
       github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=

     

       33
       33
       +
       github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=

     

       34
       34
       +
       golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=

     

       35
       35
       +
       golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=

     

       36
       36
       +
       golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=

     

       37
       37
       +
       golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=

     

       38
       38
       +
       golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=

     

       39
       39
       +
       golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=

     

       40
       40
       +
       golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=

     

       41
       41
       +
       golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=

     

       42
       42
       +
       gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

     

       43
       43
       +
       gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

+214

main.go

···

       1
       1
       +
       package main

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"archive/zip"

     

       5
       5
       +
       	"context"

     

       6
       6
       +
       	"encoding/json"

     

       7
       7
       +
       	"fmt"

     

       8
       8
       +
       	"io"

     

       9
       9
       +
       	"log/slog"

     

       10
       10
       +
       	"net/http"

     

       11
       11
       +
       	"os"

     

       12
       12
       +
       

     

       13
       13
       +
       	"github.com/joho/godotenv"

     

       14
       14
       +
       	"github.com/minio/minio-go/v7"

     

       15
       15
       +
       	"github.com/minio/minio-go/v7/pkg/credentials"

     

       16
       16
       +
       )

     

       17
       17
       +
       

     

       18
       18
       +
       func main() {

     

       19
       19
       +
       	ctx := context.Background()

     

       20
       20
       +
       

     

       21
       21
       +
       	err := godotenv.Load(".env")

     

       22
       22
       +
       	if err != nil {

     

       23
       23
       +
       		if !os.IsNotExist(err) {

     

       24
       24
       +
       			slog.Error("load env", "error", err)

     

       25
       25
       +
       			return

     

       26
       26
       +
       		}

     

       27
       27
       +
       	}

     

       28
       28
       +
       

     

       29
       29
       +
       	minioClient, err := createMinioClient()

     

       30
       30
       +
       	if err != nil {

     

       31
       31
       +
       		slog.Error("create minio client", "error", err)

     

       32
       32
       +
       		return

     

       33
       33
       +
       	}

     

       34
       34
       +
       

     

       35
       35
       +
       	bucketName := os.Getenv("BUCKET_NAME")

     

       36
       36
       +
       

     

       37
       37
       +
       	err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})

     

       38
       38
       +
       	if err != nil {

     

       39
       39
       +
       		slog.Error("create bucket", "error", err)

     

       40
       40
       +
       		return

     

       41
       41
       +
       	}

     

       42
       42
       +
       

     

       43
       43
       +
       	err = backupRepo(ctx, minioClient, bucketName)

     

       44
       44
       +
       	if err != nil {

     

       45
       45
       +
       		slog.Error("backup repo", "error", err)

     

       46
       46
       +
       		return

     

       47
       47
       +
       	}

     

       48
       48
       +
       

     

       49
       49
       +
       	err = backupBlobs(ctx, minioClient, bucketName)

     

       50
       50
       +
       	if err != nil {

     

       51
       51
       +
       		slog.Error("backup blobs", "error", err)

     

       52
       52
       +
       		return

     

       53
       53
       +
       	}

     

       54
       54
       +
       }

     

       55
       55
       +
       

     

       56
       56
       +
       func createMinioClient() (*minio.Client, error) {

     

       57
       57
       +
       	endpoint := os.Getenv("ENDPOINT")

     

       58
       58
       +
       	accessKeyID := os.Getenv("ACCESS_ID")

     

       59
       59
       +
       	secretAccessKey := os.Getenv("SECRET_ACCESS_KEY")

     

       60
       60
       +
       	useSSL := true

     

       61
       61
       +
       

     

       62
       62
       +
       	return minio.New(endpoint, &minio.Options{

     

       63
       63
       +
       		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),

     

       64
       64
       +
       		Secure: useSSL,

     

       65
       65
       +
       	})

     

       66
       66
       +
       }

     

       67
       67
       +
       

     

       68
       68
       +
       func backupRepo(ctx context.Context, minioClient *minio.Client, bucketName string) error {

     

       69
       69
       +
       	pdsHost := os.Getenv("PDS_HOST")

     

       70
       70
       +
       	did := os.Getenv("DID")

     

       71
       71
       +
       

     

       72
       72
       +
       	url := fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo?did=%s", pdsHost, did)

     

       73
       73
       +
       	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)

     

       74
       74
       +
       	if err != nil {

     

       75
       75
       +
       		return fmt.Errorf("create get repo request: %w", err)

     

       76
       76
       +
       	}

     

       77
       77
       +
       

     

       78
       78
       +
       	req.Header.Add("ACCEPT", "application/vnd.ipld.car")

     

       79
       79
       +
       	resp, err := http.DefaultClient.Do(req)

     

       80
       80
       +
       	if err != nil {

     

       81
       81
       +
       		return fmt.Errorf("get repo: %w", err)

     

       82
       82
       +
       	}

     

       83
       83
       +
       

     

       84
       84
       +
       	defer resp.Body.Close()

     

       85
       85
       +
       

     

       86
       86
       +
       	_, err = minioClient.PutObject(ctx, bucketName, "pds-repo", resp.Body, -1, minio.PutObjectOptions{})

     

       87
       87
       +
       	if err != nil {

     

       88
       88
       +
       		return fmt.Errorf("stream repo to bucket: %w", err)

     

       89
       89
       +
       	}

     

       90
       90
       +
       

     

       91
       91
       +
       	return nil

     

       92
       92
       +
       }

     

       93
       93
       +
       

     

       94
       94
       +
       func backupBlobs(ctx context.Context, minioClient *minio.Client, bucketName string) error {

     

       95
       95
       +
       	cids, err := getAllBlobCIDs(ctx)

     

       96
       96
       +
       	if err != nil {

     

       97
       97
       +
       		return fmt.Errorf("get all blob CIDs: %w", err)

     

       98
       98
       +
       	}

     

       99
       99
       +
       

     

       100
       100
       +
       	reader, writer := io.Pipe()

     

       101
       101
       +
       	defer reader.Close()

     

       102
       102
       +
       

     

       103
       103
       +
       	zipWriter := zip.NewWriter(writer)

     

       104
       104
       +
       

     

       105
       105
       +
       	go func() {

     

       106
       106
       +
       		defer writer.Close()

     

       107
       107
       +
       		defer zipWriter.Close()

     

       108
       108
       +
       

     

       109
       109
       +
       		for _, cid := range cids {

     

       110
       110
       +
       			slog.Info("processing cid", "cid", cid)

     

       111
       111
       +
       			blob, err := getBlob(ctx, cid)

     

       112
       112
       +
       			if err != nil {

     

       113
       113
       +
       				slog.Error("failed to get blob", "cid", cid, "error", err)

     

       114
       114
       +
       				continue

     

       115
       115
       +
       			}

     

       116
       116
       +
       

     

       117
       117
       +
       			zipFile, err := zipWriter.Create(cid)

     

       118
       118
       +
       			if err != nil {

     

       119
       119
       +
       				slog.Error("create new file in zipwriter", "cid", cid, "error", err)

     

       120
       120
       +
       				blob.Close()

     

       121
       121
       +
       				continue

     

       122
       122
       +
       			}

     

       123
       123
       +
       

     

       124
       124
       +
       			io.Copy(zipFile, blob)

     

       125
       125
       +
       			blob.Close()

     

       126
       126
       +
       		}

     

       127
       127
       +
       	}()

     

       128
       128
       +
       

     

       129
       129
       +
       	_, err = minioClient.PutObject(ctx, bucketName, "pds-blobs.zip", reader, -1, minio.PutObjectOptions{})

     

       130
       130
       +
       	if err != nil {

     

       131
       131
       +
       		return fmt.Errorf("stream blobs to bucket: %w", err)

     

       132
       132
       +
       	}

     

       133
       133
       +
       

     

       134
       134
       +
       	return nil

     

       135
       135
       +
       }

     

       136
       136
       +
       

     

       137
       137
       +
       func getAllBlobCIDs(ctx context.Context) ([]string, error) {

     

       138
       138
       +
       	cursor := ""

     

       139
       139
       +
       	limit := 100

     

       140
       140
       +
       	var cids []string

     

       141
       141
       +
       	for {

     

       142
       142
       +
       		res, err := listBlobs(ctx, cursor, int64(limit))

     

       143
       143
       +
       		if err != nil {

     

       144
       144
       +
       			return nil, fmt.Errorf("list blobs: %w", err)

     

       145
       145
       +
       		}

     

       146
       146
       +
       		if len(res.CIDs) == 0 {

     

       147
       147
       +
       			return cids, nil

     

       148
       148
       +
       		}

     

       149
       149
       +
       

     

       150
       150
       +
       		cids = append(cids, res.CIDs...)

     

       151
       151
       +
       

     

       152
       152
       +
       		if len(res.CIDs) < limit {

     

       153
       153
       +
       			return cids, nil

     

       154
       154
       +
       		}

     

       155
       155
       +
       

     

       156
       156
       +
       		cursor = res.Cursor

     

       157
       157
       +
       	}

     

       158
       158
       +
       }

     

       159
       159
       +
       

     

       160
       160
       +
       type listBlobsResponse struct {

     

       161
       161
       +
       	Cursor string   `json:"cursor"`

     

       162
       162
       +
       	CIDs   []string `json:"cids"`

     

       163
       163
       +
       }

     

       164
       164
       +
       

     

       165
       165
       +
       func listBlobs(ctx context.Context, cursor string, limit int64) (listBlobsResponse, error) {

     

       166
       166
       +
       	pdsHost := os.Getenv("PDS_HOST")

     

       167
       167
       +
       	did := os.Getenv("DID")

     

       168
       168
       +
       

     

       169
       169
       +
       	// TODO: do proper url encoding of query params

     

       170
       170
       +
       	url := fmt.Sprintf("%s/xrpc/com.atproto.sync.listBlobs?did=%s&cursor=%s&limit=%d", pdsHost, did, cursor, limit)

     

       171
       171
       +
       	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)

     

       172
       172
       +
       	if err != nil {

     

       173
       173
       +
       		return listBlobsResponse{}, fmt.Errorf("create list blobs request: %w", err)

     

       174
       174
       +
       	}

     

       175
       175
       +
       

     

       176
       176
       +
       	resp, err := http.DefaultClient.Do(req)

     

       177
       177
       +
       	if err != nil {

     

       178
       178
       +
       		return listBlobsResponse{}, fmt.Errorf("list blobs: %w", err)

     

       179
       179
       +
       	}

     

       180
       180
       +
       

     

       181
       181
       +
       	defer resp.Body.Close()

     

       182
       182
       +
       

     

       183
       183
       +
       	resBody, err := io.ReadAll(resp.Body)

     

       184
       184
       +
       	if err != nil {

     

       185
       185
       +
       		return listBlobsResponse{}, fmt.Errorf("failed to read response: %w", err)

     

       186
       186
       +
       	}

     

       187
       187
       +
       

     

       188
       188
       +
       	var result listBlobsResponse

     

       189
       189
       +
       	err = json.Unmarshal(resBody, &result)

     

       190
       190
       +
       	if err != nil {

     

       191
       191
       +
       		return listBlobsResponse{}, fmt.Errorf("failed to unmarshal response: %w", err)

     

       192
       192
       +
       	}

     

       193
       193
       +
       

     

       194
       194
       +
       	return result, nil

     

       195
       195
       +
       }

     

       196
       196
       +
       

     

       197
       197
       +
       func getBlob(ctx context.Context, cid string) (io.ReadCloser, error) {

     

       198
       198
       +
       	pdsHost := os.Getenv("PDS_HOST")

     

       199
       199
       +
       	did := os.Getenv("DID")

     

       200
       200
       +
       

     

       201
       201
       +
       	// TODO: do proper url encoding of query params

     

       202
       202
       +
       	url := fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", pdsHost, did, cid)

     

       203
       203
       +
       	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)

     

       204
       204
       +
       	if err != nil {

     

       205
       205
       +
       		return nil, fmt.Errorf("create get blob request: %w", err)

     

       206
       206
       +
       	}

     

       207
       207
       +
       

     

       208
       208
       +
       	resp, err := http.DefaultClient.Do(req)

     

       209
       209
       +
       	if err != nil {

     

       210
       210
       +
       		return nil, fmt.Errorf("get blob: %w", err)

     

       211
       211
       +
       	}

     

       212
       212
       +
       

     

       213
       213
       +
       	return resp.Body, nil

     

       214
       214
       +
       }

readme.md

···

       1
       1
       +
       ## Back AT it

     

       2
       2
       +
       

     

       3
       3
       +
       This is a tool I'm activly developing to back up my ATProtocol type things to S3 storage.

     

       4
       4
       +
       

     

       5
       5
       +
       At the moment it's a one shot style script that backs up the PDS repo and then the blobs but in the future I plan on being able to backup other things (next is my Tangled Knot data).

     

       6
       6
       +
       

     

       7
       7
       +
       The PDS repo data is pulled straight from the xrpc endpoint at sent straight to S3. The blob data however is streamed into a zip file and sent to S3 so that not all the data is held in memory while the backup takes place (the minio library will still keep some in memory as a multipart request).

     

       8
       8
       +
       

     

       9
       9
       +
       It's very hacky right now and needs polishing to use with caution. Although let's face it, the worst it can do at the moment it backup some bad data which is better than no data 🤪