commit 74d723a7511c1536933e2d40d1de0dafbf47ac90 · willdot.net/back-at-it

+4 -4

.env.example

···

       1
       1
       -
       ENDPOINT="S3-endpoint"

     

       2
       2
       -
       ACCESS_ID="S3-ID"

     

       3
       3
       -
       SECRET_ACCESS_KEY="S3-secret"

     

       4
       4
       -
       BUCKET_NAME="my-super-duper-bucket"

     

       1
       1
       +
       S3_ENDPOINT="S3-endpoint"

     

       2
       2
       +
       S3_ACCESS_ID="S3-ID"

     

       3
       3
       +
       S3_SECRET_ACCESS_KEY="S3-secret"

     

       4
       4
       +
       S3_BUCKET_NAME="my-super-duper-bucket"

     

       5
       5
        
       DID="the-did-to-backup"

     

       6
       6
        
       PDS_HOST="https://your-pds.com"

     

       7
       7
        
       TANGLED_KNOT_DATABASE_DIRECTORY="/path/to/database/directory"

+4 -4

main.go

···

       31
       31
        
       		return

     

       32
       32
        
       	}

     

       33
       33
        
       

     

       34
       34
       -
       	bucketName := os.Getenv("BUCKET_NAME")

     

       34
       34
       +
       	bucketName := os.Getenv("S3_BUCKET_NAME")

     

       35
       35
        
       

     

       36
       36
        
       	err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})

     

       37
       37
        
       	if err != nil {

     
···

       45
       45
        
       }

     

       46
       46
        
       

     

       47
       47
        
       func createMinioClient() (*minio.Client, error) {

     

       48
       48
       -
       	endpoint := os.Getenv("ENDPOINT")

     

       49
       49
       -
       	accessKeyID := os.Getenv("ACCESS_ID")

     

       50
       50
       -
       	secretAccessKey := os.Getenv("SECRET_ACCESS_KEY")

     

       48
       48
       +
       	endpoint := os.Getenv("S3_ENDPOINT")

     

       49
       49
       +
       	accessKeyID := os.Getenv("S3_ACCESS_ID")

     

       50
       50
       +
       	secretAccessKey := os.Getenv("S3_SECRET_ACCESS_KEY")

     

       51
       51
        
       	useSSL := true

     

       52
       52
        
       

     

       53
       53
        
       	return minio.New(endpoint, &minio.Options{

+21 -1

readme.md

···

       2
       2
        
       

     

       3
       3
        
       This is a tool I'm activly developing to back up my ATProtocol type things to S3 storage.

     

       4
       4
        
       

     

       5
       5
       -
       At the moment it's a one shot style script that backs up the PDS repo and then the blobs but in the future I plan on being able to backup other things (next is my Tangled Knot data).

     

       5
       5
       +
       Currently there are 2 things that can be backed up:

     

       6
       6
       +
       

     

       7
       7
       +
       1: PDS data - A users repo and their blobs

     

       8
       8
       +
       2: Tangled Knot data - The repositories directory that contains all of the repo data and the directory that contains the SQLite database

     

       6
       9
        
       

     

       7
       10
        
       The PDS repo data is pulled straight from the xrpc endpoint at sent straight to S3. The blob data however is streamed into a zip file and sent to S3 so that not all the data is held in memory while the backup takes place (the minio library will still keep some in memory as a multipart request).

     

       8
       11
        
       

     

       9
       12
        
       It's very hacky right now and needs polishing to use with caution. Although let's face it, the worst it can do at the moment it backup some bad data which is better than no data 🤪

     

       13
       13
       +
       

     

       14
       14
       +
       

     

       15
       15
       +
       ### How to use

     

       16
       16
       +
       

     

       17
       17
       +
       Clone the repo and copy the `.env.example` file to be `.env`. Fill in the `.env` file with you S3 variables.

     

       18
       18
       +
       

     

       19
       19
       +
       For PDS data backup you need to ensure that `DID` and `PDS_HOST` are populated.

     

       20
       20
       +
       

     

       21
       21
       +
       For Knot data backup you need to ensure that `TANGLED_KNOT_DATABASE_DIRECTORY` and `TANGLED_KNOT_REPOSITORY_DIRECTORY` are populated.

     

       22
       22
       +
       

     

       23
       23
       +
       Run `go run .` 

     

       24
       24
       +
       

     

       25
       25
       +
       ### Todo

     

       26
       26
       +
       

     

       27
       27
       +
       - [ ] - Turn this into a long running app using a cron library perhaps

     

       28
       28
       +
       - [ ] - User query params properly when creating the URLs to fetch repo and blobs

     

       29
       29
       +
       - [ ] - Allow configuring the backup of knot repo data per users DID maybe?