commit 28876f4ef761606a6743531988c748f1d9837c20 · hailey.at/cocoon

+11 -6

cmd/cocoon/main.go

···

       107
        
       				Name:    "s3-backups-enabled",

     

       108
        
       				EnvVars: []string{"COCOON_S3_BACKUPS_ENABLED"},

     

       109
        
       			},

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       110
        
       			&cli.StringFlag{

     

       111
        
       				Name:    "s3-region",

     

       112
        
       				EnvVars: []string{"COCOON_S3_REGION"},

     
···

       181
        
       			SmtpEmail:       cmd.String("smtp-email"),

     

       182
        
       			SmtpName:        cmd.String("smtp-name"),

     

       183
        
       			S3Config: &server.S3Config{

     

       184
       -
       				BackupsEnabled: cmd.Bool("s3-backups-enabled"),

     

       185
       -
       				Region:         cmd.String("s3-region"),

     

       186
       -
       				Bucket:         cmd.String("s3-bucket"),

     

       187
       -
       				Endpoint:       cmd.String("s3-endpoint"),

     

       188
       -
       				AccessKey:      cmd.String("s3-access-key"),

     

       189
       -
       				SecretKey:      cmd.String("s3-secret-key"),

     

       0
        
       
     

       190
        
       			},

     

       191
        
       			SessionSecret:     cmd.String("session-secret"),

     

       192
        
       			BlockstoreVariant: server.MustReturnBlockstoreVariant(cmd.String("blockstore-variant")),

···

       107
        
       				Name:    "s3-backups-enabled",

     

       108
        
       				EnvVars: []string{"COCOON_S3_BACKUPS_ENABLED"},

     

       109
        
       			},

     

       110
       +
       			&cli.BoolFlag{

     

       111
       +
       				Name:    "s3-blobstore-enabled",

     

       112
       +
       				EnvVars: []string{"COCOON_S3_BLOBSTORE_ENABLED"},

     

       113
       +
       			},

     

       114
        
       			&cli.StringFlag{

     

       115
        
       				Name:    "s3-region",

     

       116
        
       				EnvVars: []string{"COCOON_S3_REGION"},

     
···

       185
        
       			SmtpEmail:       cmd.String("smtp-email"),

     

       186
        
       			SmtpName:        cmd.String("smtp-name"),

     

       187
        
       			S3Config: &server.S3Config{

     

       188
       +
       				BackupsEnabled:   cmd.Bool("s3-backups-enabled"),

     

       189
       +
       				BlobstoreEnabled: cmd.Bool("s3-blobstore-enabled"),

     

       190
       +
       				Region:           cmd.String("s3-region"),

     

       191
       +
       				Bucket:           cmd.String("s3-bucket"),

     

       192
       +
       				Endpoint:         cmd.String("s3-endpoint"),

     

       193
       +
       				AccessKey:        cmd.String("s3-access-key"),

     

       194
       +
       				SecretKey:        cmd.String("s3-secret-key"),

     

       195
        
       			},

     

       196
        
       			SessionSecret:     cmd.String("session-secret"),

     

       197
        
       			BlockstoreVariant: server.MustReturnBlockstoreVariant(cmd.String("blockstore-variant")),

models/models.go

···

       106
        
       	Did       string `gorm:"index;index:idx_blob_did_cid"`

     

       107
        
       	Cid       []byte `gorm:"index;index:idx_blob_did_cid"`

     

       108
        
       	RefCount  int

     

       0
        
       
     

       109
        
       }

     

       110
        
       

     

       111
        
       type BlobPart struct {

···

       106
        
       	Did       string `gorm:"index;index:idx_blob_did_cid"`

     

       107
        
       	Cid       []byte `gorm:"index;index:idx_blob_did_cid"`

     

       108
        
       	RefCount  int

     

       109
       +
       	Storage   string `gorm:"default:sqlite"`

     

       110
        
       }

     

       111
        
       

     

       112
        
       type BlobPart struct {

+50 -8

server/handle_repo_upload_blob.go

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"bytes"

     

       0
        
       
     

       5
        
       	"io"

     

       6
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       7
        
       	"github.com/haileyok/cocoon/internal/helpers"

     

       8
        
       	"github.com/haileyok/cocoon/models"

     

       9
        
       	"github.com/ipfs/go-cid"

     
···

       34
        
       		mime = "application/octet-stream"

     

       35
        
       	}

     

       36
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       37
        
       	blob := models.Blob{

     

       38
        
       		Did:       urepo.Repo.Did,

     

       39
        
       		RefCount:  0,

     

       40
        
       		CreatedAt: s.repoman.clock.Next().String(),

     

       0
        
       
     

       41
        
       	}

     

       42
        
       

     

       43
        
       	if err := s.db.Create(&blob, nil).Error; err != nil {

     
···

       66
        
       		read += n

     

       67
        
       		fulldata.Write(data)

     

       68
        
       

     

       69
       -
       		blobPart := models.BlobPart{

     

       70
       -
       			BlobID: blob.ID,

     

       71
       -
       			Idx:    part,

     

       72
       -
       			Data:   data,

     

       73
       -
       		}

     

       0
        
       
     

       74
        
       

     

       75
       -
       		if err := s.db.Create(&blobPart, nil).Error; err != nil {

     

       76
       -
       			s.logger.Error("error adding blob part to db", "error", err)

     

       77
       -
       			return helpers.ServerError(e, nil)

     

       0
        
       
     

       78
        
       		}

     

       79
        
       		part++

     

       80
        
       

     
···

       87
        
       	if err != nil {

     

       88
        
       		s.logger.Error("error creating cid prefix", "error", err)

     

       89
        
       		return helpers.ServerError(e, nil)

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       90
        
       	}

     

       91
        
       

     

       92
        
       	if err := s.db.Exec("UPDATE blobs SET cid = ? WHERE id = ?", nil, c.Bytes(), blob.ID).Error; err != nil {

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"bytes"

     

       5
       +
       	"fmt"

     

       6
        
       	"io"

     

       7
        
       

     

       8
       +
       	"github.com/aws/aws-sdk-go/aws"

     

       9
       +
       	"github.com/aws/aws-sdk-go/aws/credentials"

     

       10
       +
       	"github.com/aws/aws-sdk-go/aws/session"

     

       11
       +
       	"github.com/aws/aws-sdk-go/service/s3"

     

       12
        
       	"github.com/haileyok/cocoon/internal/helpers"

     

       13
        
       	"github.com/haileyok/cocoon/models"

     

       14
        
       	"github.com/ipfs/go-cid"

     
···

       39
        
       		mime = "application/octet-stream"

     

       40
        
       	}

     

       41
        
       

     

       42
       +
       	storage := "sqlite"

     

       43
       +
       	s3Upload := s.s3Config != nil && s.s3Config.BlobstoreEnabled

     

       44
       +
       	if s3Upload {

     

       45
       +
       		storage = "s3"

     

       46
       +
       	}

     

       47
        
       	blob := models.Blob{

     

       48
        
       		Did:       urepo.Repo.Did,

     

       49
        
       		RefCount:  0,

     

       50
        
       		CreatedAt: s.repoman.clock.Next().String(),

     

       51
       +
       		Storage:   storage,

     

       52
        
       	}

     

       53
        
       

     

       54
        
       	if err := s.db.Create(&blob, nil).Error; err != nil {

     
···

       77
        
       		read += n

     

       78
        
       		fulldata.Write(data)

     

       79
        
       

     

       80
       +
       		if !s3Upload {

     

       81
       +
       			blobPart := models.BlobPart{

     

       82
       +
       				BlobID: blob.ID,

     

       83
       +
       				Idx:    part,

     

       84
       +
       				Data:   data,

     

       85
       +
       			}

     

       86
        
       

     

       87
       +
       			if err := s.db.Create(&blobPart, nil).Error; err != nil {

     

       88
       +
       				s.logger.Error("error adding blob part to db", "error", err)

     

       89
       +
       				return helpers.ServerError(e, nil)

     

       90
       +
       			}

     

       91
        
       		}

     

       92
        
       		part++

     

       93
        
       

     
···

       100
        
       	if err != nil {

     

       101
        
       		s.logger.Error("error creating cid prefix", "error", err)

     

       102
        
       		return helpers.ServerError(e, nil)

     

       103
       +
       	}

     

       104
       +
       

     

       105
       +
       	if s3Upload {

     

       106
       +
       		config := &aws.Config{

     

       107
       +
       			Region:      aws.String(s.s3Config.Region),

     

       108
       +
       			Credentials: credentials.NewStaticCredentials(s.s3Config.AccessKey, s.s3Config.SecretKey, ""),

     

       109
       +
       		}

     

       110
       +
       

     

       111
       +
       		if s.s3Config.Endpoint != "" {

     

       112
       +
       			config.Endpoint = aws.String(s.s3Config.Endpoint)

     

       113
       +
       			config.S3ForcePathStyle = aws.Bool(true)

     

       114
       +
       		}

     

       115
       +
       

     

       116
       +
       		sess, err := session.NewSession(config)

     

       117
       +
       		if err != nil {

     

       118
       +
       			s.logger.Error("error creating aws session", "error", err)

     

       119
       +
       			return helpers.ServerError(e, nil)

     

       120
       +
       		}

     

       121
       +
       

     

       122
       +
       		svc := s3.New(sess)

     

       123
       +
       

     

       124
       +
       		if _, err := svc.PutObject(&s3.PutObjectInput{

     

       125
       +
       			Bucket: aws.String(s.s3Config.Bucket),

     

       126
       +
       			Key:    aws.String(fmt.Sprintf("blobs/%s/%s", urepo.Repo.Did, c.String())),

     

       127
       +
       			Body:   bytes.NewReader(fulldata.Bytes()),

     

       128
       +
       		}); err != nil {

     

       129
       +
       			s.logger.Error("error uploading blob to s3", "error", err)

     

       130
       +
       			return helpers.ServerError(e, nil)

     

       131
       +
       		}

     

       132
        
       	}

     

       133
        
       

     

       134
        
       	if err := s.db.Exec("UPDATE blobs SET cid = ? WHERE id = ?", nil, c.Bytes(), blob.ID).Error; err != nil {

+70 -8

server/handle_sync_get_blob.go

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"bytes"

     

       0
        
       
     

       0
        
       
     

       5
        
       

     

       6
        
       	"github.com/Azure/go-autorest/autorest/to"

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       7
        
       	"github.com/haileyok/cocoon/internal/helpers"

     

       8
        
       	"github.com/haileyok/cocoon/models"

     

       9
        
       	"github.com/ipfs/go-cid"

     
···

       47
        
       

     

       48
        
       	buf := new(bytes.Buffer)

     

       49
        
       

     

       50
       -
       	var parts []models.BlobPart

     

       51
       -
       	if err := s.db.Raw("SELECT * FROM blob_parts WHERE blob_id = ? ORDER BY idx", nil, blob.ID).Scan(&parts).Error; err != nil {

     

       52
       -
       		s.logger.Error("error getting blob parts", "error", err)

     

       53
       -
       		return helpers.ServerError(e, nil)

     

       54
       -
       	}

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       55
        
       

     

       56
       -
       	// TODO: we can just stream this, don't need to make a buffer

     

       57
       -
       	for _, p := range parts {

     

       58
       -
       		buf.Write(p.Data)

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       59
        
       	}

     

       60
        
       

     

       61
        
       	e.Response().Header().Set(echo.HeaderContentDisposition, "attachment; filename="+c.String())

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"bytes"

     

       5
       +
       	"fmt"

     

       6
       +
       	"io"

     

       7
        
       

     

       8
        
       	"github.com/Azure/go-autorest/autorest/to"

     

       9
       +
       	"github.com/aws/aws-sdk-go/aws"

     

       10
       +
       	"github.com/aws/aws-sdk-go/aws/credentials"

     

       11
       +
       	"github.com/aws/aws-sdk-go/aws/session"

     

       12
       +
       	"github.com/aws/aws-sdk-go/service/s3"

     

       13
        
       	"github.com/haileyok/cocoon/internal/helpers"

     

       14
        
       	"github.com/haileyok/cocoon/models"

     

       15
        
       	"github.com/ipfs/go-cid"

     
···

       53
        
       

     

       54
        
       	buf := new(bytes.Buffer)

     

       55
        
       

     

       56
       +
       	if blob.Storage == "sqlite" {

     

       57
       +
       		var parts []models.BlobPart

     

       58
       +
       		if err := s.db.Raw("SELECT * FROM blob_parts WHERE blob_id = ? ORDER BY idx", nil, blob.ID).Scan(&parts).Error; err != nil {

     

       59
       +
       			s.logger.Error("error getting blob parts", "error", err)

     

       60
       +
       			return helpers.ServerError(e, nil)

     

       61
       +
       		}

     

       62
       +
       

     

       63
       +
       		// TODO: we can just stream this, don't need to make a buffer

     

       64
       +
       		for _, p := range parts {

     

       65
       +
       			buf.Write(p.Data)

     

       66
       +
       		}

     

       67
       +
       	} else if blob.Storage == "s3" {

     

       68
       +
       		if  !(s.s3Config != nil && s.s3Config.BlobstoreEnabled) {

     

       69
       +
       			s.logger.Error("s3 storage disabled")

     

       70
       +
       			return helpers.ServerError(e, nil)

     

       71
       +
       		}

     

       72
       +
       

     

       73
       +
       		config := &aws.Config{

     

       74
       +
       			Region:      aws.String(s.s3Config.Region),

     

       75
       +
       			Credentials: credentials.NewStaticCredentials(s.s3Config.AccessKey, s.s3Config.SecretKey, ""),

     

       76
       +
       		}

     

       77
        
       

     

       78
       +
       		if s.s3Config.Endpoint != "" {

     

       79
       +
       			config.Endpoint = aws.String(s.s3Config.Endpoint)

     

       80
       +
       			config.S3ForcePathStyle = aws.Bool(true)

     

       81
       +
       		}

     

       82
       +
       

     

       83
       +
       		sess, err := session.NewSession(config)

     

       84
       +
       		if err != nil {

     

       85
       +
       			s.logger.Error("error creating aws session", "error", err)

     

       86
       +
       			return helpers.ServerError(e, nil)

     

       87
       +
       		}

     

       88
       +
       

     

       89
       +
       		svc := s3.New(sess)

     

       90
       +
       		if result, err := svc.GetObject(&s3.GetObjectInput{

     

       91
       +
       			Bucket: aws.String(s.s3Config.Bucket),

     

       92
       +
       			Key:    aws.String(fmt.Sprintf("blobs/%s/%s", urepo.Repo.Did, c.String())),

     

       93
       +
       		}); err != nil {

     

       94
       +
       			s.logger.Error("error getting blob from s3", "error", err)

     

       95
       +
       			return helpers.ServerError(e, nil)

     

       96
       +
       		} else {

     

       97
       +
       			read := 0

     

       98
       +
       			part := 0

     

       99
       +
       			partBuf := make([]byte, 0x10000)

     

       100
       +
       

     

       101
       +
       			for {

     

       102
       +
       				n, err := io.ReadFull(result.Body, partBuf)

     

       103
       +
       				if err == io.ErrUnexpectedEOF || err == io.EOF {

     

       104
       +
       					if n == 0 {

     

       105
       +
       						break

     

       106
       +
       					}

     

       107
       +
       				} else if err != nil && err != io.ErrUnexpectedEOF {

     

       108
       +
       					s.logger.Error("error reading blob", "error", err)

     

       109
       +
       					return helpers.ServerError(e, nil)

     

       110
       +
       				}

     

       111
       +
       

     

       112
       +
       				data := partBuf[:n]

     

       113
       +
       				read += n

     

       114
       +
       				buf.Write(data)

     

       115
       +
       				part++

     

       116
       +
       			}

     

       117
       +
       		}

     

       118
       +
       	} else {

     

       119
       +
       		s.logger.Error("unknown storage", "storage", blob.Storage)

     

       120
       +
       		return helpers.ServerError(e, nil)

     

       121
        
       	}

     

       122
        
       

     

       123
        
       	e.Response().Header().Set(echo.HeaderContentDisposition, "attachment; filename="+c.String())

+7 -6

server/server.go

···

       52
        
       )

     

       53
        
       

     

       54
        
       type S3Config struct {

     

       55
       -
       	BackupsEnabled bool

     

       56
       -
       	Endpoint       string

     

       57
       -
       	Region         string

     

       58
       -
       	Bucket         string

     

       59
       -
       	AccessKey      string

     

       60
       -
       	SecretKey      string

     

       0
        
       
     

       61
        
       }

     

       62
        
       

     

       63
        
       type Server struct {

···

       52
        
       )

     

       53
        
       

     

       54
        
       type S3Config struct {

     

       55
       +
       	BackupsEnabled   bool

     

       56
       +
       	BlobstoreEnabled bool

     

       57
       +
       	Endpoint         string

     

       58
       +
       	Region           string

     

       59
       +
       	Bucket           string

     

       60
       +
       	AccessKey        string

     

       61
       +
       	SecretKey        string

     

       62
        
       }

     

       63
        
       

     

       64
        
       type Server struct {