commit 061ab8935c5420234b298c007cd5189addc41a4a · hailey.at/photocopy

+1 -10

consumer.go

···

       7
        
       	"net/http"

     

       8
        
       	"net/url"

     

       9
        
       	"os"

     

       10
       -
       	"strings"

     

       11
        
       	"time"

     

       12
        
       

     

       13
        
       	"github.com/bluesky-social/indigo/api/atproto"

     
···

       136
        
       				continue

     

       137
        
       			}

     

       138
        
       

     

       139
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       140
       -
       				continue

     

       141
       -
       			}

     

       142
       -
       

     

       143
       -
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String()); err != nil {

     

       144
        
       				p.logger.Error("error handling create event", "error", err)

     

       145
        
       				continue

     

       146
        
       			}

     

       147
        
       		case repomgr.EvtKindDeleteRecord:

     

       148
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       149
       -
       				continue

     

       150
       -
       			}

     

       151
       -
       

     

       152
        
       			if err := p.handleDelete(ctx, did.String(), collection.String(), rkey.String()); err != nil {

     

       153
        
       				p.logger.Error("error handling delete event", "error", err)

     

       154
        
       				continue

···

       7
        
       	"net/http"

     

       8
        
       	"net/url"

     

       9
        
       	"os"

     

       0
        
       
     

       10
        
       	"time"

     

       11
        
       

     

       12
        
       	"github.com/bluesky-social/indigo/api/atproto"

     
···

       135
        
       				continue

     

       136
        
       			}

     

       137
        
       

     

       138
       +
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String(), fmt.Sprintf("%d", evt.Seq)); err != nil {

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       139
        
       				p.logger.Error("error handling create event", "error", err)

     

       140
        
       				continue

     

       141
        
       			}

     

       142
        
       		case repomgr.EvtKindDeleteRecord:

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       143
        
       			if err := p.handleDelete(ctx, did.String(), collection.String(), rkey.String()); err != nil {

     

       144
        
       				p.logger.Error("error handling delete event", "error", err)

     

       145
        
       				continue

+31 -2

handle_create.go

···

       13
        
       	"github.com/haileyok/photocopy/models"

     

       14
        
       )

     

       15
        
       

     

       16
       -
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid string) error {

     

       17
       -
       

     

       18
        
       	iat, err := dateparse.ParseAny(indexedAt)

     

       19
        
       	if err != nil {

     

       20
        
       		return err

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       21
        
       	}

     

       22
        
       

     

       23
        
       	switch collection {

     
···

       30
        
       	default:

     

       31
        
       		return nil

     

       32
        
       	}

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       33
        
       }

     

       34
        
       

     

       35
        
       func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

···

       13
        
       	"github.com/haileyok/photocopy/models"

     

       14
        
       )

     

       15
        
       

     

       16
       +
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid, seq string) error {

     

       0
        
       
     

       17
        
       	iat, err := dateparse.ParseAny(indexedAt)

     

       18
        
       	if err != nil {

     

       19
        
       		return err

     

       20
       +
       	}

     

       21
       +
       

     

       22
       +
       	if err := p.handleCreateRecord(ctx, did, rkey, collection, cid, recb, seq); err != nil {

     

       23
       +
       		p.logger.Error("error creating record", "error", err)

     

       24
        
       	}

     

       25
        
       

     

       26
        
       	switch collection {

     
···

       33
        
       	default:

     

       34
        
       		return nil

     

       35
        
       	}

     

       36
       +
       }

     

       37
       +
       

     

       38
       +
       func (p *Photocopy) handleCreateRecord(ctx context.Context, did, rkey, collection, cid string, raw []byte, seq string) error {

     

       39
       +
       	var cat time.Time

     

       40
       +
       	prkey, err := syntax.ParseTID(rkey)

     

       41
       +
       	if err == nil {

     

       42
       +
       		cat = prkey.Time()

     

       43
       +
       	} else {

     

       44
       +
       		cat = time.Now()

     

       45
       +
       	}

     

       46
       +
       

     

       47
       +
       	rec := models.Record{

     

       48
       +
       		Did:        did,

     

       49
       +
       		Rkey:       rkey,

     

       50
       +
       		Collection: collection,

     

       51
       +
       		Cid:        cid,

     

       52
       +
       		Seq:        seq,

     

       53
       +
       		Raw:        raw,

     

       54
       +
       		CreatedAt:  cat,

     

       55
       +
       	}

     

       56
       +
       

     

       57
       +
       	if err := p.inserters.recordsInserter.Insert(ctx, rec); err != nil {

     

       58
       +
       		return err

     

       59
       +
       	}

     

       60
       +
       

     

       61
       +
       	return nil

     

       62
        
       }

     

       63
        
       

     

       64
        
       func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

+13

handle_delete.go

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"context"

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       5
        
       )

     

       6
        
       

     

       7
        
       func (p *Photocopy) handleDelete(ctx context.Context, did, collection, rkey string) error {

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       8
        
       	return nil

     

       9
        
       }

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"context"

     

       5
       +
       	"time"

     

       6
       +
       

     

       7
       +
       	"github.com/haileyok/photocopy/models"

     

       8
        
       )

     

       9
        
       

     

       10
        
       func (p *Photocopy) handleDelete(ctx context.Context, did, collection, rkey string) error {

     

       11
       +
       	del := models.Delete{

     

       12
       +
       		Did:       did,

     

       13
       +
       		Rkey:      rkey,

     

       14
       +
       		CreatedAt: time.Now(),

     

       15
       +
       	}

     

       16
       +
       

     

       17
       +
       	if err := p.inserters.deletesInserter.Insert(ctx, del); err != nil {

     

       18
       +
       		return err

     

       19
       +
       	}

     

       20
       +
       

     

       21
        
       	return nil

     

       22
        
       }

-1

models.go

···

       1
       -
       package photocopy

···

       0

+9

models/delete.go

···

       1
       +
       package models

     

       2
       +
       

     

       3
       +
       import "time"

     

       4
       +
       

     

       5
       +
       type Delete struct {

     

       6
       +
       	Did       string    `ch:"did"`

     

       7
       +
       	Rkey      string    `ch:"rkey"`

     

       8
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       9
       +
       }

+13

models/record.go

···

       1
       +
       package models

     

       2
       +
       

     

       3
       +
       import "time"

     

       4
       +
       

     

       5
       +
       type Record struct {

     

       6
       +
       	Did        string `ch:"did"`

     

       7
       +
       	Rkey       string `ch:"rkey"`

     

       8
       +
       	Collection string `ch:"collection"`

     

       9
       +
       	Cid        string `ch:"cid"`

     

       10
       +
       	Seq        string `ch:"seq"`

     

       11
       +
       	Raw        []byte `ch:"raw"`

     

       12
       +
       	CreatedAt  time.Time

     

       13
       +
       }

+28

photocopy.go

···

       34
        
       	interactionsInserter *clickhouse_inserter.Inserter

     

       35
        
       	postsInserter        *clickhouse_inserter.Inserter

     

       36
        
       	plcInserter          *clickhouse_inserter.Inserter

     

       0
        
       
     

       0
        
       
     

       37
        
       }

     

       38
        
       

     

       39
        
       type Args struct {

     
···

       111
        
       		return nil, err

     

       112
        
       	}

     

       113
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       114
        
       	is := &Inserters{

     

       115
        
       		followsInserter:      fi,

     

       116
        
       		postsInserter:        pi,

     

       117
        
       		interactionsInserter: ii,

     

       0
        
       
     

       0
        
       
     

       118
        
       	}

     

       119
        
       

     

       120
        
       	p.inserters = is

···

       34
        
       	interactionsInserter *clickhouse_inserter.Inserter

     

       35
        
       	postsInserter        *clickhouse_inserter.Inserter

     

       36
        
       	plcInserter          *clickhouse_inserter.Inserter

     

       37
       +
       	recordsInserter      *clickhouse_inserter.Inserter

     

       38
       +
       	deletesInserter      *clickhouse_inserter.Inserter

     

       39
        
       }

     

       40
        
       

     

       41
        
       type Args struct {

     
···

       113
        
       		return nil, err

     

       114
        
       	}

     

       115
        
       

     

       116
       +
       	ri, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       117
       +
       		PrometheusCounterPrefix: "photocopy_records",

     

       118
       +
       		Histogram:               insertionsHist,

     

       119
       +
       		BatchSize:               1000,

     

       120
       +
       		Logger:                  p.logger,

     

       121
       +
       		Conn:                    conn,

     

       122
       +
       		Query:                   "INSERT INTO records (did, rkey, collection, cid, seq, raw, created_at)",

     

       123
       +
       	})

     

       124
       +
       	if err != nil {

     

       125
       +
       		return nil, err

     

       126
       +
       	}

     

       127
       +
       

     

       128
       +
       	di, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       129
       +
       		PrometheusCounterPrefix: "photocopy_records",

     

       130
       +
       		Histogram:               insertionsHist,

     

       131
       +
       		BatchSize:               1000,

     

       132
       +
       		Logger:                  p.logger,

     

       133
       +
       		Conn:                    conn,

     

       134
       +
       		Query:                   "INSERT INTO deletes (did, rkey, created_at)",

     

       135
       +
       	})

     

       136
       +
       	if err != nil {

     

       137
       +
       		return nil, err

     

       138
       +
       	}

     

       139
       +
       

     

       140
        
       	is := &Inserters{

     

       141
        
       		followsInserter:      fi,

     

       142
        
       		postsInserter:        pi,

     

       143
        
       		interactionsInserter: ii,

     

       144
       +
       		recordsInserter:      ri,

     

       145
       +
       		deletesInserter:      di,

     

       146
        
       	}

     

       147
        
       

     

       148
        
       	p.inserters = is