comparing afa03f2edfb83d14b71279e5330d51b6053143a2 and main on hailey.at/photocopy

Makefile

···

       13
       13
        
       .PHONY: build

     

       14
       14
        
       build: ## Build all executables

     

       15
       15
        
       	go build -ldflags "-X main.Version=$(VERSION)" -o photocopy ./cmd/photocopy

     

       16
       16
       +
       	go build -o bodega ./cmd/bodega

     

       16
       17
        
       

     

       17
       18
        
       .PHONY: run

     

       18
       19
        
       run:

+445

backfiller.go

···

       1
       1
       +
       package photocopy

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"bytes"

     

       5
       5
       +
       	"context"

     

       6
       6
       +
       	"encoding/json"

     

       7
       7
       +
       	"fmt"

     

       8
       8
       +
       	"io"

     

       9
       9
       +
       	"net/http"

     

       10
       10
       +
       	"runtime"

     

       11
       11
       +
       	"strings"

     

       12
       12
       +
       	"sync"

     

       13
       13
       +
       	"time"

     

       14
       14
       +
       

     

       15
       15
       +
       	atproto_repo "github.com/bluesky-social/indigo/atproto/repo"

     

       16
       16
       +
       	"github.com/bluesky-social/indigo/repo"

     

       17
       17
       +
       	"github.com/bluesky-social/indigo/util"

     

       18
       18
       +
       	"github.com/ipfs/go-cid"

     

       19
       19
       +
       	"github.com/ipld/go-car"

     

       20
       20
       +
       	_ "github.com/joho/godotenv/autoload"

     

       21
       21
       +
       	"go.uber.org/ratelimit"

     

       22
       22
       +
       )

     

       23
       23
       +
       

     

       24
       24
       +
       type ProcessJob struct {

     

       25
       25
       +
       	did       string

     

       26
       26
       +
       	repoBytes []byte

     

       27
       27
       +
       }

     

       28
       28
       +
       

     

       29
       29
       +
       type RepoDownloader struct {

     

       30
       30
       +
       	clients     map[string]*http.Client

     

       31
       31
       +
       	rateLimits  map[string]ratelimit.Limiter

     

       32
       32
       +
       	processChan chan ProcessJob

     

       33
       33
       +
       	mu          sync.RWMutex

     

       34
       34
       +
       	p           *Photocopy

     

       35
       35
       +
       }

     

       36
       36
       +
       

     

       37
       37
       +
       func NewRepoDownloader(p *Photocopy) *RepoDownloader {

     

       38
       38
       +
       	return &RepoDownloader{

     

       39
       39
       +
       		clients:     make(map[string]*http.Client),

     

       40
       40
       +
       		rateLimits:  make(map[string]ratelimit.Limiter),

     

       41
       41
       +
       		p:           p,

     

       42
       42
       +
       		processChan: make(chan ProcessJob, 1000),

     

       43
       43
       +
       	}

     

       44
       44
       +
       }

     

       45
       45
       +
       

     

       46
       46
       +
       func (rd *RepoDownloader) getClient(service string) *http.Client {

     

       47
       47
       +
       	rd.mu.RLock()

     

       48
       48
       +
       	client, exists := rd.clients[service]

     

       49
       49
       +
       	rd.mu.RUnlock()

     

       50
       50
       +
       

     

       51
       51
       +
       	if exists {

     

       52
       52
       +
       		return client

     

       53
       53
       +
       	}

     

       54
       54
       +
       

     

       55
       55
       +
       	rd.mu.Lock()

     

       56
       56
       +
       	defer rd.mu.Unlock()

     

       57
       57
       +
       

     

       58
       58
       +
       	if client, exists := rd.clients[service]; exists {

     

       59
       59
       +
       		return client

     

       60
       60
       +
       	}

     

       61
       61
       +
       

     

       62
       62
       +
       	client = util.RobustHTTPClient()

     

       63
       63
       +
       	client.Timeout = 45 * time.Second

     

       64
       64
       +
       	rd.clients[service] = client

     

       65
       65
       +
       	return client

     

       66
       66
       +
       }

     

       67
       67
       +
       

     

       68
       68
       +
       func (rd *RepoDownloader) getRateLimiter(service string) ratelimit.Limiter {

     

       69
       69
       +
       	if !strings.HasSuffix(service, ".bsky.network") {

     

       70
       70
       +
       		service = "third-party"

     

       71
       71
       +
       	}

     

       72
       72
       +
       

     

       73
       73
       +
       	rd.mu.RLock()

     

       74
       74
       +
       	limiter, exists := rd.rateLimits[service]

     

       75
       75
       +
       	rd.mu.RUnlock()

     

       76
       76
       +
       

     

       77
       77
       +
       	if exists {

     

       78
       78
       +
       		return limiter

     

       79
       79
       +
       	}

     

       80
       80
       +
       

     

       81
       81
       +
       	rd.mu.Lock()

     

       82
       82
       +
       	defer rd.mu.Unlock()

     

       83
       83
       +
       

     

       84
       84
       +
       	if limiter, exists := rd.rateLimits[service]; exists {

     

       85
       85
       +
       		return limiter

     

       86
       86
       +
       	}

     

       87
       87
       +
       

     

       88
       88
       +
       	// 3000 per five minutes

     

       89
       89
       +
       	limiter = ratelimit.New(10)

     

       90
       90
       +
       	rd.rateLimits[service] = limiter

     

       91
       91
       +
       	return limiter

     

       92
       92
       +
       }

     

       93
       93
       +
       

     

       94
       94
       +
       func (rd *RepoDownloader) downloadRepo(service, did string) ([]byte, error) {

     

       95
       95
       +
       	dlurl := fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo?did=%s", service, did)

     

       96
       96
       +
       

     

       97
       97
       +
       	req, err := http.NewRequestWithContext(context.TODO(), "GET", dlurl, nil)

     

       98
       98
       +
       	if err != nil {

     

       99
       99
       +
       		return nil, fmt.Errorf("failed to create request: %w", err)

     

       100
       100
       +
       	}

     

       101
       101
       +
       

     

       102
       102
       +
       	if rd.p.ratelimitBypassKey != "" && strings.HasSuffix(service, ".bsky.network") {

     

       103
       103
       +
       		req.Header.Set("x-ratelimit-bypass", rd.p.ratelimitBypassKey)

     

       104
       104
       +
       	}

     

       105
       105
       +
       

     

       106
       106
       +
       	client := rd.getClient(service)

     

       107
       107
       +
       

     

       108
       108
       +
       	resp, err := client.Do(req)

     

       109
       109
       +
       	if err != nil {

     

       110
       110
       +
       		return nil, fmt.Errorf("failed to download repo: %w", err)

     

       111
       111
       +
       	}

     

       112
       112
       +
       	defer resp.Body.Close()

     

       113
       113
       +
       

     

       114
       114
       +
       	if resp.StatusCode != http.StatusOK {

     

       115
       115
       +
       		if resp.StatusCode == 400 {

     

       116
       116
       +
       			return nil, nil

     

       117
       117
       +
       		}

     

       118
       118
       +
       		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)

     

       119
       119
       +
       	}

     

       120
       120
       +
       

     

       121
       121
       +
       	b, err := io.ReadAll(resp.Body)

     

       122
       122
       +
       	if err != nil {

     

       123
       123
       +
       		return nil, fmt.Errorf("could not read bytes from response: %w", err)

     

       124
       124
       +
       	}

     

       125
       125
       +
       

     

       126
       126
       +
       	return b, nil

     

       127
       127
       +
       }

     

       128
       128
       +
       

     

       129
       129
       +
       func (p *Photocopy) processRepo(ctx context.Context, b []byte, did string) error {

     

       130
       130
       +
       	bs := atproto_repo.NewTinyBlockstore()

     

       131
       131
       +
       	cs, err := car.NewCarReader(bytes.NewReader(b))

     

       132
       132
       +
       	if err != nil {

     

       133
       133
       +
       		fmt.Println("error opening car", err)

     

       134
       134
       +
       		return fmt.Errorf("error opening car: %v\n", err)

     

       135
       135
       +
       	}

     

       136
       136
       +
       

     

       137
       137
       +
       	currBlock, _ := cs.Next()

     

       138
       138
       +
       	for currBlock != nil {

     

       139
       139
       +
       		bs.Put(context.TODO(), currBlock)

     

       140
       140
       +
       		next, _ := cs.Next()

     

       141
       141
       +
       		currBlock = next

     

       142
       142
       +
       	}

     

       143
       143
       +
       

     

       144
       144
       +
       	r, err := repo.OpenRepo(context.TODO(), bs, cs.Header.Roots[0])

     

       145
       145
       +
       	if err != nil || r == nil {

     

       146
       146
       +
       		fmt.Println("error opening repo", err)

     

       147
       147
       +
       		fmt.Printf("could not open repo: %v", err)

     

       148
       148
       +
       		return nil

     

       149
       149
       +
       	}

     

       150
       150
       +
       

     

       151
       151
       +
       	if err := r.ForEach(context.TODO(), "", func(key string, cid cid.Cid) error {

     

       152
       152
       +
       		pts := strings.Split(key, "/")

     

       153
       153
       +
       		nsid := pts[0]

     

       154
       154
       +
       		rkey := pts[1]

     

       155
       155
       +
       		cidStr := cid.String()

     

       156
       156
       +
       		b, err := bs.Get(context.TODO(), cid)

     

       157
       157
       +
       		if err != nil {

     

       158
       158
       +
       			return nil

     

       159
       159
       +
       		}

     

       160
       160
       +
       		if err := p.handleCreate(ctx, b.RawData(), time.Now().Format(time.RFC3339Nano), "unk", did, nsid, rkey, cidStr, "unk"); err != nil {

     

       161
       161
       +
       			return err

     

       162
       162
       +
       		}

     

       163
       163
       +
       		return nil

     

       164
       164
       +
       	}); err != nil {

     

       165
       165
       +
       		return fmt.Errorf("erorr traversing records: %v", err)

     

       166
       166
       +
       	}

     

       167
       167
       +
       

     

       168
       168
       +
       	return nil

     

       169
       169
       +
       }

     

       170
       170
       +
       

     

       171
       171
       +
       type ListReposResponse struct {

     

       172
       172
       +
       	Cursor string          `json:"cursor"`

     

       173
       173
       +
       	Repos  []ListReposRepo `json:"repos"`

     

       174
       174
       +
       }

     

       175
       175
       +
       

     

       176
       176
       +
       type ListReposRepo struct {

     

       177
       177
       +
       	Did    string  `json:"did"`

     

       178
       178
       +
       	Head   string  `json:"head"`

     

       179
       179
       +
       	Rev    string  `json:"rev"`

     

       180
       180
       +
       	Active bool    `json:"active"`

     

       181
       181
       +
       	Status *string `json:"status,omitempty"`

     

       182
       182
       +
       }

     

       183
       183
       +
       

     

       184
       184
       +
       func (rd *RepoDownloader) getDidsFromService(ctx context.Context, service string) ([]ListReposRepo, error) {

     

       185
       185
       +
       	var cursor string

     

       186
       186
       +
       	var repos []ListReposRepo

     

       187
       187
       +
       	if service == "https://atproto.brid.gy" {

     

       188
       188
       +
       		return nil, nil

     

       189
       189
       +
       	}

     

       190
       190
       +
       	for {

     

       191
       191
       +
       		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/xrpc/com.atproto.sync.listRepos?limit=1000&cursor=%s", service, cursor), nil)

     

       192
       192
       +
       		if err != nil {

     

       193
       193
       +
       			return nil, err

     

       194
       194
       +
       		}

     

       195
       195
       +
       

     

       196
       196
       +
       		if rd.p.ratelimitBypassKey != "" && strings.HasSuffix(service, ".bsky.network") {

     

       197
       197
       +
       			req.Header.Set("x-ratelimit-bypass", rd.p.ratelimitBypassKey)

     

       198
       198
       +
       		}

     

       199
       199
       +
       

     

       200
       200
       +
       		rl := rd.getRateLimiter(service)

     

       201
       201
       +
       		rl.Take()

     

       202
       202
       +
       

     

       203
       203
       +
       		cli := rd.getClient(service)

     

       204
       204
       +
       		resp, err := cli.Do(req)

     

       205
       205
       +
       		if err != nil {

     

       206
       206
       +
       			return nil, err

     

       207
       207
       +
       		}

     

       208
       208
       +
       		defer resp.Body.Close()

     

       209
       209
       +
       

     

       210
       210
       +
       		if resp.StatusCode != http.StatusOK {

     

       211
       211
       +
       			return nil, fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       212
       212
       +
       		}

     

       213
       213
       +
       

     

       214
       214
       +
       		var reposResp ListReposResponse

     

       215
       215
       +
       		if err := json.NewDecoder(resp.Body).Decode(&reposResp); err != nil {

     

       216
       216
       +
       			return nil, fmt.Errorf("error decoding repos response: %w", err)

     

       217
       217
       +
       		}

     

       218
       218
       +
       

     

       219
       219
       +
       		for _, repo := range reposResp.Repos {

     

       220
       220
       +
       			if repo.Status != nil {

     

       221
       221
       +
       				if *repo.Status == "deleted" || *repo.Status == "takendown" || *repo.Status == "deactivated" {

     

       222
       222
       +
       					continue

     

       223
       223
       +
       				}

     

       224
       224
       +
       			}

     

       225
       225
       +
       

     

       226
       226
       +
       			repos = append(repos, repo)

     

       227
       227
       +
       		}

     

       228
       228
       +
       

     

       229
       229
       +
       		if len(reposResp.Repos) != 1000 || reposResp.Cursor == "" {

     

       230
       230
       +
       			break

     

       231
       231
       +
       		}

     

       232
       232
       +
       

     

       233
       233
       +
       		fmt.Printf("cursor %s service %s\n", reposResp.Cursor, service)

     

       234
       234
       +
       

     

       235
       235
       +
       		cursor = reposResp.Cursor

     

       236
       236
       +
       	}

     

       237
       237
       +
       

     

       238
       238
       +
       	return repos, nil

     

       239
       239
       +
       }

     

       240
       240
       +
       

     

       241
       241
       +
       type ListServicesResponse struct {

     

       242
       242
       +
       	Cursor string                     `json:"cursor"`

     

       243
       243
       +
       	Hosts  []ListServicesResponseItem `json:"hosts"`

     

       244
       244
       +
       }

     

       245
       245
       +
       

     

       246
       246
       +
       type ListServicesResponseItem struct {

     

       247
       247
       +
       	Hostname string `json:"hostname"`

     

       248
       248
       +
       	Status   string `json:"status"`

     

       249
       249
       +
       }

     

       250
       250
       +
       

     

       251
       251
       +
       func (p *Photocopy) runProcessRepoWorker(ctx context.Context, jobs <-chan ProcessJob) {

     

       252
       252
       +
       	for j := range jobs {

     

       253
       253
       +
       		p.processRepo(ctx, j.repoBytes, j.did)

     

       254
       254
       +
       	}

     

       255
       255
       +
       }

     

       256
       256
       +
       

     

       257
       257
       +
       func (p *Photocopy) runBackfiller(ctx context.Context) error {

     

       258
       258
       +
       	startTime := time.Now()

     

       259
       259
       +
       

     

       260
       260
       +
       	fmt.Println("querying clickhouse for dids and services...")

     

       261
       261
       +
       

     

       262
       262
       +
       	type alreadyFetchedItem struct {

     

       263
       263
       +
       		Did string `ch:"did"`

     

       264
       264
       +
       	}

     

       265
       265
       +
       	var alreadyFetched []alreadyFetchedItem

     

       266
       266
       +
       	if err := p.conn.Select(ctx, &alreadyFetched, "SELECT DISTINCT(did) FROM default.record WHERE created_at < '2025-07-01'"); err != nil {

     

       267
       267
       +
       		return err

     

       268
       268
       +
       	}

     

       269
       269
       +
       

     

       270
       270
       +
       	alreadyFetchedMap := map[string]bool{}

     

       271
       271
       +
       	for _, d := range alreadyFetched {

     

       272
       272
       +
       		alreadyFetchedMap[d.Did] = true

     

       273
       273
       +
       	}

     

       274
       274
       +
       

     

       275
       275
       +
       	fmt.Println("getting dids")

     

       276
       276
       +
       

     

       277
       277
       +
       	var hostsCursor string

     

       278
       278
       +
       	var sevs []ListServicesResponseItem

     

       279
       279
       +
       	for {

     

       280
       280
       +
       		if hostsCursor != "" {

     

       281
       281
       +
       			hostsCursor = "&cursor=" + hostsCursor

     

       282
       282
       +
       		}

     

       283
       283
       +
       		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://relay1.us-east.bsky.network/xrpc/com.atproto.sync.listHosts?limit=1000%s", hostsCursor), nil)

     

       284
       284
       +
       		if err != nil {

     

       285
       285
       +
       			return err

     

       286
       286
       +
       		}

     

       287
       287
       +
       

     

       288
       288
       +
       		resp, err := http.DefaultClient.Do(req)

     

       289
       289
       +
       		if err != nil {

     

       290
       290
       +
       			return err

     

       291
       291
       +
       		}

     

       292
       292
       +
       		defer resp.Body.Close()

     

       293
       293
       +
       

     

       294
       294
       +
       		if resp.StatusCode != http.StatusOK {

     

       295
       295
       +
       			return fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       296
       296
       +
       		}

     

       297
       297
       +
       

     

       298
       298
       +
       		var sevsResp ListServicesResponse

     

       299
       299
       +
       		if err := json.NewDecoder(resp.Body).Decode(&sevsResp); err != nil {

     

       300
       300
       +
       			return fmt.Errorf("error decoding sevs response: %w", err)

     

       301
       301
       +
       		}

     

       302
       302
       +
       

     

       303
       303
       +
       		for _, sev := range sevsResp.Hosts {

     

       304
       304
       +
       			if sev.Status != "active" {

     

       305
       305
       +
       				continue

     

       306
       306
       +
       			}

     

       307
       307
       +
       

     

       308
       308
       +
       			sevs = append(sevs, sev)

     

       309
       309
       +
       		}

     

       310
       310
       +
       

     

       311
       311
       +
       		if len(sevsResp.Hosts) != 1000 || sevsResp.Cursor == "" {

     

       312
       312
       +
       			break

     

       313
       313
       +
       		}

     

       314
       314
       +
       

     

       315
       315
       +
       		hostsCursor = sevsResp.Cursor

     

       316
       316
       +
       	}

     

       317
       317
       +
       

     

       318
       318
       +
       	servicesDids := map[string][]string{}

     

       319
       319
       +
       	for _, sev := range sevs {

     

       320
       320
       +
       		servicesDids["https://"+sev.Hostname] = []string{}

     

       321
       321
       +
       	}

     

       322
       322
       +
       

     

       323
       323
       +
       	fmt.Printf("found %d services\n", len(servicesDids))

     

       324
       324
       +
       

     

       325
       325
       +
       	fmt.Printf("collecting dids...\n")

     

       326
       326
       +
       

     

       327
       327
       +
       	fmt.Printf("building download buckets...")

     

       328
       328
       +
       

     

       329
       329
       +
       	skipped := 0

     

       330
       330
       +
       	downloader := NewRepoDownloader(p)

     

       331
       331
       +
       	serviceDids := map[string][]string{}

     

       332
       332
       +
       

     

       333
       333
       +
       	for range runtime.NumCPU() / 2 {

     

       334
       334
       +
       		go p.runProcessRepoWorker(ctx, downloader.processChan)

     

       335
       335
       +
       	}

     

       336
       336
       +
       

     

       337
       337
       +
       	wg := sync.WaitGroup{}

     

       338
       338
       +
       	mplk := sync.Mutex{}

     

       339
       339
       +
       	for s := range servicesDids {

     

       340
       340
       +
       		wg.Add(1)

     

       341
       341
       +
       		go func() {

     

       342
       342
       +
       			defer wg.Done()

     

       343
       343
       +
       			repos, err := downloader.getDidsFromService(context.TODO(), s)

     

       344
       344
       +
       			if err != nil {

     

       345
       345
       +
       				fmt.Printf("error getting dids for services %s: %v", s, err)

     

       346
       346
       +
       				return

     

       347
       347
       +
       			}

     

       348
       348
       +
       			dids := []string{}

     

       349
       349
       +
       			for _, r := range repos {

     

       350
       350
       +
       				if alreadyFetchedMap[r.Did] {

     

       351
       351
       +
       					skipped++

     

       352
       352
       +
       					continue

     

       353
       353
       +
       				}

     

       354
       354
       +
       				dids = append(dids, r.Did)

     

       355
       355
       +
       			}

     

       356
       356
       +
       			mplk.Lock()

     

       357
       357
       +
       			defer mplk.Unlock()

     

       358
       358
       +
       			serviceDids[s] = dids

     

       359
       359
       +
       		}()

     

       360
       360
       +
       	}

     

       361
       361
       +
       

     

       362
       362
       +
       	fmt.Println("getting all the repos...")

     

       363
       363
       +
       	wg.Wait()

     

       364
       364
       +
       

     

       365
       365
       +
       	fmt.Printf("was able to skip %d repos\n", skipped)

     

       366
       366
       +
       

     

       367
       367
       +
       	total := 0

     

       368
       368
       +
       

     

       369
       369
       +
       	for service, dids := range serviceDids {

     

       370
       370
       +
       		if len(dids) < 100 {

     

       371
       371
       +
       			continue

     

       372
       372
       +
       		}

     

       373
       373
       +
       		fmt.Printf("%s: %d jobs\n", service, len(dids))

     

       374
       374
       +
       		total += len(dids)

     

       375
       375
       +
       	}

     

       376
       376
       +
       

     

       377
       377
       +
       	fmt.Printf("Total jobs: %d across %d services \n", total, len(serviceDids))

     

       378
       378
       +
       

     

       379
       379
       +
       	for _, c := range downloader.clients {

     

       380
       380
       +
       		c.Timeout = 10 * time.Minute

     

       381
       381
       +
       	}

     

       382
       382
       +
       

     

       383
       383
       +
       	for s := range downloader.rateLimits {

     

       384
       384
       +
       		if p.ratelimitBypassKey != "" && strings.HasSuffix(s, ".bsky.network") {

     

       385
       385
       +
       			downloader.rateLimits[s] = ratelimit.New(25)

     

       386
       386
       +
       		}

     

       387
       387
       +
       	}

     

       388
       388
       +
       

     

       389
       389
       +
       	processed := 0

     

       390
       390
       +
       	errored := 0

     

       391
       391
       +
       	var errors []error

     

       392
       392
       +
       	for service, dids := range serviceDids {

     

       393
       393
       +
       		go func() {

     

       394
       394
       +
       			for _, did := range dids {

     

       395
       395
       +
       				ratelimiter := downloader.getRateLimiter(service)

     

       396
       396
       +
       				ratelimiter.Take()

     

       397
       397
       +
       

     

       398
       398
       +
       				b, err := downloader.downloadRepo(service, did)

     

       399
       399
       +
       				if err != nil {

     

       400
       400
       +
       					errored++

     

       401
       401
       +
       					processed++

     

       402
       402
       +
       					errors = append(errors, err)

     

       403
       403
       +
       					continue

     

       404
       404
       +
       				}

     

       405
       405
       +
       

     

       406
       406
       +
       				go func(b []byte, did string) {

     

       407
       407
       +
       					downloader.processChan <- ProcessJob{repoBytes: b, did: did}

     

       408
       408
       +
       				}(b, did)

     

       409
       409
       +
       

     

       410
       410
       +
       				processed++

     

       411
       411
       +
       			}

     

       412
       412
       +
       		}()

     

       413
       413
       +
       	}

     

       414
       414
       +
       

     

       415
       415
       +
       	ticker := time.NewTicker(1 * time.Second)

     

       416
       416
       +
       	defer ticker.Stop()

     

       417
       417
       +
       

     

       418
       418
       +
       	for range ticker.C {

     

       419
       419
       +
       		elapsed := time.Since(startTime)

     

       420
       420
       +
       		rate := float64(processed) / elapsed.Seconds()

     

       421
       421
       +
       		remaining := total - processed

     

       422
       422
       +
       

     

       423
       423
       +
       		var eta string

     

       424
       424
       +
       		if rate > 0 {

     

       425
       425
       +
       			etaSeconds := float64(remaining) / rate

     

       426
       426
       +
       			etaDuration := time.Duration(etaSeconds * float64(time.Second))

     

       427
       427
       +
       			eta = fmt.Sprintf(", ETA: %v", etaDuration.Round(time.Second))

     

       428
       428
       +
       		} else {

     

       429
       429
       +
       			eta = ", ETA: calculating..."

     

       430
       430
       +
       		}

     

       431
       431
       +
       

     

       432
       432
       +
       		for _, err := range errors {

     

       433
       433
       +
       			fmt.Printf("%v\n", err)

     

       434
       434
       +
       		}

     

       435
       435
       +
       

     

       436
       436
       +
       		errors = nil

     

       437
       437
       +
       

     

       438
       438
       +
       		fmt.Printf("\rProgress: %d/%d processed (%.1f%%), %d skipped, %d errors, %.1f jobs/sec%s",

     

       439
       439
       +
       			processed, total, float64(processed)/float64(total)*100, skipped, errored, rate, eta)

     

       440
       440
       +
       	}

     

       441
       441
       +
       

     

       442
       442
       +
       	fmt.Printf("\nCompleted: %d processed, %d errors\n", processed, errored)

     

       443
       443
       +
       

     

       444
       444
       +
       	return nil

     

       445
       445
       +
       }

+37 -6

clickhouse_inserter/inserter.go

···

       3
       3
        
       import (

     

       4
       4
        
       	"context"

     

       5
       5
        
       	"log/slog"

     

       6
       6
       +
       	"reflect"

     

       6
       7
        
       	"slices"

     

       7
       8
        
       	"sync"

     

       8
       9
        
       	"time"

     
···

       10
       11
        
       	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"

     

       11
       12
        
       	"github.com/prometheus/client_golang/prometheus"

     

       12
       13
        
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       14
       14
       +
       	"go.uber.org/ratelimit"

     

       13
       15
        
       )

     

       14
       16
        
       

     

       15
       17
        
       type Inserter struct {

     
···

       23
       25
        
       	histogram      *prometheus.HistogramVec

     

       24
       26
        
       	logger         *slog.Logger

     

       25
       27
        
       	prefix         string

     

       28
       28
       +
       	rateLimit      ratelimit.Limiter

     

       26
       29
        
       }

     

       27
       30
        
       

     

       28
       31
        
       type Args struct {

     
···

       32
       35
        
       	PrometheusCounterPrefix string

     

       33
       36
        
       	Logger                  *slog.Logger

     

       34
       37
        
       	Histogram               *prometheus.HistogramVec

     

       38
       38
       +
       	RateLimit               int

     

       35
       39
        
       }

     

       36
       40
        
       

     

       37
       41
        
       func New(ctx context.Context, args *Args) (*Inserter, error) {

     
···

       47
       51
        
       		histogram: args.Histogram,

     

       48
       52
        
       		logger:    args.Logger,

     

       49
       53
        
       		prefix:    args.PrometheusCounterPrefix,

     

       54
       54
       +
       	}

     

       55
       55
       +
       

     

       56
       56
       +
       	if args.RateLimit != 0 {

     

       57
       57
       +
       		rateLimit := ratelimit.New(args.RateLimit)

     

       58
       58
       +
       		inserter.rateLimit = rateLimit

     

       50
       59
        
       	}

     

       51
       60
        
       

     

       52
       61
        
       	if args.PrometheusCounterPrefix != "" {

     
···

       109
       118
        
       }

     

       110
       119
        
       

     

       111
       120
        
       func (i *Inserter) sendStream(ctx context.Context, toInsert []any) {

     

       112
       112
       -
       	i.pendingSends.Inc()

     

       113
       113
       -
       	defer i.pendingSends.Dec()

     

       121
       121
       +
       	if i.pendingSends != nil {

     

       122
       122
       +
       		i.pendingSends.Inc()

     

       123
       123
       +
       		defer i.pendingSends.Dec()

     

       124
       124
       +
       	}

     

       114
       125
        
       

     

       115
       126
        
       	if i.histogram != nil {

     

       116
       127
        
       		start := time.Now()

     
···

       124
       135
        
       	}

     

       125
       136
        
       

     

       126
       137
        
       	status := "ok"

     

       127
       127
       -
       	defer func() {

     

       128
       128
       -
       		i.insertsCounter.WithLabelValues(status).Add(float64(len(toInsert)))

     

       129
       129
       -
       	}()

     

       138
       138
       +
       	if i.insertsCounter != nil {

     

       139
       139
       +
       		defer func() {

     

       140
       140
       +
       			i.insertsCounter.WithLabelValues(status).Add(float64(len(toInsert)))

     

       141
       141
       +
       		}()

     

       142
       142
       +
       	}

     

       130
       143
        
       

     

       131
       144
        
       	batch, err := i.conn.PrepareBatch(ctx, i.query)

     

       132
       145
        
       	if err != nil {

     
···

       136
       149
        
       	}

     

       137
       150
        
       

     

       138
       151
        
       	for _, d := range toInsert {

     

       139
       139
       -
       		if err := batch.AppendStruct(&d); err != nil {

     

       152
       152
       +
       		var structPtr any

     

       153
       153
       +
       		if reflect.TypeOf(d).Kind() == reflect.Ptr {

     

       154
       154
       +
       			structPtr = d

     

       155
       155
       +
       		} else {

     

       156
       156
       +
       			v := reflect.ValueOf(d)

     

       157
       157
       +
       			if v.CanAddr() {

     

       158
       158
       +
       				structPtr = v.Addr().Addr().Interface()

     

       159
       159
       +
       			} else {

     

       160
       160
       +
       				ptr := reflect.New(v.Type())

     

       161
       161
       +
       				ptr.Elem().Set(v)

     

       162
       162
       +
       				structPtr = ptr.Interface()

     

       163
       163
       +
       			}

     

       164
       164
       +
       		}

     

       165
       165
       +
       

     

       166
       166
       +
       		if err := batch.AppendStruct(structPtr); err != nil {

     

       140
       167
        
       			i.logger.Error("error appending to batch", "prefix", i.prefix, "error", err)

     

       141
       168
        
       		}

     

       169
       169
       +
       	}

     

       170
       170
       +
       

     

       171
       171
       +
       	if i.rateLimit != nil {

     

       172
       172
       +
       		i.rateLimit.Take()

     

       142
       173
        
       	}

     

       143
       174
        
       

     

       144
       175
        
       	if err := batch.Send(); err != nil {

+21 -1

cmd/photocopy/main.go

···

       62
       62
        
       				EnvVars:  []string{"PHOTOCOPY_CLICKHOUSE_PASS"},

     

       63
       63
        
       				Required: true,

     

       64
       64
        
       			},

     

       65
       65
       +
       			&cli.StringFlag{

     

       66
       66
       +
       				Name:     "ratelimit-bypass-key",

     

       67
       67
       +
       				EnvVars:  []string{"PHOTOCOPY_RATELIMIT_BYPASS_KEY"},

     

       68
       68
       +
       				Required: false,

     

       69
       69
       +
       			},

     

       70
       70
       +
       			&cli.BoolFlag{

     

       71
       71
       +
       				Name: "with-backfill",

     

       72
       72
       +
       			},

     

       73
       73
       +
       			&cli.StringFlag{

     

       74
       74
       +
       				Name:    "nervana-endpoint",

     

       75
       75
       +
       				EnvVars: []string{"PHOTOCOPY_NERVANA_ENDPOINT"},

     

       76
       76
       +
       			},

     

       77
       77
       +
       			&cli.StringFlag{

     

       78
       78
       +
       				Name:    "nervana-api-key",

     

       79
       79
       +
       				EnvVars: []string{"PHOTOCOPY_NERVANA_API_KEY"},

     

       80
       80
       +
       			},

     

       65
       81
        
       		},

     

       66
       82
        
       		Commands: cli.Commands{

     

       67
       83
        
       			&cli.Command{

     
···

       112
       128
        
       		ClickhouseDatabase:   cmd.String("clickhouse-database"),

     

       113
       129
        
       		ClickhouseUser:       cmd.String("clickhouse-user"),

     

       114
       130
        
       		ClickhousePass:       cmd.String("clickhouse-pass"),

     

       131
       131
       +
       		RatelimitBypassKey:   cmd.String("ratelimit-bypass-key"),

     

       132
       132
       +
       		NervanaEndpoint:      cmd.String("nervana-endpoint"),

     

       133
       133
       +
       		NervanaApiKey:        cmd.String("nervana-api-key"),

     

       115
       134
        
       	})

     

       116
       135
        
       	if err != nil {

     

       117
       136
        
       		panic(err)

     
···

       124
       143
        
       		sig := <-exitSignals

     

       125
       144
        
       

     

       126
       145
        
       		l.Info("received os exit signal", "signal", sig)

     

       146
       146
       +
       		cancel()

     

       127
       147
        
       	}()

     

       128
       148
        
       

     

       129
       129
       -
       	if err := p.Run(ctx); err != nil {

     

       149
       149
       +
       	if err := p.Run(ctx, cmd.Bool("with-backfill")); err != nil {

     

       130
       150
        
       		panic(err)

     

       131
       151
        
       	}

     

       132
       152

+1 -10

consumer.go

···

       7
       7
        
       	"net/http"

     

       8
       8
        
       	"net/url"

     

       9
       9
        
       	"os"

     

       10
       10
       -
       	"strings"

     

       11
       10
        
       	"time"

     

       12
       11
        
       

     

       13
       12
        
       	"github.com/bluesky-social/indigo/api/atproto"

     
···

       136
       135
        
       				continue

     

       137
       136
        
       			}

     

       138
       137
        
       

     

       139
       139
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       140
       140
       -
       				continue

     

       141
       141
       -
       			}

     

       142
       142
       -
       

     

       143
       143
       -
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String()); err != nil {

     

       138
       138
       +
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String(), fmt.Sprintf("%d", evt.Seq)); err != nil {

     

       144
       139
        
       				p.logger.Error("error handling create event", "error", err)

     

       145
       140
        
       				continue

     

       146
       141
        
       			}

     

       147
       142
        
       		case repomgr.EvtKindDeleteRecord:

     

       148
       148
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       149
       149
       -
       				continue

     

       150
       150
       -
       			}

     

       151
       151
       -
       

     

       152
       143
        
       			if err := p.handleDelete(ctx, did.String(), collection.String(), rkey.String()); err != nil {

     

       153
       144
        
       				p.logger.Error("error handling delete event", "error", err)

     

       154
       145
        
       				continue

+3 -3

go.mod

···

       3
       3
        
       go 1.24.4

     

       4
       4
        
       

     

       5
       5
        
       require (

     

       6
       6
       -
       	github.com/ClickHouse/clickhouse-go v1.5.4

     

       7
       6
        
       	github.com/ClickHouse/clickhouse-go/v2 v2.37.2

     

       8
       7
        
       	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de

     

       9
       8
        
       	github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325

     

       10
       9
        
       	github.com/gorilla/websocket v1.5.1

     

       11
       10
        
       	github.com/ipfs/go-cid v0.5.0

     

       11
       11
       +
       	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4

     

       12
       12
        
       	github.com/joho/godotenv v1.5.1

     

       13
       13
        
       	github.com/prometheus/client_golang v1.22.0

     

       14
       14
        
       	github.com/urfave/cli/v2 v2.25.7

     

       15
       15
       +
       	go.uber.org/ratelimit v0.3.1

     

       15
       16
        
       )

     

       16
       17
        
       

     

       17
       18
        
       require (

     

       18
       19
        
       	github.com/ClickHouse/ch-go v0.66.1 // indirect

     

       19
       20
        
       	github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect

     

       20
       21
        
       	github.com/andybalholm/brotli v1.1.1 // indirect

     

       22
       22
       +
       	github.com/benbjohnson/clock v1.3.0 // indirect

     

       21
       23
        
       	github.com/beorn7/perks v1.0.1 // indirect

     

       22
       24
        
       	github.com/carlmjohnson/versioninfo v0.22.5 // indirect

     

       23
       25
        
       	github.com/cespare/xxhash/v2 v2.3.0 // indirect

     

       24
       24
       -
       	github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect

     

       25
       26
        
       	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect

     

       26
       27
        
       	github.com/felixge/httpsnoop v1.0.4 // indirect

     

       27
       28
        
       	github.com/go-faster/city v1.0.1 // indirect

     
···

       54
       55
        
       	github.com/ipfs/go-merkledag v0.11.0 // indirect

     

       55
       56
        
       	github.com/ipfs/go-metrics-interface v0.0.1 // indirect

     

       56
       57
        
       	github.com/ipfs/go-verifcid v0.0.3 // indirect

     

       57
       57
       -
       	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect

     

       58
       58
        
       	github.com/ipld/go-codec-dagpb v1.6.0 // indirect

     

       59
       59
        
       	github.com/ipld/go-ipld-prime v0.21.0 // indirect

     

       60
       60
        
       	github.com/jackc/pgpassfile v1.0.0 // indirect

+2 -12

go.sum

···

       1
       1
        
       github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

     

       2
       2
        
       github.com/ClickHouse/ch-go v0.66.1 h1:LQHFslfVYZsISOY0dnOYOXGkOUvpv376CCm8g7W74A4=

     

       3
       3
        
       github.com/ClickHouse/ch-go v0.66.1/go.mod h1:NEYcg3aOFv2EmTJfo4m2WF7sHB/YFbLUuIWv9iq76xY=

     

       4
       4
       -
       github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=

     

       5
       5
       -
       github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=

     

       6
       4
        
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2 h1:wRLNKoynvHQEN4znnVHNLaYnrqVc9sGJmGYg+GGCfto=

     

       7
       5
        
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2/go.mod h1:pH2zrBGp5Y438DMwAxXMm1neSXPPjSI7tD4MURVULw8=

     

       8
       6
        
       github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU=

     
···

       20
       18
        
       github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=

     

       21
       19
        
       github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=

     

       22
       20
        
       github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=

     

       23
       23
       -
       github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=

     

       24
       24
       -
       github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=

     

       25
       21
        
       github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325 h1:Bftt2EcoLZK2Z2m12Ih5QqbReX8j29hbf4zJU/FKzaY=

     

       26
       22
        
       github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325/go.mod h1:8FlFpF5cIq3DQG0kEHqyTkPV/5MDQoaWLcVwza5ZPJU=

     

       27
       23
        
       github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=

     
···

       30
       26
        
       github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=

     

       31
       27
        
       github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

     

       32
       28
        
       github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

     

       33
       33
       -
       github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=

     

       34
       34
       -
       github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=

     

       35
       29
        
       github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=

     

       36
       30
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=

     

       37
       31
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

     
···

       57
       51
        
       github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=

     

       58
       52
        
       github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=

     

       59
       53
        
       github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=

     

       60
       60
       -
       github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=

     

       61
       54
        
       github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=

     

       62
       55
        
       github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus=

     

       63
       56
        
       github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4=

     
···

       177
       170
        
       github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=

     

       178
       171
        
       github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=

     

       179
       172
        
       github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=

     

       180
       180
       -
       github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=

     

       181
       173
        
       github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=

     

       182
       174
        
       github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=

     

       183
       175
        
       github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=

     
···

       200
       192
        
       github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=

     

       201
       193
        
       github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=

     

       202
       194
        
       github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=

     

       203
       203
       -
       github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=

     

       204
       195
        
       github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=

     

       205
       196
        
       github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=

     

       206
       197
        
       github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=

     
···

       223
       214
        
       github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=

     

       224
       215
        
       github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=

     

       225
       216
        
       github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=

     

       226
       226
       -
       github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=

     

       227
       217
        
       github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=

     

       228
       218
        
       github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=

     

       229
       219
        
       github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=

     
···

       262
       252
        
       github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=

     

       263
       253
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=

     

       264
       254
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=

     

       265
       265
       -
       github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=

     

       266
       266
       -
       github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

     

       267
       255
        
       github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=

     

       268
       256
        
       github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=

     

       269
       257
        
       github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

     
···

       360
       348
        
       go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=

     

       361
       349
        
       go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

     

       362
       350
        
       go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=

     

       351
       351
       +
       go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0=

     

       352
       352
       +
       go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk=

     

       363
       353
        
       go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=

     

       364
       354
        
       go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=

     

       365
       355
        
       go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=

+241 -130

handle_create.go

···

       1
       1
        
       package photocopy

     

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
       +
       	"bytes"

     

       4
       5
        
       	"context"

     

       5
       6
        
       	"fmt"

     

       7
       7
       +
       	"slices"

     

       8
       8
       +
       	"strings"

     

       6
       9
        
       	"time"

     

       7
       10
        
       

     

       8
       11
        
       	"github.com/araddon/dateparse"

     

       9
       12
        
       	"github.com/bluesky-social/indigo/api/bsky"

     

       10
       13
        
       	"github.com/bluesky-social/indigo/atproto/syntax"

     

       14
       14
       +
       	"github.com/haileyok/photocopy/models"

     

       11
       15
        
       )

     

       12
       16
        
       

     

       13
       13
       -
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid string) error {

     

       14
       14
       -
       	_, err := dateparse.ParseAny(indexedAt)

     

       17
       17
       +
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid, seq string) error {

     

       18
       18
       +
       	iat, err := dateparse.ParseAny(indexedAt)

     

       15
       19
        
       	if err != nil {

     

       16
       20
        
       		return err

     

       21
       21
       +
       	}

     

       22
       22
       +
       

     

       23
       23
       +
       	if err := p.handleCreateRecord(ctx, did, rkey, collection, cid, recb, seq); err != nil {

     

       24
       24
       +
       		p.logger.Error("error creating record", "error", err)

     

       17
       25
        
       	}

     

       18
       26
        
       

     

       19
       27
        
       	switch collection {

     

       28
       28
       +
       	case "app.bsky.feed.post":

     

       29
       29
       +
       		return p.handleCreatePost(ctx, rev, recb, uriFromParts(did, collection, rkey), did, collection, rkey, cid, iat)

     

       30
       30
       +
       	case "app.bsky.graph.follow":

     

       31
       31
       +
       		return p.handleCreateFollow(ctx, recb, uriFromParts(did, collection, rkey), did, rkey, iat)

     

       32
       32
       +
       	case "app.bsky.feed.like", "app.bsky.feed.repost":

     

       33
       33
       +
       		return p.handleCreateInteraction(ctx, recb, uriFromParts(did, collection, rkey), did, collection, rkey, iat)

     

       34
       34
       +
       	default:

     

       35
       35
       +
       		return nil

     

       36
       36
       +
       	}

     

       37
       37
       +
       }

     

       38
       38
       +
       

     

       39
       39
       +
       func (p *Photocopy) handleCreateRecord(ctx context.Context, did, rkey, collection, cid string, raw []byte, seq string) error {

     

       40
       40
       +
       	var cat time.Time

     

       41
       41
       +
       	prkey, err := syntax.ParseTID(rkey)

     

       42
       42
       +
       	if err == nil {

     

       43
       43
       +
       		cat = prkey.Time()

     

       44
       44
       +
       	} else {

     

       45
       45
       +
       		cat = time.Now()

     

       46
       46
       +
       	}

     

       47
       47
       +
       

     

       48
       48
       +
       	rec := models.Record{

     

       49
       49
       +
       		Did:        did,

     

       50
       50
       +
       		Rkey:       rkey,

     

       51
       51
       +
       		Collection: collection,

     

       52
       52
       +
       		Cid:        cid,

     

       53
       53
       +
       		Seq:        seq,

     

       54
       54
       +
       		Raw:        string(raw),

     

       55
       55
       +
       		CreatedAt:  cat,

     

       56
       56
       +
       	}

     

       57
       57
       +
       

     

       58
       58
       +
       	if err := p.inserters.recordsInserter.Insert(ctx, rec); err != nil {

     

       59
       59
       +
       		return err

     

       20
       60
        
       	}

     

       21
       61
        
       

     

       22
       62
        
       	return nil

     

       23
       63
        
       }

     

       24
       64
        
       

     

       25
       25
       -
       // func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

     

       26
       26
       -
       // 	var rec bsky.FeedPost

     

       27
       27
       -
       // 	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       28
       28
       -
       // 		return err

     

       29
       29
       -
       // 	}

     

       30
       30
       -
       //

     

       31
       31
       -
       // 	jb, err := json.Marshal(rec)

     

       32
       32
       -
       // 	if err != nil {

     

       33
       33
       -
       // 		return err

     

       34
       34
       -
       // 	}

     

       35
       35
       -
       //

     

       36
       36
       -
       // 	cat, err := parseTimeFromRecord(rec, rkey)

     

       37
       37
       -
       // 	if err != nil {

     

       38
       38
       -
       // 		return err

     

       39
       39
       -
       // 	}

     

       40
       40
       -
       //

     

       41
       41
       -
       // 	bqrec := &BigQueryRecord{

     

       42
       42
       -
       // 		Uri:        uri,

     

       43
       43
       -
       // 		AuthorDid:  did,

     

       44
       44
       -
       // 		Collection: collection,

     

       45
       45
       -
       // 		Rkey:       rkey,

     

       46
       46
       -
       // 		CreatedAt:  *cat,

     

       47
       47
       -
       // 		IndexedAt:  indexedAt,

     

       48
       48
       -
       // 		Raw:        recb,

     

       49
       49
       -
       // 		Json:       string(jb),

     

       50
       50
       -
       // 		Cid:        cid,

     

       51
       51
       -
       // 		Rev:        rev,

     

       52
       52
       -
       // 	}

     

       53
       53
       -
       //

     

       54
       54
       -
       // 	if rec.Reply != nil {

     

       55
       55
       -
       // 		if rec.Reply.Parent != nil {

     

       56
       56
       -
       // 			bqrec.ReplyToUri = rec.Reply.Parent.Uri

     

       57
       57
       -
       // 		}

     

       58
       58
       -
       // 		if rec.Reply.Root != nil {

     

       59
       59
       -
       // 			bqrec.InThreadUri = rec.Reply.Root.Uri

     

       60
       60
       -
       // 		}

     

       61
       61
       -
       // 	}

     

       62
       62
       -
       //

     

       63
       63
       -
       // 	if err := p.inserters.genericInserter.Insert(ctx, bqrec); err != nil {

     

       64
       64
       -
       // 		return err

     

       65
       65
       -
       // 	}

     

       66
       66
       -
       //

     

       67
       67
       -
       // 	return nil

     

       68
       68
       -
       // }

     

       65
       65
       +
       func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

     

       66
       66
       +
       	var rec bsky.FeedPost

     

       67
       67
       +
       	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       68
       68
       +
       		return err

     

       69
       69
       +
       	}

     

       70
       70
       +
       

     

       71
       71
       +
       	cat, err := parseTimeFromRecord(rec, rkey)

     

       72
       72
       +
       	if err != nil {

     

       73
       73
       +
       		return err

     

       74
       74
       +
       	}

     

       75
       75
       +
       

     

       76
       76
       +
       	lang := ""

     

       77
       77
       +
       	if len(rec.Langs) != 0 {

     

       78
       78
       +
       		lang = rec.Langs[0]

     

       79
       79
       +
       	}

     

       80
       80
       +
       

     

       81
       81
       +
       	post := models.Post{

     

       82
       82
       +
       		Uri:       uri,

     

       83
       83
       +
       		Rkey:      rkey,

     

       84
       84
       +
       		CreatedAt: *cat,

     

       85
       85
       +
       		IndexedAt: indexedAt,

     

       86
       86
       +
       		Did:       did,

     

       87
       87
       +
       		Lang:      lang,

     

       88
       88
       +
       		Text:      rec.Text,

     

       89
       89
       +
       	}

     

       90
       90
       +
       

     

       91
       91
       +
       	if rec.Reply != nil {

     

       92
       92
       +
       		if rec.Reply.Parent != nil {

     

       93
       93
       +
       			aturi, err := syntax.ParseATURI(rec.Reply.Parent.Uri)

     

       94
       94
       +
       			if err != nil {

     

       95
       95
       +
       				return fmt.Errorf("error parsing at-uri: %w", err)

     

       96
       96
       +
       

     

       97
       97
       +
       			}

     

       98
       98
       +
       			post.ParentDid = aturi.Authority().String()

     

       99
       99
       +
       			post.ParentUri = rec.Reply.Parent.Uri

     

       100
       100
       +
       		}

     

       101
       101
       +
       		if rec.Reply.Root != nil {

     

       102
       102
       +
       			aturi, err := syntax.ParseATURI(rec.Reply.Root.Uri)

     

       103
       103
       +
       			if err != nil {

     

       104
       104
       +
       				return fmt.Errorf("error parsing at-uri: %w", err)

     

       105
       105
       +
       

     

       106
       106
       +
       			}

     

       107
       107
       +
       			post.RootDid = aturi.Authority().String()

     

       108
       108
       +
       			post.RootUri = rec.Reply.Root.Uri

     

       109
       109
       +
       		}

     

       110
       110
       +
       	}

     

       111
       111
       +
       

     

       112
       112
       +
       	if rec.Embed != nil && rec.Embed.EmbedRecord != nil && rec.Embed.EmbedRecord.Record != nil {

     

       113
       113
       +
       		aturi, err := syntax.ParseATURI(rec.Embed.EmbedRecord.Record.Uri)

     

       114
       114
       +
       		if err != nil {

     

       115
       115
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       116
       116
       +
       

     

       117
       117
       +
       		}

     

       118
       118
       +
       		post.QuoteDid = aturi.Authority().String()

     

       119
       119
       +
       		post.QuoteUri = rec.Embed.EmbedRecord.Record.Uri

     

       120
       120
       +
       	} else if rec.Embed != nil && rec.Embed.EmbedRecordWithMedia != nil && rec.Embed.EmbedRecordWithMedia.Record != nil && rec.Embed.EmbedRecordWithMedia.Record.Record != nil {

     

       121
       121
       +
       		aturi, err := syntax.ParseATURI(rec.Embed.EmbedRecordWithMedia.Record.Record.Uri)

     

       122
       122
       +
       		if err != nil {

     

       123
       123
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       124
       124
       +
       

     

       125
       125
       +
       		}

     

       126
       126
       +
       		post.QuoteDid = aturi.Authority().String()

     

       127
       127
       +
       		post.QuoteUri = rec.Embed.EmbedRecordWithMedia.Record.Record.Uri

     

       128
       128
       +
       	}

     

       129
       129
       +
       

     

       130
       130
       +
       	if err := p.inserters.postsInserter.Insert(ctx, post); err != nil {

     

       131
       131
       +
       		return err

     

       132
       132
       +
       	}

     

       133
       133
       +
       

     

       134
       134
       +
       	isEn := slices.Contains(rec.Langs, "en")

     

       135
       135
       +
       	if rec.Text != "" && rec.Reply == nil && isEn && p.nervanaClient != nil {

     

       136
       136
       +
       		go func(ctx context.Context, rec bsky.FeedPost, did, rkey string) {

     

       137
       137
       +
       			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)

     

       138
       138
       +
       			defer cancel()

     

       139
       139
       +
       

     

       140
       140
       +
       			nervanaItems, err := p.nervanaClient.MakeRequest(ctx, rec.Text)

     

       141
       141
       +
       			if err != nil {

     

       142
       142
       +
       				p.logger.Error("error making nervana items request", "error", err)

     

       143
       143
       +
       				return

     

       144
       144
       +
       			}

     

       145
       145
       +
       

     

       146
       146
       +
       			for _, ni := range nervanaItems {

     

       147
       147
       +
       				postLabel := models.PostLabel{

     

       148
       148
       +
       					Did:         did,

     

       149
       149
       +
       					Rkey:        rkey,

     

       150
       150
       +
       					Text:        ni.Text,

     

       151
       151
       +
       					Label:       ni.Label,

     

       152
       152
       +
       					EntityId:    ni.EntityId,

     

       153
       153
       +
       					Description: ni.Description,

     

       154
       154
       +
       					Topic:       "",

     

       155
       155
       +
       					CreatedAt:   time.Now(),

     

       156
       156
       +
       				}

     

       157
       157
       +
       				p.inserters.labelsInserter.Insert(ctx, postLabel)

     

       158
       158
       +
       			}

     

       159
       159
       +
       		}(ctx, rec, did, rkey)

     

       160
       160
       +
       	}

     

       161
       161
       +
       

     

       162
       162
       +
       	return nil

     

       163
       163
       +
       }

     

       164
       164
       +
       

     

       165
       165
       +
       func (p *Photocopy) handleCreateFollow(ctx context.Context, recb []byte, uri, did, rkey string, indexedAt time.Time) error {

     

       166
       166
       +
       	var rec bsky.GraphFollow

     

       167
       167
       +
       	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       168
       168
       +
       		return err

     

       169
       169
       +
       	}

     

       170
       170
       +
       

     

       171
       171
       +
       	cat, err := parseTimeFromRecord(rec, rkey)

     

       172
       172
       +
       	if err != nil {

     

       173
       173
       +
       		return err

     

       174
       174
       +
       	}

     

       175
       175
       +
       

     

       176
       176
       +
       	follow := models.Follow{

     

       177
       177
       +
       		Uri:       uri,

     

       178
       178
       +
       		Did:       did,

     

       179
       179
       +
       		Rkey:      rkey,

     

       180
       180
       +
       		CreatedAt: *cat,

     

       181
       181
       +
       		IndexedAt: indexedAt,

     

       182
       182
       +
       		Subject:   rec.Subject,

     

       183
       183
       +
       	}

     

       184
       184
       +
       

     

       185
       185
       +
       	if err := p.inserters.followsInserter.Insert(ctx, follow); err != nil {

     

       186
       186
       +
       		return err

     

       187
       187
       +
       	}

     

       188
       188
       +
       

     

       189
       189
       +
       	return nil

     

       190
       190
       +
       }

     

       191
       191
       +
       

     

       192
       192
       +
       func (p *Photocopy) handleCreateInteraction(ctx context.Context, recb []byte, uri, did, collection, rkey string, indexedAt time.Time) error {

     

       193
       193
       +
       	colPts := strings.Split(collection, ".")

     

       194
       194
       +
       	if len(colPts) < 4 {

     

       195
       195
       +
       		return fmt.Errorf("invalid collection type %s", collection)

     

       196
       196
       +
       	}

     

       197
       197
       +
       

     

       198
       198
       +
       	interaction := models.Interaction{

     

       199
       199
       +
       		Uri:        uri,

     

       200
       200
       +
       		Kind:       colPts[3],

     

       201
       201
       +
       		Rkey:       rkey,

     

       202
       202
       +
       		IndexedAt:  indexedAt,

     

       203
       203
       +
       		Did:        did,

     

       204
       204
       +
       		SubjectUri: uri,

     

       205
       205
       +
       		SubjectDid: did,

     

       206
       206
       +
       	}

     

       207
       207
       +
       

     

       208
       208
       +
       	switch collection {

     

       209
       209
       +
       	case "app.bsky.feed.like":

     

       210
       210
       +
       		var rec bsky.FeedLike

     

       211
       211
       +
       		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       212
       212
       +
       			return err

     

       213
       213
       +
       		}

     

       214
       214
       +
       

     

       215
       215
       +
       		cat, err := parseTimeFromRecord(rec, rkey)

     

       216
       216
       +
       		if err != nil {

     

       217
       217
       +
       			return err

     

       218
       218
       +
       		}

     

       219
       219
       +
       

     

       220
       220
       +
       		if rec.Subject == nil {

     

       221
       221
       +
       			return fmt.Errorf("invalid subject in like")

     

       222
       222
       +
       		}

     

       223
       223
       +
       

     

       224
       224
       +
       		aturi, err := syntax.ParseATURI(rec.Subject.Uri)

     

       225
       225
       +
       		if err != nil {

     

       226
       226
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       227
       227
       +
       

     

       228
       228
       +
       		}

     

       229
       229
       +
       

     

       230
       230
       +
       		interaction.SubjectDid = aturi.Authority().String()

     

       231
       231
       +
       		interaction.SubjectUri = rec.Subject.Uri

     

       232
       232
       +
       		interaction.CreatedAt = *cat

     

       233
       233
       +
       	case "app.bsky.feed.repost":

     

       234
       234
       +
       		var rec bsky.FeedRepost

     

       235
       235
       +
       		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       236
       236
       +
       			return err

     

       237
       237
       +
       		}

     

       238
       238
       +
       

     

       239
       239
       +
       		cat, err := parseTimeFromRecord(rec, rkey)

     

       240
       240
       +
       		if err != nil {

     

       241
       241
       +
       			return err

     

       242
       242
       +
       		}

     

       243
       243
       +
       

     

       244
       244
       +
       		if rec.Subject == nil {

     

       245
       245
       +
       			return fmt.Errorf("invalid subject in repost")

     

       246
       246
       +
       		}

     

       247
       247
       +
       

     

       248
       248
       +
       		aturi, err := syntax.ParseATURI(rec.Subject.Uri)

     

       249
       249
       +
       		if err != nil {

     

       250
       250
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       251
       251
       +
       

     

       252
       252
       +
       		}

     

       253
       253
       +
       

     

       254
       254
       +
       		interaction.SubjectDid = aturi.Authority().String()

     

       255
       255
       +
       		interaction.SubjectUri = rec.Subject.Uri

     

       256
       256
       +
       		interaction.CreatedAt = *cat

     

       257
       257
       +
       	}

     

       258
       258
       +
       

     

       259
       259
       +
       	if err := p.inserters.interactionsInserter.Insert(ctx, interaction); err != nil {

     

       260
       260
       +
       		return err

     

       261
       261
       +
       	}

     

       69
       262
        
       

     

       70
       70
       -
       // func (p *Photocopy) handleCreateFollow(ctx context.Context, recb []byte, uri, did, rkey string, indexedAt time.Time) error {

     

       71
       71
       -
       // 	var rec bsky.GraphFollow

     

       72
       72
       -
       // 	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       73
       73
       -
       // 		return err

     

       74
       74
       -
       // 	}

     

       75
       75
       -
       //

     

       76
       76
       -
       // 	cat, err := parseTimeFromRecord(rec, rkey)

     

       77
       77
       -
       // 	if err != nil {

     

       78
       78
       -
       // 		return err

     

       79
       79
       -
       // 	}

     

       80
       80
       -
       //

     

       81
       81
       -
       // 	bqrec := &BigQueryFollow{

     

       82
       82
       -
       // 		Uri:        uri,

     

       83
       83
       -
       // 		AuthorDid:  did,

     

       84
       84
       -
       // 		Rkey:       rkey,

     

       85
       85
       -
       // 		CreatedAt:  *cat,

     

       86
       86
       -
       // 		IndexedAt:  indexedAt,

     

       87
       87
       -
       // 		SubjectDid: rec.Subject,

     

       88
       88
       -
       // 	}

     

       89
       89
       -
       //

     

       90
       90
       -
       // 	if err := p.inserters.followsInserter.Insert(ctx, bqrec); err != nil {

     

       91
       91
       -
       // 		return err

     

       92
       92
       -
       // 	}

     

       93
       93
       -
       //

     

       94
       94
       -
       // 	return nil

     

       95
       95
       -
       // }

     

       96
       96
       -
       //

     

       97
       97
       -
       // func (p *Photocopy) handleCreateInteraction(ctx context.Context, recb []byte, uri, did, collection, rkey string, indexedAt time.Time) error {

     

       98
       98
       -
       // 	colPts := strings.Split(collection, ".")

     

       99
       99
       -
       // 	if len(colPts) < 4 {

     

       100
       100
       -
       // 		return fmt.Errorf("invalid collection type %s", collection)

     

       101
       101
       -
       // 	}

     

       102
       102
       -
       //

     

       103
       103
       -
       // 	bqi := &BigQueryInteraction{

     

       104
       104
       -
       // 		Uri:       uri,

     

       105
       105
       -
       // 		Kind:      colPts[3],

     

       106
       106
       -
       // 		AuthorDid: did,

     

       107
       107
       -
       // 		Rkey:      rkey,

     

       108
       108
       -
       // 		IndexedAt: indexedAt,

     

       109
       109
       -
       // 	}

     

       110
       110
       -
       //

     

       111
       111
       -
       // 	switch collection {

     

       112
       112
       -
       // 	case "app.bsky.feed.like":

     

       113
       113
       -
       // 		var rec bsky.FeedLike

     

       114
       114
       -
       // 		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       115
       115
       -
       // 			return err

     

       116
       116
       -
       // 		}

     

       117
       117
       -
       //

     

       118
       118
       -
       // 		cat, err := parseTimeFromRecord(rec, rkey)

     

       119
       119
       -
       // 		if err != nil {

     

       120
       120
       -
       // 			return err

     

       121
       121
       -
       // 		}

     

       122
       122
       -
       //

     

       123
       123
       -
       // 		if rec.Subject == nil {

     

       124
       124
       -
       // 			return fmt.Errorf("invalid subject in like")

     

       125
       125
       -
       // 		}

     

       126
       126
       -
       //

     

       127
       127
       -
       // 		bqi.SubjectUri = rec.Subject.Uri

     

       128
       128
       -
       // 		bqi.CreatedAt = *cat

     

       129
       129
       -
       // 	case "app.bsky.feed.repost":

     

       130
       130
       -
       // 		var rec bsky.FeedRepost

     

       131
       131
       -
       // 		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       132
       132
       -
       // 			return err

     

       133
       133
       -
       // 		}

     

       134
       134
       -
       //

     

       135
       135
       -
       // 		cat, err := parseTimeFromRecord(rec, rkey)

     

       136
       136
       -
       // 		if err != nil {

     

       137
       137
       -
       // 			return err

     

       138
       138
       -
       // 		}

     

       139
       139
       -
       //

     

       140
       140
       -
       // 		if rec.Subject == nil {

     

       141
       141
       -
       // 			return fmt.Errorf("invalid subject in repost")

     

       142
       142
       -
       // 		}

     

       143
       143
       -
       //

     

       144
       144
       -
       // 		bqi.SubjectUri = rec.Subject.Uri

     

       145
       145
       -
       // 		bqi.CreatedAt = *cat

     

       146
       146
       -
       // 	}

     

       147
       147
       -
       //

     

       148
       148
       -
       // 	if err := p.inserters.interactionsInserter.Insert(ctx, bqi); err != nil {

     

       149
       149
       -
       // 		return err

     

       150
       150
       -
       // 	}

     

       151
       151
       -
       //

     

       152
       152
       -
       // 	return nil

     

       153
       153
       -
       // }

     

       263
       263
       +
       	return nil

     

       264
       264
       +
       }

     

       154
       265
        
       

     

       155
       266
        
       func parseTimeFromRecord(rec any, rkey string) (*time.Time, error) {

     

       156
       267
        
       	var rkeyTime time.Time

+13

handle_delete.go

···

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
        
       	"context"

     

       5
       5
       +
       	"time"

     

       6
       6
       +
       

     

       7
       7
       +
       	"github.com/haileyok/photocopy/models"

     

       5
       8
        
       )

     

       6
       9
        
       

     

       7
       10
        
       func (p *Photocopy) handleDelete(ctx context.Context, did, collection, rkey string) error {

     

       11
       11
       +
       	del := models.Delete{

     

       12
       12
       +
       		Did:       did,

     

       13
       13
       +
       		Rkey:      rkey,

     

       14
       14
       +
       		CreatedAt: time.Now(),

     

       15
       15
       +
       	}

     

       16
       16
       +
       

     

       17
       17
       +
       	if err := p.inserters.deletesInserter.Insert(ctx, del); err != nil {

     

       18
       18
       +
       		return err

     

       19
       19
       +
       	}

     

       20
       20
       +
       

     

       8
       21
        
       	return nil

     

       9
       22
        
       }

models/delete.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Delete struct {

     

       6
       6
       +
       	Did       string    `ch:"did"`

     

       7
       7
       +
       	Rkey      string    `ch:"rkey"`

     

       8
       8
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       9
       9
       +
       }

+12

models/follow.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Follow struct {

     

       6
       6
       +
       	Uri       string    `ch:"uri"`

     

       7
       7
       +
       	Did       string    `ch:"did"`

     

       8
       8
       +
       	Rkey      string    `ch:"rkey"`

     

       9
       9
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       10
       10
       +
       	IndexedAt time.Time `ch:"indexed_at"`

     

       11
       11
       +
       	Subject   string    `ch:"subject"`

     

       12
       12
       +
       }

+14

models/interaction.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Interaction struct {

     

       6
       6
       +
       	Uri        string    `ch:"uri"`

     

       7
       7
       +
       	Did        string    `ch:"did"`

     

       8
       8
       +
       	Rkey       string    `ch:"rkey"`

     

       9
       9
       +
       	Kind       string    `ch:"kind"`

     

       10
       10
       +
       	CreatedAt  time.Time `ch:"created_at"`

     

       11
       11
       +
       	IndexedAt  time.Time `ch:"indexed_at"`

     

       12
       12
       +
       	SubjectUri string    `ch:"subject_uri"`

     

       13
       13
       +
       	SubjectDid string    `ch:"subject_did"`

     

       14
       14
       +
       }

+19

models/post.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Post struct {

     

       6
       6
       +
       	Uri       string    `ch:"uri"`

     

       7
       7
       +
       	Did       string    `ch:"did"`

     

       8
       8
       +
       	Rkey      string    `ch:"rkey"`

     

       9
       9
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       10
       10
       +
       	IndexedAt time.Time `ch:"indexed_at"`

     

       11
       11
       +
       	RootUri   string    `ch:"root_uri"`

     

       12
       12
       +
       	RootDid   string    `ch:"root_did"`

     

       13
       13
       +
       	ParentUri string    `ch:"parent_uri"`

     

       14
       14
       +
       	ParentDid string    `ch:"parent_did"`

     

       15
       15
       +
       	QuoteUri  string    `ch:"quote_uri"`

     

       16
       16
       +
       	QuoteDid  string    `ch:"quote_did"`

     

       17
       17
       +
       	Lang      string    `ch:"lang"`

     

       18
       18
       +
       	Text      string    `ch:"text"`

     

       19
       19
       +
       }

+14

models/post_label.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type PostLabel struct {

     

       6
       6
       +
       	Did         string    `ch:"did"`

     

       7
       7
       +
       	Rkey        string    `ch:"rkey"`

     

       8
       8
       +
       	CreatedAt   time.Time `ch:"created_at"`

     

       9
       9
       +
       	Text        string    `ch:"text"`

     

       10
       10
       +
       	Label       string    `ch:"label"`

     

       11
       11
       +
       	EntityId    string    `ch:"entity_id"`

     

       12
       12
       +
       	Description string    `ch:"description"`

     

       13
       13
       +
       	Topic       string    `ch:"topic"`

     

       14
       14
       +
       }

+13

models/record.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Record struct {

     

       6
       6
       +
       	Did        string    `ch:"did"`

     

       7
       7
       +
       	Rkey       string    `ch:"rkey"`

     

       8
       8
       +
       	Collection string    `ch:"collection"`

     

       9
       9
       +
       	Cid        string    `ch:"cid"`

     

       10
       10
       +
       	Seq        string    `ch:"seq"`

     

       11
       11
       +
       	Raw        string    `ch:"raw"`

     

       12
       12
       +
       	CreatedAt  time.Time `ch:"created_at"`

     

       13
       13
       +
       }

-1

models.go

···

       1
       1
       -
       package photocopy

+77

nervana/client.go

···

       1
       1
       +
       package nervana

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"bytes"

     

       5
       5
       +
       	"context"

     

       6
       6
       +
       	"encoding/json"

     

       7
       7
       +
       	"fmt"

     

       8
       8
       +
       	"io"

     

       9
       9
       +
       	"net/http"

     

       10
       10
       +
       	"time"

     

       11
       11
       +
       )

     

       12
       12
       +
       

     

       13
       13
       +
       type Client struct {

     

       14
       14
       +
       	cli      *http.Client

     

       15
       15
       +
       	endpoint string

     

       16
       16
       +
       	apiKey   string

     

       17
       17
       +
       }

     

       18
       18
       +
       

     

       19
       19
       +
       func NewClient(endpoint string, apiKey string) *Client {

     

       20
       20
       +
       	return &Client{

     

       21
       21
       +
       		cli: &http.Client{

     

       22
       22
       +
       			Timeout: 5 * time.Second,

     

       23
       23
       +
       		},

     

       24
       24
       +
       		endpoint: endpoint,

     

       25
       25
       +
       		apiKey:   apiKey,

     

       26
       26
       +
       	}

     

       27
       27
       +
       }

     

       28
       28
       +
       

     

       29
       29
       +
       type NervanaItem struct {

     

       30
       30
       +
       	Text        string `json:"text"`

     

       31
       31
       +
       	Label       string `json:"label"`

     

       32
       32
       +
       	EntityId    string `json:"entityId"`

     

       33
       33
       +
       	Description string `json:"description"`

     

       34
       34
       +
       }

     

       35
       35
       +
       

     

       36
       36
       +
       func (c *Client) newRequest(ctx context.Context, text string) (*http.Request, error) {

     

       37
       37
       +
       	payload := map[string]string{

     

       38
       38
       +
       		"text":     text,

     

       39
       39
       +
       		"language": "en",

     

       40
       40
       +
       	}

     

       41
       41
       +
       

     

       42
       42
       +
       	b, err := json.Marshal(payload)

     

       43
       43
       +
       	if err != nil {

     

       44
       44
       +
       		return nil, err

     

       45
       45
       +
       	}

     

       46
       46
       +
       

     

       47
       47
       +
       	req, err := http.NewRequestWithContext(ctx, "POST", c.endpoint, bytes.NewReader(b))

     

       48
       48
       +
       

     

       49
       49
       +
       	req.Header.Set("Authorization", "Bearer "+c.apiKey)

     

       50
       50
       +
       

     

       51
       51
       +
       	return req, err

     

       52
       52
       +
       }

     

       53
       53
       +
       

     

       54
       54
       +
       func (c *Client) MakeRequest(ctx context.Context, text string) ([]NervanaItem, error) {

     

       55
       55
       +
       	req, err := c.newRequest(ctx, text)

     

       56
       56
       +
       	if err != nil {

     

       57
       57
       +
       		return nil, err

     

       58
       58
       +
       	}

     

       59
       59
       +
       

     

       60
       60
       +
       	resp, err := c.cli.Do(req)

     

       61
       61
       +
       	if err != nil {

     

       62
       62
       +
       		return nil, err

     

       63
       63
       +
       	}

     

       64
       64
       +
       	defer resp.Body.Close()

     

       65
       65
       +
       

     

       66
       66
       +
       	if resp.StatusCode != 200 {

     

       67
       67
       +
       		io.Copy(io.Discard, resp.Body)

     

       68
       68
       +
       		return nil, fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       69
       69
       +
       	}

     

       70
       70
       +
       

     

       71
       71
       +
       	var nervanaResp []NervanaItem

     

       72
       72
       +
       	if err := json.NewDecoder(resp.Body).Decode(&nervanaResp); err != nil {

     

       73
       73
       +
       		return nil, err

     

       74
       74
       +
       	}

     

       75
       75
       +
       

     

       76
       76
       +
       	return nervanaResp, nil

     

       77
       77
       +
       }

+166 -15

photocopy.go

···

       9
       9
        
       	"time"

     

       10
       10
        
       

     

       11
       11
        
       	"github.com/ClickHouse/clickhouse-go/v2"

     

       12
       12
       +
       	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"

     

       12
       13
        
       	"github.com/haileyok/photocopy/clickhouse_inserter"

     

       14
       14
       +
       	"github.com/haileyok/photocopy/nervana"

     

       13
       15
        
       	"github.com/prometheus/client_golang/prometheus"

     

       14
       16
        
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       15
       17
        
       	"github.com/prometheus/client_golang/prometheus/promhttp"

     
···

       27
       29
        
       	inserters *Inserters

     

       28
       30
        
       

     

       29
       31
        
       	plcScraper *PLCScraper

     

       32
       32
       +
       

     

       33
       33
       +
       	ratelimitBypassKey string

     

       34
       34
       +
       

     

       35
       35
       +
       	conn driver.Conn

     

       36
       36
       +
       

     

       37
       37
       +
       	nervanaClient   *nervana.Client

     

       38
       38
       +
       	nervanaEndpoint string

     

       39
       39
       +
       	nervanaApiKey   string

     

       30
       40
        
       }

     

       31
       41
        
       

     

       32
       42
        
       type Inserters struct {

     

       33
       33
       -
       	followsInserter *clickhouse_inserter.Inserter

     

       34
       34
       -
       	plcInserter     *clickhouse_inserter.Inserter

     

       43
       43
       +
       	followsInserter      *clickhouse_inserter.Inserter

     

       44
       44
       +
       	interactionsInserter *clickhouse_inserter.Inserter

     

       45
       45
       +
       	postsInserter        *clickhouse_inserter.Inserter

     

       46
       46
       +
       	plcInserter          *clickhouse_inserter.Inserter

     

       47
       47
       +
       	recordsInserter      *clickhouse_inserter.Inserter

     

       48
       48
       +
       	deletesInserter      *clickhouse_inserter.Inserter

     

       49
       49
       +
       	labelsInserter       *clickhouse_inserter.Inserter

     

       35
       50
        
       }

     

       36
       51
        
       

     

       37
       52
        
       type Args struct {

     
···

       44
       59
        
       	ClickhouseDatabase   string

     

       45
       60
        
       	ClickhouseUser       string

     

       46
       61
        
       	ClickhousePass       string

     

       62
       62
       +
       	RatelimitBypassKey   string

     

       63
       63
       +
       	NervanaEndpoint      string

     

       64
       64
       +
       	NervanaApiKey        string

     

       47
       65
        
       }

     

       48
       66
        
       

     

       49
       67
        
       func New(ctx context.Context, args *Args) (*Photocopy, error) {

     

       50
       50
       -
       	p := &Photocopy{

     

       51
       51
       -
       		logger:      args.Logger,

     

       52
       52
       -
       		metricsAddr: args.MetricsAddr,

     

       53
       53
       -
       		relayHost:   args.RelayHost,

     

       54
       54
       -
       		wg:          sync.WaitGroup{},

     

       55
       55
       -
       		cursorFile:  args.CursorFile,

     

       56
       56
       -
       	}

     

       57
       57
       -
       

     

       58
       68
        
       	conn, err := clickhouse.Open(&clickhouse.Options{

     

       59
       69
        
       		Addr: []string{args.ClickhouseAddr},

     

       60
       70
        
       		Auth: clickhouse.Auth{

     
···

       67
       77
        
       		return nil, err

     

       68
       78
        
       	}

     

       69
       79
        
       

     

       80
       80
       +
       	p := &Photocopy{

     

       81
       81
       +
       		logger:             args.Logger,

     

       82
       82
       +
       		metricsAddr:        args.MetricsAddr,

     

       83
       83
       +
       		relayHost:          args.RelayHost,

     

       84
       84
       +
       		wg:                 sync.WaitGroup{},

     

       85
       85
       +
       		cursorFile:         args.CursorFile,

     

       86
       86
       +
       		ratelimitBypassKey: args.RatelimitBypassKey,

     

       87
       87
       +
       		conn:               conn,

     

       88
       88
       +
       	}

     

       89
       89
       +
       

     

       70
       90
        
       	insertionsHist := promauto.NewHistogramVec(prometheus.HistogramOpts{

     

       71
       91
        
       		Name:    "photocopy_inserts_time",

     

       72
       92
        
       		Help:    "histogram of photocopy inserts",

     
···

       76
       96
        
       	fi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       77
       97
        
       		PrometheusCounterPrefix: "photocopy_follows",

     

       78
       98
        
       		Histogram:               insertionsHist,

     

       99
       99
       +
       		BatchSize:               500,

     

       100
       100
       +
       		Logger:                  p.logger,

     

       101
       101
       +
       		Conn:                    conn,

     

       102
       102
       +
       		Query:                   "INSERT INTO follow (uri, did, rkey, created_at, indexed_at, subject)",

     

       103
       103
       +
       		RateLimit:               3,

     

       104
       104
       +
       	})

     

       105
       105
       +
       	if err != nil {

     

       106
       106
       +
       		return nil, err

     

       107
       107
       +
       	}

     

       108
       108
       +
       

     

       109
       109
       +
       	pi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       110
       110
       +
       		PrometheusCounterPrefix: "photocopy_posts",

     

       111
       111
       +
       		Histogram:               insertionsHist,

     

       112
       112
       +
       		BatchSize:               300,

     

       113
       113
       +
       		Logger:                  p.logger,

     

       114
       114
       +
       		Conn:                    conn,

     

       115
       115
       +
       		Query:                   "INSERT INTO post (uri, did, rkey, created_at, indexed_at, root_uri, root_did, parent_uri, parent_did, quote_uri, quote_did, lang)",

     

       116
       116
       +
       		RateLimit:               3,

     

       117
       117
       +
       	})

     

       118
       118
       +
       	if err != nil {

     

       119
       119
       +
       		return nil, err

     

       120
       120
       +
       	}

     

       121
       121
       +
       

     

       122
       122
       +
       	ii, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       123
       123
       +
       		PrometheusCounterPrefix: "photocopy_interactions",

     

       124
       124
       +
       		Histogram:               insertionsHist,

     

       125
       125
       +
       		BatchSize:               1000,

     

       126
       126
       +
       		Logger:                  p.logger,

     

       127
       127
       +
       		Conn:                    conn,

     

       128
       128
       +
       		Query:                   "INSERT INTO interaction (uri, did, rkey, kind, created_at, indexed_at, subject_uri, subject_did)",

     

       129
       129
       +
       		RateLimit:               3,

     

       130
       130
       +
       	})

     

       131
       131
       +
       	if err != nil {

     

       132
       132
       +
       		return nil, err

     

       133
       133
       +
       	}

     

       134
       134
       +
       

     

       135
       135
       +
       	ri, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       136
       136
       +
       		PrometheusCounterPrefix: "photocopy_records",

     

       137
       137
       +
       		Histogram:               insertionsHist,

     

       138
       138
       +
       		BatchSize:               2500,

     

       139
       139
       +
       		Logger:                  p.logger,

     

       140
       140
       +
       		Conn:                    conn,

     

       141
       141
       +
       		Query:                   "INSERT INTO record (did, rkey, collection, cid, seq, raw, created_at)",

     

       142
       142
       +
       		RateLimit:               3,

     

       143
       143
       +
       	})

     

       144
       144
       +
       	if err != nil {

     

       145
       145
       +
       		return nil, err

     

       146
       146
       +
       	}

     

       147
       147
       +
       

     

       148
       148
       +
       	di, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       149
       149
       +
       		PrometheusCounterPrefix: "photocopy_deletes",

     

       150
       150
       +
       		Histogram:               insertionsHist,

     

       151
       151
       +
       		BatchSize:               500,

     

       152
       152
       +
       		Logger:                  p.logger,

     

       153
       153
       +
       		Conn:                    conn,

     

       154
       154
       +
       		Query:                   "INSERT INTO delete (did, rkey, created_at)",

     

       155
       155
       +
       		RateLimit:               3,

     

       156
       156
       +
       	})

     

       157
       157
       +
       	if err != nil {

     

       158
       158
       +
       		return nil, err

     

       159
       159
       +
       	}

     

       160
       160
       +
       

     

       161
       161
       +
       	li, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       162
       162
       +
       		PrometheusCounterPrefix: "photocopy_labels",

     

       163
       163
       +
       		Histogram:               insertionsHist,

     

       79
       164
        
       		BatchSize:               100,

     

       80
       165
        
       		Logger:                  p.logger,

     

       81
       166
        
       		Conn:                    conn,

     

       82
       82
       -
       		Query:                   "",

     

       167
       167
       +
       		Query:                   "INSERT INTO post_label (did, rkey, text, label, entity_id, description, topic, created_at)",

     

       168
       168
       +
       		RateLimit:               3,

     

       83
       169
        
       	})

     

       84
       170
        
       	if err != nil {

     

       85
       171
        
       		return nil, err

     

       86
       172
        
       	}

     

       87
       173
        
       

     

       88
       174
        
       	is := &Inserters{

     

       89
       89
       -
       		followsInserter: fi,

     

       175
       175
       +
       		followsInserter:      fi,

     

       176
       176
       +
       		postsInserter:        pi,

     

       177
       177
       +
       		interactionsInserter: ii,

     

       178
       178
       +
       		recordsInserter:      ri,

     

       179
       179
       +
       		deletesInserter:      di,

     

       180
       180
       +
       		labelsInserter:       li,

     

       90
       181
        
       	}

     

       91
       182
        
       

     

       92
       183
        
       	p.inserters = is

     
···

       97
       188
        
       		BatchSize:               100,

     

       98
       189
        
       		Logger:                  args.Logger,

     

       99
       190
        
       		Conn:                    conn,

     

       100
       100
       -
       		Query: `INSERT INTO your_table_name (

     

       191
       191
       +
       		Query: `INSERT INTO plc (

     

       101
       192
        
       			did, cid, nullified, created_at, plc_op_sig, plc_op_prev, plc_op_type,

     

       102
       193
        
       			plc_op_services, plc_op_also_known_as, plc_op_rotation_keys,

     

       103
       103
       -
       			plc_op_verification_methods, plc_tomb_sig, plc_tomb_prev, plc_tomb_type,

     

       194
       194
       +
       			 plc_tomb_sig, plc_tomb_prev, plc_tomb_type,

     

       104
       195
        
       			legacy_op_sig, legacy_op_prev, legacy_op_type, legacy_op_handle,

     

       105
       196
        
       			legacy_op_service, legacy_op_signing_key, legacy_op_recovery_key

     

       106
       197
        
       		)`,

     
···

       121
       212
        
       	p.inserters.plcInserter = plci

     

       122
       213
        
       	p.plcScraper = plcs

     

       123
       214
        
       

     

       215
       215
       +
       	if args.NervanaApiKey != "" && args.NervanaEndpoint != "" {

     

       216
       216
       +
       		p.nervanaClient = nervana.NewClient(args.NervanaEndpoint, args.NervanaApiKey)

     

       217
       217
       +
       	}

     

       218
       218
       +
       

     

       124
       219
        
       	return p, nil

     

       125
       220
        
       }

     

       126
       221
        
       

     

       127
       127
       -
       func (p *Photocopy) Run(baseCtx context.Context) error {

     

       222
       222
       +
       func (p *Photocopy) Run(baseCtx context.Context, withBackfill bool) error {

     

       128
       223
        
       	ctx, cancel := context.WithCancel(baseCtx)

     

       129
       224
        
       

     

       130
       225
        
       	metricsServer := http.NewServeMux()

     
···

       150
       245
        
       		}

     

       151
       246
        
       	}(ctx)

     

       152
       247
        
       

     

       248
       248
       +
       	if withBackfill {

     

       249
       249
       +
       		go func(ctx context.Context) {

     

       250
       250
       +
       			if err := p.runBackfiller(ctx); err != nil {

     

       251
       251
       +
       				panic(fmt.Errorf("error starting backfiller: %w", err))

     

       252
       252
       +
       			}

     

       253
       253
       +
       		}(ctx)

     

       254
       254
       +
       	}

     

       255
       255
       +
       

     

       153
       256
        
       	<-ctx.Done()

     

       154
       257
        
       

     

       155
       258
        
       	if p.inserters != nil {

     
···

       166
       269
        
       					return

     

       167
       270
        
       				}

     

       168
       271
        
       				p.logger.Info("follows inserter closed")

     

       272
       272
       +
       			}()

     

       273
       273
       +
       		}

     

       274
       274
       +
       

     

       275
       275
       +
       		if p.inserters.interactionsInserter != nil {

     

       276
       276
       +
       			p.wg.Add(1)

     

       277
       277
       +
       			go func() {

     

       278
       278
       +
       				defer p.wg.Done()

     

       279
       279
       +
       				if err := p.inserters.interactionsInserter.Close(ctx); err != nil {

     

       280
       280
       +
       					p.logger.Error("failed to close interactions inserter", "error", err)

     

       281
       281
       +
       					return

     

       282
       282
       +
       				}

     

       283
       283
       +
       				p.logger.Info("interactions inserter closed")

     

       284
       284
       +
       			}()

     

       285
       285
       +
       		}

     

       286
       286
       +
       

     

       287
       287
       +
       		if p.inserters.postsInserter != nil {

     

       288
       288
       +
       			p.wg.Add(1)

     

       289
       289
       +
       			go func() {

     

       290
       290
       +
       				defer p.wg.Done()

     

       291
       291
       +
       				if err := p.inserters.postsInserter.Close(ctx); err != nil {

     

       292
       292
       +
       					p.logger.Error("failed to close posts inserter", "error", err)

     

       293
       293
       +
       					return

     

       294
       294
       +
       				}

     

       295
       295
       +
       				p.logger.Info("posts inserter closed")

     

       296
       296
       +
       			}()

     

       297
       297
       +
       		}

     

       298
       298
       +
       

     

       299
       299
       +
       		if p.inserters.recordsInserter != nil {

     

       300
       300
       +
       			p.wg.Add(1)

     

       301
       301
       +
       			go func() {

     

       302
       302
       +
       				defer p.wg.Done()

     

       303
       303
       +
       				if err := p.inserters.recordsInserter.Close(ctx); err != nil {

     

       304
       304
       +
       					p.logger.Error("failed to close records inserter", "error", err)

     

       305
       305
       +
       					return

     

       306
       306
       +
       				}

     

       307
       307
       +
       				p.logger.Info("records inserter closed")

     

       308
       308
       +
       			}()

     

       309
       309
       +
       		}

     

       310
       310
       +
       

     

       311
       311
       +
       		if p.inserters.deletesInserter != nil {

     

       312
       312
       +
       			p.wg.Add(1)

     

       313
       313
       +
       			go func() {

     

       314
       314
       +
       				defer p.wg.Done()

     

       315
       315
       +
       				if err := p.inserters.deletesInserter.Close(ctx); err != nil {

     

       316
       316
       +
       					p.logger.Error("failed to close deletes inserter", "error", err)

     

       317
       317
       +
       					return

     

       318
       318
       +
       				}

     

       319
       319
       +
       				p.logger.Info("deletes inserter closed")

     

       169
       320
        
       			}()

     

       170
       321
        
       		}

     

       171
       322

+31 -34

plc_models.go

···

       53
       53
        
       }

     

       54
       54
        
       

     

       55
       55
        
       type ClickhousePLCEntry struct {

     

       56
       56
       -
       	Did                      string    `ch:"did"`

     

       57
       57
       -
       	Cid                      string    `ch:"cid"`

     

       58
       58
       -
       	Nullified                bool      `ch:"nullified"`

     

       59
       59
       -
       	CreatedAt                time.Time `ch:"created_at"`

     

       60
       60
       -
       	PlcOpSig                 string    `ch:"plc_op_sig"`

     

       61
       61
       -
       	PlcOpPrev                *string   `ch:"plc_op_prev"`

     

       62
       62
       -
       	PlcOpType                string    `ch:"plc_op_type"`

     

       63
       63
       -
       	PlcOpServices            string    `ch:"plc_op_services"`

     

       64
       64
       -
       	PlcOpAlsoKnownAs         []string  `ch:"plc_op_also_known_as"`

     

       65
       65
       -
       	PlcOpRotationKeys        []string  `ch:"plc_op_rotation_keys"`

     

       66
       66
       -
       	PlcOpVerificationMethods string    `ch:"plc_op_verification_methods"`

     

       67
       67
       -
       	PlcTombSig               string    `ch:"plc_tomb_sig"`

     

       68
       68
       -
       	PlcTombPrev              string    `ch:"plc_tomb_prev"`

     

       69
       69
       -
       	PlcTombType              string    `ch:"plc_tomb_type"`

     

       70
       70
       -
       	LegacyOpSig              string    `ch:"legacy_op_sig"`

     

       71
       71
       -
       	LegacyOpPrev             string    `ch:"legacy_op_prev"`

     

       72
       72
       -
       	LegacyOpType             string    `ch:"legacy_op_type"`

     

       73
       73
       -
       	LegacyOpHandle           string    `ch:"legacy_op_handle"`

     

       74
       74
       -
       	LegacyOpService          string    `ch:"legacy_op_service"`

     

       75
       75
       -
       	LegacyOpSigningKey       string    `ch:"legacy_op_signing_key"`

     

       76
       76
       -
       	LegacyOpRecoveryKey      string    `ch:"legacy_op_recovery_key"`

     

       56
       56
       +
       	Did                 string    `ch:"did"`

     

       57
       57
       +
       	Cid                 string    `ch:"cid"`

     

       58
       58
       +
       	Nullified           bool      `ch:"nullified"`

     

       59
       59
       +
       	CreatedAt           time.Time `ch:"created_at"`

     

       60
       60
       +
       	PlcOpSig            string    `ch:"plc_op_sig"`

     

       61
       61
       +
       	PlcOpPrev           string    `ch:"plc_op_prev"`

     

       62
       62
       +
       	PlcOpType           string    `ch:"plc_op_type"`

     

       63
       63
       +
       	PlcOpServices       []string  `ch:"plc_op_services"`

     

       64
       64
       +
       	PlcOpAlsoKnownAs    []string  `ch:"plc_op_also_known_as"`

     

       65
       65
       +
       	PlcOpRotationKeys   []string  `ch:"plc_op_rotation_keys"`

     

       66
       66
       +
       	PlcTombSig          string    `ch:"plc_tomb_sig"`

     

       67
       67
       +
       	PlcTombPrev         string    `ch:"plc_tomb_prev"`

     

       68
       68
       +
       	PlcTombType         string    `ch:"plc_tomb_type"`

     

       69
       69
       +
       	LegacyOpSig         string    `ch:"legacy_op_sig"`

     

       70
       70
       +
       	LegacyOpPrev        string    `ch:"legacy_op_prev"`

     

       71
       71
       +
       	LegacyOpType        string    `ch:"legacy_op_type"`

     

       72
       72
       +
       	LegacyOpHandle      string    `ch:"legacy_op_handle"`

     

       73
       73
       +
       	LegacyOpService     string    `ch:"legacy_op_service"`

     

       74
       74
       +
       	LegacyOpSigningKey  string    `ch:"legacy_op_signing_key"`

     

       75
       75
       +
       	LegacyOpRecoveryKey string    `ch:"legacy_op_recovery_key"`

     

       77
       76
        
       }

     

       78
       77
        
       

     

       79
       78
        
       func (o *PLCOperationType) UnmarshalJSON(data []byte) error {

     
···

       134
       133
        
       	if e.Operation.PLCOperation != nil {

     

       135
       134
        
       		pop := e.Operation.PLCOperation

     

       136
       135
        
       		che.PlcOpSig = pop.Sig

     

       137
       137
       -
       		che.PlcOpPrev = pop.Prev

     

       136
       136
       +
       		if pop.Prev != nil {

     

       137
       137
       +
       			che.PlcOpPrev = *pop.Prev

     

       138
       138
       +
       		}

     

       138
       139
        
       		che.PlcOpType = pop.Type

     

       140
       140
       +
       		services := []string{}

     

       141
       141
       +
       		for _, s := range pop.Services {

     

       142
       142
       +
       			services = append(services, s.Endpoint)

     

       143
       143
       +
       		}

     

       144
       144
       +
       		che.PlcOpServices = services

     

       139
       145
        
       		che.PlcOpAlsoKnownAs = pop.AlsoKnownAs

     

       140
       146
        
       		che.PlcOpRotationKeys = pop.RotationKeys

     

       141
       141
       -
       		if e.Operation.PLCOperation.Services != nil {

     

       142
       142
       -
       			b, err := json.Marshal(e.Operation.PLCOperation.Services)

     

       143
       143
       -
       			if err != nil {

     

       144
       144
       -
       				return nil, fmt.Errorf("error marshaling services: %w", err)

     

       145
       145
       -
       			}

     

       146
       146
       -
       			che.PlcOpServices = string(b)

     

       147
       147
       -
       		}

     

       148
       148
       -
       		if e.Operation.PLCOperation.VerificationMethods != nil {

     

       149
       149
       -
       			b, err := json.Marshal(e.Operation.PLCOperation.VerificationMethods)

     

       150
       150
       -
       			if err != nil {

     

       151
       151
       -
       				return nil, fmt.Errorf("error marshaling verification methods: %w", err)

     

       147
       147
       +
       		if e.Operation.PLCOperation.Services == nil {

     

       148
       148
       +
       			for _, s := range e.Operation.PLCOperation.Services {

     

       149
       149
       +
       				che.PlcOpServices = append(che.PlcOpServices, s.Endpoint)

     

       152
       150
        
       			}

     

       153
       153
       -
       			che.PlcOpVerificationMethods = string(b)

     

       154
       151
        
       		}

     

       155
       152
        
       		return che, nil

     

       156
       153
        
       	} else if e.Operation.PLCTombstone != nil {

+2 -2

plc_scraper.go

···

       73
       73
        
       			ustr += "&after=" + s.cursor

     

       74
       74
        
       			t, _ := time.Parse(time.RFC3339Nano, s.cursor)

     

       75
       75
        
       			if time.Since(t) > 1*time.Hour {

     

       76
       76
       -
       				setTickerDuration(600 * time.Millisecond)

     

       76
       76
       +
       				setTickerDuration(800 * time.Millisecond)

     

       77
       77
        
       			} else {

     

       78
       78
        
       				setTickerDuration(3 * time.Second)

     

       79
       79
        
       			}

     
···

       135
       135
        
       				continue

     

       136
       136
        
       			}

     

       137
       137
        
       

     

       138
       138
       -
       			s.inserter.Insert(ctx, chEntry)

     

       138
       138
       +
       			s.inserter.Insert(ctx, *chEntry)

     

       139
       139
        
       		}

     

       140
       140
        
       	}

     

       141
       141

Compare changes