comparing 5ed35ecfa067c11f8609fcd84ea2ceca3f87b28d and main on hailey.at/photocopy

Makefile

···

       13
        
       .PHONY: build

     

       14
        
       build: ## Build all executables

     

       15
        
       	go build -ldflags "-X main.Version=$(VERSION)" -o photocopy ./cmd/photocopy

     

       0
        
       
     

       16
        
       

     

       17
        
       .PHONY: run

     

       18
        
       run:

···

       13
        
       .PHONY: build

     

       14
        
       build: ## Build all executables

     

       15
        
       	go build -ldflags "-X main.Version=$(VERSION)" -o photocopy ./cmd/photocopy

     

       16
       +
       	go build -o bodega ./cmd/bodega

     

       17
        
       

     

       18
        
       .PHONY: run

     

       19
        
       run:

+445

backfiller.go

···

       1
       +
       package photocopy

     

       2
       +
       

     

       3
       +
       import (

     

       4
       +
       	"bytes"

     

       5
       +
       	"context"

     

       6
       +
       	"encoding/json"

     

       7
       +
       	"fmt"

     

       8
       +
       	"io"

     

       9
       +
       	"net/http"

     

       10
       +
       	"runtime"

     

       11
       +
       	"strings"

     

       12
       +
       	"sync"

     

       13
       +
       	"time"

     

       14
       +
       

     

       15
       +
       	atproto_repo "github.com/bluesky-social/indigo/atproto/repo"

     

       16
       +
       	"github.com/bluesky-social/indigo/repo"

     

       17
       +
       	"github.com/bluesky-social/indigo/util"

     

       18
       +
       	"github.com/ipfs/go-cid"

     

       19
       +
       	"github.com/ipld/go-car"

     

       20
       +
       	_ "github.com/joho/godotenv/autoload"

     

       21
       +
       	"go.uber.org/ratelimit"

     

       22
       +
       )

     

       23
       +
       

     

       24
       +
       type ProcessJob struct {

     

       25
       +
       	did       string

     

       26
       +
       	repoBytes []byte

     

       27
       +
       }

     

       28
       +
       

     

       29
       +
       type RepoDownloader struct {

     

       30
       +
       	clients     map[string]*http.Client

     

       31
       +
       	rateLimits  map[string]ratelimit.Limiter

     

       32
       +
       	processChan chan ProcessJob

     

       33
       +
       	mu          sync.RWMutex

     

       34
       +
       	p           *Photocopy

     

       35
       +
       }

     

       36
       +
       

     

       37
       +
       func NewRepoDownloader(p *Photocopy) *RepoDownloader {

     

       38
       +
       	return &RepoDownloader{

     

       39
       +
       		clients:     make(map[string]*http.Client),

     

       40
       +
       		rateLimits:  make(map[string]ratelimit.Limiter),

     

       41
       +
       		p:           p,

     

       42
       +
       		processChan: make(chan ProcessJob, 1000),

     

       43
       +
       	}

     

       44
       +
       }

     

       45
       +
       

     

       46
       +
       func (rd *RepoDownloader) getClient(service string) *http.Client {

     

       47
       +
       	rd.mu.RLock()

     

       48
       +
       	client, exists := rd.clients[service]

     

       49
       +
       	rd.mu.RUnlock()

     

       50
       +
       

     

       51
       +
       	if exists {

     

       52
       +
       		return client

     

       53
       +
       	}

     

       54
       +
       

     

       55
       +
       	rd.mu.Lock()

     

       56
       +
       	defer rd.mu.Unlock()

     

       57
       +
       

     

       58
       +
       	if client, exists := rd.clients[service]; exists {

     

       59
       +
       		return client

     

       60
       +
       	}

     

       61
       +
       

     

       62
       +
       	client = util.RobustHTTPClient()

     

       63
       +
       	client.Timeout = 45 * time.Second

     

       64
       +
       	rd.clients[service] = client

     

       65
       +
       	return client

     

       66
       +
       }

     

       67
       +
       

     

       68
       +
       func (rd *RepoDownloader) getRateLimiter(service string) ratelimit.Limiter {

     

       69
       +
       	if !strings.HasSuffix(service, ".bsky.network") {

     

       70
       +
       		service = "third-party"

     

       71
       +
       	}

     

       72
       +
       

     

       73
       +
       	rd.mu.RLock()

     

       74
       +
       	limiter, exists := rd.rateLimits[service]

     

       75
       +
       	rd.mu.RUnlock()

     

       76
       +
       

     

       77
       +
       	if exists {

     

       78
       +
       		return limiter

     

       79
       +
       	}

     

       80
       +
       

     

       81
       +
       	rd.mu.Lock()

     

       82
       +
       	defer rd.mu.Unlock()

     

       83
       +
       

     

       84
       +
       	if limiter, exists := rd.rateLimits[service]; exists {

     

       85
       +
       		return limiter

     

       86
       +
       	}

     

       87
       +
       

     

       88
       +
       	// 3000 per five minutes

     

       89
       +
       	limiter = ratelimit.New(10)

     

       90
       +
       	rd.rateLimits[service] = limiter

     

       91
       +
       	return limiter

     

       92
       +
       }

     

       93
       +
       

     

       94
       +
       func (rd *RepoDownloader) downloadRepo(service, did string) ([]byte, error) {

     

       95
       +
       	dlurl := fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo?did=%s", service, did)

     

       96
       +
       

     

       97
       +
       	req, err := http.NewRequestWithContext(context.TODO(), "GET", dlurl, nil)

     

       98
       +
       	if err != nil {

     

       99
       +
       		return nil, fmt.Errorf("failed to create request: %w", err)

     

       100
       +
       	}

     

       101
       +
       

     

       102
       +
       	if rd.p.ratelimitBypassKey != "" && strings.HasSuffix(service, ".bsky.network") {

     

       103
       +
       		req.Header.Set("x-ratelimit-bypass", rd.p.ratelimitBypassKey)

     

       104
       +
       	}

     

       105
       +
       

     

       106
       +
       	client := rd.getClient(service)

     

       107
       +
       

     

       108
       +
       	resp, err := client.Do(req)

     

       109
       +
       	if err != nil {

     

       110
       +
       		return nil, fmt.Errorf("failed to download repo: %w", err)

     

       111
       +
       	}

     

       112
       +
       	defer resp.Body.Close()

     

       113
       +
       

     

       114
       +
       	if resp.StatusCode != http.StatusOK {

     

       115
       +
       		if resp.StatusCode == 400 {

     

       116
       +
       			return nil, nil

     

       117
       +
       		}

     

       118
       +
       		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)

     

       119
       +
       	}

     

       120
       +
       

     

       121
       +
       	b, err := io.ReadAll(resp.Body)

     

       122
       +
       	if err != nil {

     

       123
       +
       		return nil, fmt.Errorf("could not read bytes from response: %w", err)

     

       124
       +
       	}

     

       125
       +
       

     

       126
       +
       	return b, nil

     

       127
       +
       }

     

       128
       +
       

     

       129
       +
       func (p *Photocopy) processRepo(ctx context.Context, b []byte, did string) error {

     

       130
       +
       	bs := atproto_repo.NewTinyBlockstore()

     

       131
       +
       	cs, err := car.NewCarReader(bytes.NewReader(b))

     

       132
       +
       	if err != nil {

     

       133
       +
       		fmt.Println("error opening car", err)

     

       134
       +
       		return fmt.Errorf("error opening car: %v\n", err)

     

       135
       +
       	}

     

       136
       +
       

     

       137
       +
       	currBlock, _ := cs.Next()

     

       138
       +
       	for currBlock != nil {

     

       139
       +
       		bs.Put(context.TODO(), currBlock)

     

       140
       +
       		next, _ := cs.Next()

     

       141
       +
       		currBlock = next

     

       142
       +
       	}

     

       143
       +
       

     

       144
       +
       	r, err := repo.OpenRepo(context.TODO(), bs, cs.Header.Roots[0])

     

       145
       +
       	if err != nil || r == nil {

     

       146
       +
       		fmt.Println("error opening repo", err)

     

       147
       +
       		fmt.Printf("could not open repo: %v", err)

     

       148
       +
       		return nil

     

       149
       +
       	}

     

       150
       +
       

     

       151
       +
       	if err := r.ForEach(context.TODO(), "", func(key string, cid cid.Cid) error {

     

       152
       +
       		pts := strings.Split(key, "/")

     

       153
       +
       		nsid := pts[0]

     

       154
       +
       		rkey := pts[1]

     

       155
       +
       		cidStr := cid.String()

     

       156
       +
       		b, err := bs.Get(context.TODO(), cid)

     

       157
       +
       		if err != nil {

     

       158
       +
       			return nil

     

       159
       +
       		}

     

       160
       +
       		if err := p.handleCreate(ctx, b.RawData(), time.Now().Format(time.RFC3339Nano), "unk", did, nsid, rkey, cidStr, "unk"); err != nil {

     

       161
       +
       			return err

     

       162
       +
       		}

     

       163
       +
       		return nil

     

       164
       +
       	}); err != nil {

     

       165
       +
       		return fmt.Errorf("erorr traversing records: %v", err)

     

       166
       +
       	}

     

       167
       +
       

     

       168
       +
       	return nil

     

       169
       +
       }

     

       170
       +
       

     

       171
       +
       type ListReposResponse struct {

     

       172
       +
       	Cursor string          `json:"cursor"`

     

       173
       +
       	Repos  []ListReposRepo `json:"repos"`

     

       174
       +
       }

     

       175
       +
       

     

       176
       +
       type ListReposRepo struct {

     

       177
       +
       	Did    string  `json:"did"`

     

       178
       +
       	Head   string  `json:"head"`

     

       179
       +
       	Rev    string  `json:"rev"`

     

       180
       +
       	Active bool    `json:"active"`

     

       181
       +
       	Status *string `json:"status,omitempty"`

     

       182
       +
       }

     

       183
       +
       

     

       184
       +
       func (rd *RepoDownloader) getDidsFromService(ctx context.Context, service string) ([]ListReposRepo, error) {

     

       185
       +
       	var cursor string

     

       186
       +
       	var repos []ListReposRepo

     

       187
       +
       	if service == "https://atproto.brid.gy" {

     

       188
       +
       		return nil, nil

     

       189
       +
       	}

     

       190
       +
       	for {

     

       191
       +
       		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/xrpc/com.atproto.sync.listRepos?limit=1000&cursor=%s", service, cursor), nil)

     

       192
       +
       		if err != nil {

     

       193
       +
       			return nil, err

     

       194
       +
       		}

     

       195
       +
       

     

       196
       +
       		if rd.p.ratelimitBypassKey != "" && strings.HasSuffix(service, ".bsky.network") {

     

       197
       +
       			req.Header.Set("x-ratelimit-bypass", rd.p.ratelimitBypassKey)

     

       198
       +
       		}

     

       199
       +
       

     

       200
       +
       		rl := rd.getRateLimiter(service)

     

       201
       +
       		rl.Take()

     

       202
       +
       

     

       203
       +
       		cli := rd.getClient(service)

     

       204
       +
       		resp, err := cli.Do(req)

     

       205
       +
       		if err != nil {

     

       206
       +
       			return nil, err

     

       207
       +
       		}

     

       208
       +
       		defer resp.Body.Close()

     

       209
       +
       

     

       210
       +
       		if resp.StatusCode != http.StatusOK {

     

       211
       +
       			return nil, fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       212
       +
       		}

     

       213
       +
       

     

       214
       +
       		var reposResp ListReposResponse

     

       215
       +
       		if err := json.NewDecoder(resp.Body).Decode(&reposResp); err != nil {

     

       216
       +
       			return nil, fmt.Errorf("error decoding repos response: %w", err)

     

       217
       +
       		}

     

       218
       +
       

     

       219
       +
       		for _, repo := range reposResp.Repos {

     

       220
       +
       			if repo.Status != nil {

     

       221
       +
       				if *repo.Status == "deleted" || *repo.Status == "takendown" || *repo.Status == "deactivated" {

     

       222
       +
       					continue

     

       223
       +
       				}

     

       224
       +
       			}

     

       225
       +
       

     

       226
       +
       			repos = append(repos, repo)

     

       227
       +
       		}

     

       228
       +
       

     

       229
       +
       		if len(reposResp.Repos) != 1000 || reposResp.Cursor == "" {

     

       230
       +
       			break

     

       231
       +
       		}

     

       232
       +
       

     

       233
       +
       		fmt.Printf("cursor %s service %s\n", reposResp.Cursor, service)

     

       234
       +
       

     

       235
       +
       		cursor = reposResp.Cursor

     

       236
       +
       	}

     

       237
       +
       

     

       238
       +
       	return repos, nil

     

       239
       +
       }

     

       240
       +
       

     

       241
       +
       type ListServicesResponse struct {

     

       242
       +
       	Cursor string                     `json:"cursor"`

     

       243
       +
       	Hosts  []ListServicesResponseItem `json:"hosts"`

     

       244
       +
       }

     

       245
       +
       

     

       246
       +
       type ListServicesResponseItem struct {

     

       247
       +
       	Hostname string `json:"hostname"`

     

       248
       +
       	Status   string `json:"status"`

     

       249
       +
       }

     

       250
       +
       

     

       251
       +
       func (p *Photocopy) runProcessRepoWorker(ctx context.Context, jobs <-chan ProcessJob) {

     

       252
       +
       	for j := range jobs {

     

       253
       +
       		p.processRepo(ctx, j.repoBytes, j.did)

     

       254
       +
       	}

     

       255
       +
       }

     

       256
       +
       

     

       257
       +
       func (p *Photocopy) runBackfiller(ctx context.Context) error {

     

       258
       +
       	startTime := time.Now()

     

       259
       +
       

     

       260
       +
       	fmt.Println("querying clickhouse for dids and services...")

     

       261
       +
       

     

       262
       +
       	type alreadyFetchedItem struct {

     

       263
       +
       		Did string `ch:"did"`

     

       264
       +
       	}

     

       265
       +
       	var alreadyFetched []alreadyFetchedItem

     

       266
       +
       	if err := p.conn.Select(ctx, &alreadyFetched, "SELECT DISTINCT(did) FROM default.record WHERE created_at < '2025-07-01'"); err != nil {

     

       267
       +
       		return err

     

       268
       +
       	}

     

       269
       +
       

     

       270
       +
       	alreadyFetchedMap := map[string]bool{}

     

       271
       +
       	for _, d := range alreadyFetched {

     

       272
       +
       		alreadyFetchedMap[d.Did] = true

     

       273
       +
       	}

     

       274
       +
       

     

       275
       +
       	fmt.Println("getting dids")

     

       276
       +
       

     

       277
       +
       	var hostsCursor string

     

       278
       +
       	var sevs []ListServicesResponseItem

     

       279
       +
       	for {

     

       280
       +
       		if hostsCursor != "" {

     

       281
       +
       			hostsCursor = "&cursor=" + hostsCursor

     

       282
       +
       		}

     

       283
       +
       		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://relay1.us-east.bsky.network/xrpc/com.atproto.sync.listHosts?limit=1000%s", hostsCursor), nil)

     

       284
       +
       		if err != nil {

     

       285
       +
       			return err

     

       286
       +
       		}

     

       287
       +
       

     

       288
       +
       		resp, err := http.DefaultClient.Do(req)

     

       289
       +
       		if err != nil {

     

       290
       +
       			return err

     

       291
       +
       		}

     

       292
       +
       		defer resp.Body.Close()

     

       293
       +
       

     

       294
       +
       		if resp.StatusCode != http.StatusOK {

     

       295
       +
       			return fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       296
       +
       		}

     

       297
       +
       

     

       298
       +
       		var sevsResp ListServicesResponse

     

       299
       +
       		if err := json.NewDecoder(resp.Body).Decode(&sevsResp); err != nil {

     

       300
       +
       			return fmt.Errorf("error decoding sevs response: %w", err)

     

       301
       +
       		}

     

       302
       +
       

     

       303
       +
       		for _, sev := range sevsResp.Hosts {

     

       304
       +
       			if sev.Status != "active" {

     

       305
       +
       				continue

     

       306
       +
       			}

     

       307
       +
       

     

       308
       +
       			sevs = append(sevs, sev)

     

       309
       +
       		}

     

       310
       +
       

     

       311
       +
       		if len(sevsResp.Hosts) != 1000 || sevsResp.Cursor == "" {

     

       312
       +
       			break

     

       313
       +
       		}

     

       314
       +
       

     

       315
       +
       		hostsCursor = sevsResp.Cursor

     

       316
       +
       	}

     

       317
       +
       

     

       318
       +
       	servicesDids := map[string][]string{}

     

       319
       +
       	for _, sev := range sevs {

     

       320
       +
       		servicesDids["https://"+sev.Hostname] = []string{}

     

       321
       +
       	}

     

       322
       +
       

     

       323
       +
       	fmt.Printf("found %d services\n", len(servicesDids))

     

       324
       +
       

     

       325
       +
       	fmt.Printf("collecting dids...\n")

     

       326
       +
       

     

       327
       +
       	fmt.Printf("building download buckets...")

     

       328
       +
       

     

       329
       +
       	skipped := 0

     

       330
       +
       	downloader := NewRepoDownloader(p)

     

       331
       +
       	serviceDids := map[string][]string{}

     

       332
       +
       

     

       333
       +
       	for range runtime.NumCPU() / 2 {

     

       334
       +
       		go p.runProcessRepoWorker(ctx, downloader.processChan)

     

       335
       +
       	}

     

       336
       +
       

     

       337
       +
       	wg := sync.WaitGroup{}

     

       338
       +
       	mplk := sync.Mutex{}

     

       339
       +
       	for s := range servicesDids {

     

       340
       +
       		wg.Add(1)

     

       341
       +
       		go func() {

     

       342
       +
       			defer wg.Done()

     

       343
       +
       			repos, err := downloader.getDidsFromService(context.TODO(), s)

     

       344
       +
       			if err != nil {

     

       345
       +
       				fmt.Printf("error getting dids for services %s: %v", s, err)

     

       346
       +
       				return

     

       347
       +
       			}

     

       348
       +
       			dids := []string{}

     

       349
       +
       			for _, r := range repos {

     

       350
       +
       				if alreadyFetchedMap[r.Did] {

     

       351
       +
       					skipped++

     

       352
       +
       					continue

     

       353
       +
       				}

     

       354
       +
       				dids = append(dids, r.Did)

     

       355
       +
       			}

     

       356
       +
       			mplk.Lock()

     

       357
       +
       			defer mplk.Unlock()

     

       358
       +
       			serviceDids[s] = dids

     

       359
       +
       		}()

     

       360
       +
       	}

     

       361
       +
       

     

       362
       +
       	fmt.Println("getting all the repos...")

     

       363
       +
       	wg.Wait()

     

       364
       +
       

     

       365
       +
       	fmt.Printf("was able to skip %d repos\n", skipped)

     

       366
       +
       

     

       367
       +
       	total := 0

     

       368
       +
       

     

       369
       +
       	for service, dids := range serviceDids {

     

       370
       +
       		if len(dids) < 100 {

     

       371
       +
       			continue

     

       372
       +
       		}

     

       373
       +
       		fmt.Printf("%s: %d jobs\n", service, len(dids))

     

       374
       +
       		total += len(dids)

     

       375
       +
       	}

     

       376
       +
       

     

       377
       +
       	fmt.Printf("Total jobs: %d across %d services \n", total, len(serviceDids))

     

       378
       +
       

     

       379
       +
       	for _, c := range downloader.clients {

     

       380
       +
       		c.Timeout = 10 * time.Minute

     

       381
       +
       	}

     

       382
       +
       

     

       383
       +
       	for s := range downloader.rateLimits {

     

       384
       +
       		if p.ratelimitBypassKey != "" && strings.HasSuffix(s, ".bsky.network") {

     

       385
       +
       			downloader.rateLimits[s] = ratelimit.New(25)

     

       386
       +
       		}

     

       387
       +
       	}

     

       388
       +
       

     

       389
       +
       	processed := 0

     

       390
       +
       	errored := 0

     

       391
       +
       	var errors []error

     

       392
       +
       	for service, dids := range serviceDids {

     

       393
       +
       		go func() {

     

       394
       +
       			for _, did := range dids {

     

       395
       +
       				ratelimiter := downloader.getRateLimiter(service)

     

       396
       +
       				ratelimiter.Take()

     

       397
       +
       

     

       398
       +
       				b, err := downloader.downloadRepo(service, did)

     

       399
       +
       				if err != nil {

     

       400
       +
       					errored++

     

       401
       +
       					processed++

     

       402
       +
       					errors = append(errors, err)

     

       403
       +
       					continue

     

       404
       +
       				}

     

       405
       +
       

     

       406
       +
       				go func(b []byte, did string) {

     

       407
       +
       					downloader.processChan <- ProcessJob{repoBytes: b, did: did}

     

       408
       +
       				}(b, did)

     

       409
       +
       

     

       410
       +
       				processed++

     

       411
       +
       			}

     

       412
       +
       		}()

     

       413
       +
       	}

     

       414
       +
       

     

       415
       +
       	ticker := time.NewTicker(1 * time.Second)

     

       416
       +
       	defer ticker.Stop()

     

       417
       +
       

     

       418
       +
       	for range ticker.C {

     

       419
       +
       		elapsed := time.Since(startTime)

     

       420
       +
       		rate := float64(processed) / elapsed.Seconds()

     

       421
       +
       		remaining := total - processed

     

       422
       +
       

     

       423
       +
       		var eta string

     

       424
       +
       		if rate > 0 {

     

       425
       +
       			etaSeconds := float64(remaining) / rate

     

       426
       +
       			etaDuration := time.Duration(etaSeconds * float64(time.Second))

     

       427
       +
       			eta = fmt.Sprintf(", ETA: %v", etaDuration.Round(time.Second))

     

       428
       +
       		} else {

     

       429
       +
       			eta = ", ETA: calculating..."

     

       430
       +
       		}

     

       431
       +
       

     

       432
       +
       		for _, err := range errors {

     

       433
       +
       			fmt.Printf("%v\n", err)

     

       434
       +
       		}

     

       435
       +
       

     

       436
       +
       		errors = nil

     

       437
       +
       

     

       438
       +
       		fmt.Printf("\rProgress: %d/%d processed (%.1f%%), %d skipped, %d errors, %.1f jobs/sec%s",

     

       439
       +
       			processed, total, float64(processed)/float64(total)*100, skipped, errored, rate, eta)

     

       440
       +
       	}

     

       441
       +
       

     

       442
       +
       	fmt.Printf("\nCompleted: %d processed, %d errors\n", processed, errored)

     

       443
       +
       

     

       444
       +
       	return nil

     

       445
       +
       }

+21 -5

clickhouse_inserter/inserter.go

···

       11
        
       	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"

     

       12
        
       	"github.com/prometheus/client_golang/prometheus"

     

       13
        
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       0
        
       
     

       14
        
       )

     

       15
        
       

     

       16
        
       type Inserter struct {

     
···

       24
        
       	histogram      *prometheus.HistogramVec

     

       25
        
       	logger         *slog.Logger

     

       26
        
       	prefix         string

     

       0
        
       
     

       27
        
       }

     

       28
        
       

     

       29
        
       type Args struct {

     
···

       33
        
       	PrometheusCounterPrefix string

     

       34
        
       	Logger                  *slog.Logger

     

       35
        
       	Histogram               *prometheus.HistogramVec

     

       0
        
       
     

       36
        
       }

     

       37
        
       

     

       38
        
       func New(ctx context.Context, args *Args) (*Inserter, error) {

     
···

       48
        
       		histogram: args.Histogram,

     

       49
        
       		logger:    args.Logger,

     

       50
        
       		prefix:    args.PrometheusCounterPrefix,

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       51
        
       	}

     

       52
        
       

     

       53
        
       	if args.PrometheusCounterPrefix != "" {

     
···

       110
        
       }

     

       111
        
       

     

       112
        
       func (i *Inserter) sendStream(ctx context.Context, toInsert []any) {

     

       113
       -
       	i.pendingSends.Inc()

     

       114
       -
       	defer i.pendingSends.Dec()

     

       0
        
       
     

       0
        
       
     

       115
        
       

     

       116
        
       	if i.histogram != nil {

     

       117
        
       		start := time.Now()

     
···

       125
        
       	}

     

       126
        
       

     

       127
        
       	status := "ok"

     

       128
       -
       	defer func() {

     

       129
       -
       		i.insertsCounter.WithLabelValues(status).Add(float64(len(toInsert)))

     

       130
       -
       	}()

     

       0
        
       
     

       0
        
       
     

       131
        
       

     

       132
        
       	batch, err := i.conn.PrepareBatch(ctx, i.query)

     

       133
        
       	if err != nil {

     
···

       154
        
       		if err := batch.AppendStruct(structPtr); err != nil {

     

       155
        
       			i.logger.Error("error appending to batch", "prefix", i.prefix, "error", err)

     

       156
        
       		}

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       157
        
       	}

     

       158
        
       

     

       159
        
       	if err := batch.Send(); err != nil {

···

       11
        
       	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"

     

       12
        
       	"github.com/prometheus/client_golang/prometheus"

     

       13
        
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       14
       +
       	"go.uber.org/ratelimit"

     

       15
        
       )

     

       16
        
       

     

       17
        
       type Inserter struct {

     
···

       25
        
       	histogram      *prometheus.HistogramVec

     

       26
        
       	logger         *slog.Logger

     

       27
        
       	prefix         string

     

       28
       +
       	rateLimit      ratelimit.Limiter

     

       29
        
       }

     

       30
        
       

     

       31
        
       type Args struct {

     
···

       35
        
       	PrometheusCounterPrefix string

     

       36
        
       	Logger                  *slog.Logger

     

       37
        
       	Histogram               *prometheus.HistogramVec

     

       38
       +
       	RateLimit               int

     

       39
        
       }

     

       40
        
       

     

       41
        
       func New(ctx context.Context, args *Args) (*Inserter, error) {

     
···

       51
        
       		histogram: args.Histogram,

     

       52
        
       		logger:    args.Logger,

     

       53
        
       		prefix:    args.PrometheusCounterPrefix,

     

       54
       +
       	}

     

       55
       +
       

     

       56
       +
       	if args.RateLimit != 0 {

     

       57
       +
       		rateLimit := ratelimit.New(args.RateLimit)

     

       58
       +
       		inserter.rateLimit = rateLimit

     

       59
        
       	}

     

       60
        
       

     

       61
        
       	if args.PrometheusCounterPrefix != "" {

     
···

       118
        
       }

     

       119
        
       

     

       120
        
       func (i *Inserter) sendStream(ctx context.Context, toInsert []any) {

     

       121
       +
       	if i.pendingSends != nil {

     

       122
       +
       		i.pendingSends.Inc()

     

       123
       +
       		defer i.pendingSends.Dec()

     

       124
       +
       	}

     

       125
        
       

     

       126
        
       	if i.histogram != nil {

     

       127
        
       		start := time.Now()

     
···

       135
        
       	}

     

       136
        
       

     

       137
        
       	status := "ok"

     

       138
       +
       	if i.insertsCounter != nil {

     

       139
       +
       		defer func() {

     

       140
       +
       			i.insertsCounter.WithLabelValues(status).Add(float64(len(toInsert)))

     

       141
       +
       		}()

     

       142
       +
       	}

     

       143
        
       

     

       144
        
       	batch, err := i.conn.PrepareBatch(ctx, i.query)

     

       145
        
       	if err != nil {

     
···

       166
        
       		if err := batch.AppendStruct(structPtr); err != nil {

     

       167
        
       			i.logger.Error("error appending to batch", "prefix", i.prefix, "error", err)

     

       168
        
       		}

     

       169
       +
       	}

     

       170
       +
       

     

       171
       +
       	if i.rateLimit != nil {

     

       172
       +
       		i.rateLimit.Take()

     

       173
        
       	}

     

       174
        
       

     

       175
        
       	if err := batch.Send(); err != nil {

+21 -1

cmd/photocopy/main.go

···

       62
        
       				EnvVars:  []string{"PHOTOCOPY_CLICKHOUSE_PASS"},

     

       63
        
       				Required: true,

     

       64
        
       			},

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       65
        
       		},

     

       66
        
       		Commands: cli.Commands{

     

       67
        
       			&cli.Command{

     
···

       112
        
       		ClickhouseDatabase:   cmd.String("clickhouse-database"),

     

       113
        
       		ClickhouseUser:       cmd.String("clickhouse-user"),

     

       114
        
       		ClickhousePass:       cmd.String("clickhouse-pass"),

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       115
        
       	})

     

       116
        
       	if err != nil {

     

       117
        
       		panic(err)

     
···

       124
        
       		sig := <-exitSignals

     

       125
        
       

     

       126
        
       		l.Info("received os exit signal", "signal", sig)

     

       0
        
       
     

       127
        
       	}()

     

       128
        
       

     

       129
       -
       	if err := p.Run(ctx); err != nil {

     

       130
        
       		panic(err)

     

       131
        
       	}

     

       132

···

       62
        
       				EnvVars:  []string{"PHOTOCOPY_CLICKHOUSE_PASS"},

     

       63
        
       				Required: true,

     

       64
        
       			},

     

       65
       +
       			&cli.StringFlag{

     

       66
       +
       				Name:     "ratelimit-bypass-key",

     

       67
       +
       				EnvVars:  []string{"PHOTOCOPY_RATELIMIT_BYPASS_KEY"},

     

       68
       +
       				Required: false,

     

       69
       +
       			},

     

       70
       +
       			&cli.BoolFlag{

     

       71
       +
       				Name: "with-backfill",

     

       72
       +
       			},

     

       73
       +
       			&cli.StringFlag{

     

       74
       +
       				Name:    "nervana-endpoint",

     

       75
       +
       				EnvVars: []string{"PHOTOCOPY_NERVANA_ENDPOINT"},

     

       76
       +
       			},

     

       77
       +
       			&cli.StringFlag{

     

       78
       +
       				Name:    "nervana-api-key",

     

       79
       +
       				EnvVars: []string{"PHOTOCOPY_NERVANA_API_KEY"},

     

       80
       +
       			},

     

       81
        
       		},

     

       82
        
       		Commands: cli.Commands{

     

       83
        
       			&cli.Command{

     
···

       128
        
       		ClickhouseDatabase:   cmd.String("clickhouse-database"),

     

       129
        
       		ClickhouseUser:       cmd.String("clickhouse-user"),

     

       130
        
       		ClickhousePass:       cmd.String("clickhouse-pass"),

     

       131
       +
       		RatelimitBypassKey:   cmd.String("ratelimit-bypass-key"),

     

       132
       +
       		NervanaEndpoint:      cmd.String("nervana-endpoint"),

     

       133
       +
       		NervanaApiKey:        cmd.String("nervana-api-key"),

     

       134
        
       	})

     

       135
        
       	if err != nil {

     

       136
        
       		panic(err)

     
···

       143
        
       		sig := <-exitSignals

     

       144
        
       

     

       145
        
       		l.Info("received os exit signal", "signal", sig)

     

       146
       +
       		cancel()

     

       147
        
       	}()

     

       148
        
       

     

       149
       +
       	if err := p.Run(ctx, cmd.Bool("with-backfill")); err != nil {

     

       150
        
       		panic(err)

     

       151
        
       	}

     

       152

+1 -10

consumer.go

···

       7
        
       	"net/http"

     

       8
        
       	"net/url"

     

       9
        
       	"os"

     

       10
       -
       	"strings"

     

       11
        
       	"time"

     

       12
        
       

     

       13
        
       	"github.com/bluesky-social/indigo/api/atproto"

     
···

       136
        
       				continue

     

       137
        
       			}

     

       138
        
       

     

       139
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       140
       -
       				continue

     

       141
       -
       			}

     

       142
       -
       

     

       143
       -
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String()); err != nil {

     

       144
        
       				p.logger.Error("error handling create event", "error", err)

     

       145
        
       				continue

     

       146
        
       			}

     

       147
        
       		case repomgr.EvtKindDeleteRecord:

     

       148
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       149
       -
       				continue

     

       150
       -
       			}

     

       151
       -
       

     

       152
        
       			if err := p.handleDelete(ctx, did.String(), collection.String(), rkey.String()); err != nil {

     

       153
        
       				p.logger.Error("error handling delete event", "error", err)

     

       154
        
       				continue

···

       7
        
       	"net/http"

     

       8
        
       	"net/url"

     

       9
        
       	"os"

     

       0
        
       
     

       10
        
       	"time"

     

       11
        
       

     

       12
        
       	"github.com/bluesky-social/indigo/api/atproto"

     
···

       135
        
       				continue

     

       136
        
       			}

     

       137
        
       

     

       138
       +
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String(), fmt.Sprintf("%d", evt.Seq)); err != nil {

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       139
        
       				p.logger.Error("error handling create event", "error", err)

     

       140
        
       				continue

     

       141
        
       			}

     

       142
        
       		case repomgr.EvtKindDeleteRecord:

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       143
        
       			if err := p.handleDelete(ctx, did.String(), collection.String(), rkey.String()); err != nil {

     

       144
        
       				p.logger.Error("error handling delete event", "error", err)

     

       145
        
       				continue

+3 -3

go.mod

···

       3
        
       go 1.24.4

     

       4
        
       

     

       5
        
       require (

     

       6
       -
       	github.com/ClickHouse/clickhouse-go v1.5.4

     

       7
        
       	github.com/ClickHouse/clickhouse-go/v2 v2.37.2

     

       8
        
       	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de

     

       9
        
       	github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325

     

       10
        
       	github.com/gorilla/websocket v1.5.1

     

       11
        
       	github.com/ipfs/go-cid v0.5.0

     

       0
        
       
     

       12
        
       	github.com/joho/godotenv v1.5.1

     

       13
        
       	github.com/prometheus/client_golang v1.22.0

     

       14
        
       	github.com/urfave/cli/v2 v2.25.7

     

       0
        
       
     

       15
        
       )

     

       16
        
       

     

       17
        
       require (

     

       18
        
       	github.com/ClickHouse/ch-go v0.66.1 // indirect

     

       19
        
       	github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect

     

       20
        
       	github.com/andybalholm/brotli v1.1.1 // indirect

     

       0
        
       
     

       21
        
       	github.com/beorn7/perks v1.0.1 // indirect

     

       22
        
       	github.com/carlmjohnson/versioninfo v0.22.5 // indirect

     

       23
        
       	github.com/cespare/xxhash/v2 v2.3.0 // indirect

     

       24
       -
       	github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect

     

       25
        
       	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect

     

       26
        
       	github.com/felixge/httpsnoop v1.0.4 // indirect

     

       27
        
       	github.com/go-faster/city v1.0.1 // indirect

     
···

       54
        
       	github.com/ipfs/go-merkledag v0.11.0 // indirect

     

       55
        
       	github.com/ipfs/go-metrics-interface v0.0.1 // indirect

     

       56
        
       	github.com/ipfs/go-verifcid v0.0.3 // indirect

     

       57
       -
       	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect

     

       58
        
       	github.com/ipld/go-codec-dagpb v1.6.0 // indirect

     

       59
        
       	github.com/ipld/go-ipld-prime v0.21.0 // indirect

     

       60
        
       	github.com/jackc/pgpassfile v1.0.0 // indirect

···

       3
        
       go 1.24.4

     

       4
        
       

     

       5
        
       require (

     

       0
        
       
     

       6
        
       	github.com/ClickHouse/clickhouse-go/v2 v2.37.2

     

       7
        
       	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de

     

       8
        
       	github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325

     

       9
        
       	github.com/gorilla/websocket v1.5.1

     

       10
        
       	github.com/ipfs/go-cid v0.5.0

     

       11
       +
       	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4

     

       12
        
       	github.com/joho/godotenv v1.5.1

     

       13
        
       	github.com/prometheus/client_golang v1.22.0

     

       14
        
       	github.com/urfave/cli/v2 v2.25.7

     

       15
       +
       	go.uber.org/ratelimit v0.3.1

     

       16
        
       )

     

       17
        
       

     

       18
        
       require (

     

       19
        
       	github.com/ClickHouse/ch-go v0.66.1 // indirect

     

       20
        
       	github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect

     

       21
        
       	github.com/andybalholm/brotli v1.1.1 // indirect

     

       22
       +
       	github.com/benbjohnson/clock v1.3.0 // indirect

     

       23
        
       	github.com/beorn7/perks v1.0.1 // indirect

     

       24
        
       	github.com/carlmjohnson/versioninfo v0.22.5 // indirect

     

       25
        
       	github.com/cespare/xxhash/v2 v2.3.0 // indirect

     

       0
        
       
     

       26
        
       	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect

     

       27
        
       	github.com/felixge/httpsnoop v1.0.4 // indirect

     

       28
        
       	github.com/go-faster/city v1.0.1 // indirect

     
···

       55
        
       	github.com/ipfs/go-merkledag v0.11.0 // indirect

     

       56
        
       	github.com/ipfs/go-metrics-interface v0.0.1 // indirect

     

       57
        
       	github.com/ipfs/go-verifcid v0.0.3 // indirect

     

       0
        
       
     

       58
        
       	github.com/ipld/go-codec-dagpb v1.6.0 // indirect

     

       59
        
       	github.com/ipld/go-ipld-prime v0.21.0 // indirect

     

       60
        
       	github.com/jackc/pgpassfile v1.0.0 // indirect

+2 -12

go.sum

···

       1
        
       github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

     

       2
        
       github.com/ClickHouse/ch-go v0.66.1 h1:LQHFslfVYZsISOY0dnOYOXGkOUvpv376CCm8g7W74A4=

     

       3
        
       github.com/ClickHouse/ch-go v0.66.1/go.mod h1:NEYcg3aOFv2EmTJfo4m2WF7sHB/YFbLUuIWv9iq76xY=

     

       4
       -
       github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=

     

       5
       -
       github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=

     

       6
        
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2 h1:wRLNKoynvHQEN4znnVHNLaYnrqVc9sGJmGYg+GGCfto=

     

       7
        
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2/go.mod h1:pH2zrBGp5Y438DMwAxXMm1neSXPPjSI7tD4MURVULw8=

     

       8
        
       github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU=

     
···

       20
        
       github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=

     

       21
        
       github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=

     

       22
        
       github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=

     

       23
       -
       github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=

     

       24
       -
       github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=

     

       25
        
       github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325 h1:Bftt2EcoLZK2Z2m12Ih5QqbReX8j29hbf4zJU/FKzaY=

     

       26
        
       github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325/go.mod h1:8FlFpF5cIq3DQG0kEHqyTkPV/5MDQoaWLcVwza5ZPJU=

     

       27
        
       github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=

     
···

       30
        
       github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=

     

       31
        
       github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

     

       32
        
       github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

     

       33
       -
       github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=

     

       34
       -
       github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=

     

       35
        
       github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=

     

       36
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=

     

       37
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

     
···

       57
        
       github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=

     

       58
        
       github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=

     

       59
        
       github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=

     

       60
       -
       github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=

     

       61
        
       github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=

     

       62
        
       github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus=

     

       63
        
       github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4=

     
···

       177
        
       github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=

     

       178
        
       github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=

     

       179
        
       github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=

     

       180
       -
       github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=

     

       181
        
       github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=

     

       182
        
       github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=

     

       183
        
       github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=

     
···

       200
        
       github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=

     

       201
        
       github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=

     

       202
        
       github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=

     

       203
       -
       github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=

     

       204
        
       github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=

     

       205
        
       github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=

     

       206
        
       github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=

     
···

       223
        
       github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=

     

       224
        
       github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=

     

       225
        
       github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=

     

       226
       -
       github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=

     

       227
        
       github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=

     

       228
        
       github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=

     

       229
        
       github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=

     
···

       262
        
       github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=

     

       263
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=

     

       264
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=

     

       265
       -
       github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=

     

       266
       -
       github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

     

       267
        
       github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=

     

       268
        
       github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=

     

       269
        
       github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

     
···

       360
        
       go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=

     

       361
        
       go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

     

       362
        
       go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=

     

       0
        
       
     

       0
        
       
     

       363
        
       go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=

     

       364
        
       go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=

     

       365
        
       go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=

···

       1
        
       github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

     

       2
        
       github.com/ClickHouse/ch-go v0.66.1 h1:LQHFslfVYZsISOY0dnOYOXGkOUvpv376CCm8g7W74A4=

     

       3
        
       github.com/ClickHouse/ch-go v0.66.1/go.mod h1:NEYcg3aOFv2EmTJfo4m2WF7sHB/YFbLUuIWv9iq76xY=

     

       0
        
       
     

       0
        
       
     

       4
        
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2 h1:wRLNKoynvHQEN4znnVHNLaYnrqVc9sGJmGYg+GGCfto=

     

       5
        
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2/go.mod h1:pH2zrBGp5Y438DMwAxXMm1neSXPPjSI7tD4MURVULw8=

     

       6
        
       github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU=

     
···

       18
        
       github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=

     

       19
        
       github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=

     

       20
        
       github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=

     

       0
        
       
     

       0
        
       
     

       21
        
       github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325 h1:Bftt2EcoLZK2Z2m12Ih5QqbReX8j29hbf4zJU/FKzaY=

     

       22
        
       github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325/go.mod h1:8FlFpF5cIq3DQG0kEHqyTkPV/5MDQoaWLcVwza5ZPJU=

     

       23
        
       github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=

     
···

       26
        
       github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=

     

       27
        
       github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

     

       28
        
       github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

     

       0
        
       
     

       0
        
       
     

       29
        
       github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=

     

       30
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=

     

       31
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

     
···

       51
        
       github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=

     

       52
        
       github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=

     

       53
        
       github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=

     

       0
        
       
     

       54
        
       github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=

     

       55
        
       github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus=

     

       56
        
       github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4=

     
···

       170
        
       github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=

     

       171
        
       github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=

     

       172
        
       github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=

     

       0
        
       
     

       173
        
       github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=

     

       174
        
       github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=

     

       175
        
       github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=

     
···

       192
        
       github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=

     

       193
        
       github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=

     

       194
        
       github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=

     

       0
        
       
     

       195
        
       github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=

     

       196
        
       github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=

     

       197
        
       github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=

     
···

       214
        
       github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=

     

       215
        
       github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=

     

       216
        
       github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=

     

       0
        
       
     

       217
        
       github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=

     

       218
        
       github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=

     

       219
        
       github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=

     
···

       252
        
       github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=

     

       253
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=

     

       254
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=

     

       0
        
       
     

       0
        
       
     

       255
        
       github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=

     

       256
        
       github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=

     

       257
        
       github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

     
···

       348
        
       go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=

     

       349
        
       go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

     

       350
        
       go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=

     

       351
       +
       go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0=

     

       352
       +
       go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk=

     

       353
        
       go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=

     

       354
        
       go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=

     

       355
        
       go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=

+67 -2

handle_create.go

···

       4
        
       	"bytes"

     

       5
        
       	"context"

     

       6
        
       	"fmt"

     

       0
        
       
     

       7
        
       	"strings"

     

       8
        
       	"time"

     

       9
        
       

     
···

       13
        
       	"github.com/haileyok/photocopy/models"

     

       14
        
       )

     

       15
        
       

     

       16
       -
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid string) error {

     

       17
       -
       

     

       18
        
       	iat, err := dateparse.ParseAny(indexedAt)

     

       19
        
       	if err != nil {

     

       20
        
       		return err

     

       21
        
       	}

     

       22
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       23
        
       	switch collection {

     

       24
        
       	case "app.bsky.feed.post":

     

       25
        
       		return p.handleCreatePost(ctx, rev, recb, uriFromParts(did, collection, rkey), did, collection, rkey, cid, iat)

     
···

       32
        
       	}

     

       33
        
       }

     

       34
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       35
        
       func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

     

       36
        
       	var rec bsky.FeedPost

     

       37
        
       	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     
···

       43
        
       		return err

     

       44
        
       	}

     

       45
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       46
        
       	post := models.Post{

     

       47
        
       		Uri:       uri,

     

       48
        
       		Rkey:      rkey,

     

       49
        
       		CreatedAt: *cat,

     

       50
        
       		IndexedAt: indexedAt,

     

       51
        
       		Did:       did,

     

       0
        
       
     

       0
        
       
     

       52
        
       	}

     

       53
        
       

     

       54
        
       	if rec.Reply != nil {

     
···

       92
        
       

     

       93
        
       	if err := p.inserters.postsInserter.Insert(ctx, post); err != nil {

     

       94
        
       		return err

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       95
        
       	}

     

       96
        
       

     

       97
        
       	return nil

···

       4
        
       	"bytes"

     

       5
        
       	"context"

     

       6
        
       	"fmt"

     

       7
       +
       	"slices"

     

       8
        
       	"strings"

     

       9
        
       	"time"

     

       10
        
       

     
···

       14
        
       	"github.com/haileyok/photocopy/models"

     

       15
        
       )

     

       16
        
       

     

       17
       +
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid, seq string) error {

     

       0
        
       
     

       18
        
       	iat, err := dateparse.ParseAny(indexedAt)

     

       19
        
       	if err != nil {

     

       20
        
       		return err

     

       21
        
       	}

     

       22
        
       

     

       23
       +
       	if err := p.handleCreateRecord(ctx, did, rkey, collection, cid, recb, seq); err != nil {

     

       24
       +
       		p.logger.Error("error creating record", "error", err)

     

       25
       +
       	}

     

       26
       +
       

     

       27
        
       	switch collection {

     

       28
        
       	case "app.bsky.feed.post":

     

       29
        
       		return p.handleCreatePost(ctx, rev, recb, uriFromParts(did, collection, rkey), did, collection, rkey, cid, iat)

     
···

       36
        
       	}

     

       37
        
       }

     

       38
        
       

     

       39
       +
       func (p *Photocopy) handleCreateRecord(ctx context.Context, did, rkey, collection, cid string, raw []byte, seq string) error {

     

       40
       +
       	var cat time.Time

     

       41
       +
       	prkey, err := syntax.ParseTID(rkey)

     

       42
       +
       	if err == nil {

     

       43
       +
       		cat = prkey.Time()

     

       44
       +
       	} else {

     

       45
       +
       		cat = time.Now()

     

       46
       +
       	}

     

       47
       +
       

     

       48
       +
       	rec := models.Record{

     

       49
       +
       		Did:        did,

     

       50
       +
       		Rkey:       rkey,

     

       51
       +
       		Collection: collection,

     

       52
       +
       		Cid:        cid,

     

       53
       +
       		Seq:        seq,

     

       54
       +
       		Raw:        string(raw),

     

       55
       +
       		CreatedAt:  cat,

     

       56
       +
       	}

     

       57
       +
       

     

       58
       +
       	if err := p.inserters.recordsInserter.Insert(ctx, rec); err != nil {

     

       59
       +
       		return err

     

       60
       +
       	}

     

       61
       +
       

     

       62
       +
       	return nil

     

       63
       +
       }

     

       64
       +
       

     

       65
        
       func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

     

       66
        
       	var rec bsky.FeedPost

     

       67
        
       	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     
···

       73
        
       		return err

     

       74
        
       	}

     

       75
        
       

     

       76
       +
       	lang := ""

     

       77
       +
       	if len(rec.Langs) != 0 {

     

       78
       +
       		lang = rec.Langs[0]

     

       79
       +
       	}

     

       80
       +
       

     

       81
        
       	post := models.Post{

     

       82
        
       		Uri:       uri,

     

       83
        
       		Rkey:      rkey,

     

       84
        
       		CreatedAt: *cat,

     

       85
        
       		IndexedAt: indexedAt,

     

       86
        
       		Did:       did,

     

       87
       +
       		Lang:      lang,

     

       88
       +
       		Text:      rec.Text,

     

       89
        
       	}

     

       90
        
       

     

       91
        
       	if rec.Reply != nil {

     
···

       129
        
       

     

       130
        
       	if err := p.inserters.postsInserter.Insert(ctx, post); err != nil {

     

       131
        
       		return err

     

       132
       +
       	}

     

       133
       +
       

     

       134
       +
       	isEn := slices.Contains(rec.Langs, "en")

     

       135
       +
       	if rec.Text != "" && rec.Reply == nil && isEn && p.nervanaClient != nil {

     

       136
       +
       		go func(ctx context.Context, rec bsky.FeedPost, did, rkey string) {

     

       137
       +
       			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)

     

       138
       +
       			defer cancel()

     

       139
       +
       

     

       140
       +
       			nervanaItems, err := p.nervanaClient.MakeRequest(ctx, rec.Text)

     

       141
       +
       			if err != nil {

     

       142
       +
       				p.logger.Error("error making nervana items request", "error", err)

     

       143
       +
       				return

     

       144
       +
       			}

     

       145
       +
       

     

       146
       +
       			for _, ni := range nervanaItems {

     

       147
       +
       				postLabel := models.PostLabel{

     

       148
       +
       					Did:         did,

     

       149
       +
       					Rkey:        rkey,

     

       150
       +
       					Text:        ni.Text,

     

       151
       +
       					Label:       ni.Label,

     

       152
       +
       					EntityId:    ni.EntityId,

     

       153
       +
       					Description: ni.Description,

     

       154
       +
       					Topic:       "",

     

       155
       +
       					CreatedAt:   time.Now(),

     

       156
       +
       				}

     

       157
       +
       				p.inserters.labelsInserter.Insert(ctx, postLabel)

     

       158
       +
       			}

     

       159
       +
       		}(ctx, rec, did, rkey)

     

       160
        
       	}

     

       161
        
       

     

       162
        
       	return nil

+13

handle_delete.go

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"context"

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       5
        
       )

     

       6
        
       

     

       7
        
       func (p *Photocopy) handleDelete(ctx context.Context, did, collection, rkey string) error {

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       8
        
       	return nil

     

       9
        
       }

···

       2
        
       

     

       3
        
       import (

     

       4
        
       	"context"

     

       5
       +
       	"time"

     

       6
       +
       

     

       7
       +
       	"github.com/haileyok/photocopy/models"

     

       8
        
       )

     

       9
        
       

     

       10
        
       func (p *Photocopy) handleDelete(ctx context.Context, did, collection, rkey string) error {

     

       11
       +
       	del := models.Delete{

     

       12
       +
       		Did:       did,

     

       13
       +
       		Rkey:      rkey,

     

       14
       +
       		CreatedAt: time.Now(),

     

       15
       +
       	}

     

       16
       +
       

     

       17
       +
       	if err := p.inserters.deletesInserter.Insert(ctx, del); err != nil {

     

       18
       +
       		return err

     

       19
       +
       	}

     

       20
       +
       

     

       21
        
       	return nil

     

       22
        
       }

models/delete.go

···

       1
       +
       package models

     

       2
       +
       

     

       3
       +
       import "time"

     

       4
       +
       

     

       5
       +
       type Delete struct {

     

       6
       +
       	Did       string    `ch:"did"`

     

       7
       +
       	Rkey      string    `ch:"rkey"`

     

       8
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       9
       +
       }

models/post.go

···

       14
        
       	ParentDid string    `ch:"parent_did"`

     

       15
        
       	QuoteUri  string    `ch:"quote_uri"`

     

       16
        
       	QuoteDid  string    `ch:"quote_did"`

     

       0
        
       
     

       0
        
       
     

       17
        
       }

···

       14
        
       	ParentDid string    `ch:"parent_did"`

     

       15
        
       	QuoteUri  string    `ch:"quote_uri"`

     

       16
        
       	QuoteDid  string    `ch:"quote_did"`

     

       17
       +
       	Lang      string    `ch:"lang"`

     

       18
       +
       	Text      string    `ch:"text"`

     

       19
        
       }

+14

models/post_label.go

···

       1
       +
       package models

     

       2
       +
       

     

       3
       +
       import "time"

     

       4
       +
       

     

       5
       +
       type PostLabel struct {

     

       6
       +
       	Did         string    `ch:"did"`

     

       7
       +
       	Rkey        string    `ch:"rkey"`

     

       8
       +
       	CreatedAt   time.Time `ch:"created_at"`

     

       9
       +
       	Text        string    `ch:"text"`

     

       10
       +
       	Label       string    `ch:"label"`

     

       11
       +
       	EntityId    string    `ch:"entity_id"`

     

       12
       +
       	Description string    `ch:"description"`

     

       13
       +
       	Topic       string    `ch:"topic"`

     

       14
       +
       }

+13

models/record.go

···

       1
       +
       package models

     

       2
       +
       

     

       3
       +
       import "time"

     

       4
       +
       

     

       5
       +
       type Record struct {

     

       6
       +
       	Did        string    `ch:"did"`

     

       7
       +
       	Rkey       string    `ch:"rkey"`

     

       8
       +
       	Collection string    `ch:"collection"`

     

       9
       +
       	Cid        string    `ch:"cid"`

     

       10
       +
       	Seq        string    `ch:"seq"`

     

       11
       +
       	Raw        string    `ch:"raw"`

     

       12
       +
       	CreatedAt  time.Time `ch:"created_at"`

     

       13
       +
       }

-1

models.go

···

       1
       -
       package photocopy

···

       0

+77

nervana/client.go

···

       1
       +
       package nervana

     

       2
       +
       

     

       3
       +
       import (

     

       4
       +
       	"bytes"

     

       5
       +
       	"context"

     

       6
       +
       	"encoding/json"

     

       7
       +
       	"fmt"

     

       8
       +
       	"io"

     

       9
       +
       	"net/http"

     

       10
       +
       	"time"

     

       11
       +
       )

     

       12
       +
       

     

       13
       +
       type Client struct {

     

       14
       +
       	cli      *http.Client

     

       15
       +
       	endpoint string

     

       16
       +
       	apiKey   string

     

       17
       +
       }

     

       18
       +
       

     

       19
       +
       func NewClient(endpoint string, apiKey string) *Client {

     

       20
       +
       	return &Client{

     

       21
       +
       		cli: &http.Client{

     

       22
       +
       			Timeout: 5 * time.Second,

     

       23
       +
       		},

     

       24
       +
       		endpoint: endpoint,

     

       25
       +
       		apiKey:   apiKey,

     

       26
       +
       	}

     

       27
       +
       }

     

       28
       +
       

     

       29
       +
       type NervanaItem struct {

     

       30
       +
       	Text        string `json:"text"`

     

       31
       +
       	Label       string `json:"label"`

     

       32
       +
       	EntityId    string `json:"entityId"`

     

       33
       +
       	Description string `json:"description"`

     

       34
       +
       }

     

       35
       +
       

     

       36
       +
       func (c *Client) newRequest(ctx context.Context, text string) (*http.Request, error) {

     

       37
       +
       	payload := map[string]string{

     

       38
       +
       		"text":     text,

     

       39
       +
       		"language": "en",

     

       40
       +
       	}

     

       41
       +
       

     

       42
       +
       	b, err := json.Marshal(payload)

     

       43
       +
       	if err != nil {

     

       44
       +
       		return nil, err

     

       45
       +
       	}

     

       46
       +
       

     

       47
       +
       	req, err := http.NewRequestWithContext(ctx, "POST", c.endpoint, bytes.NewReader(b))

     

       48
       +
       

     

       49
       +
       	req.Header.Set("Authorization", "Bearer "+c.apiKey)

     

       50
       +
       

     

       51
       +
       	return req, err

     

       52
       +
       }

     

       53
       +
       

     

       54
       +
       func (c *Client) MakeRequest(ctx context.Context, text string) ([]NervanaItem, error) {

     

       55
       +
       	req, err := c.newRequest(ctx, text)

     

       56
       +
       	if err != nil {

     

       57
       +
       		return nil, err

     

       58
       +
       	}

     

       59
       +
       

     

       60
       +
       	resp, err := c.cli.Do(req)

     

       61
       +
       	if err != nil {

     

       62
       +
       		return nil, err

     

       63
       +
       	}

     

       64
       +
       	defer resp.Body.Close()

     

       65
       +
       

     

       66
       +
       	if resp.StatusCode != 200 {

     

       67
       +
       		io.Copy(io.Discard, resp.Body)

     

       68
       +
       		return nil, fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       69
       +
       	}

     

       70
       +
       

     

       71
       +
       	var nervanaResp []NervanaItem

     

       72
       +
       	if err := json.NewDecoder(resp.Body).Decode(&nervanaResp); err != nil {

     

       73
       +
       		return nil, err

     

       74
       +
       	}

     

       75
       +
       

     

       76
       +
       	return nervanaResp, nil

     

       77
       +
       }

+135 -12

photocopy.go

···

       9
        
       	"time"

     

       10
        
       

     

       11
        
       	"github.com/ClickHouse/clickhouse-go/v2"

     

       0
        
       
     

       12
        
       	"github.com/haileyok/photocopy/clickhouse_inserter"

     

       0
        
       
     

       13
        
       	"github.com/prometheus/client_golang/prometheus"

     

       14
        
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       15
        
       	"github.com/prometheus/client_golang/prometheus/promhttp"

     
···

       27
        
       	inserters *Inserters

     

       28
        
       

     

       29
        
       	plcScraper *PLCScraper

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       30
        
       }

     

       31
        
       

     

       32
        
       type Inserters struct {

     
···

       34
        
       	interactionsInserter *clickhouse_inserter.Inserter

     

       35
        
       	postsInserter        *clickhouse_inserter.Inserter

     

       36
        
       	plcInserter          *clickhouse_inserter.Inserter

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       37
        
       }

     

       38
        
       

     

       39
        
       type Args struct {

     
···

       46
        
       	ClickhouseDatabase   string

     

       47
        
       	ClickhouseUser       string

     

       48
        
       	ClickhousePass       string

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       49
        
       }

     

       50
        
       

     

       51
        
       func New(ctx context.Context, args *Args) (*Photocopy, error) {

     

       52
       -
       	p := &Photocopy{

     

       53
       -
       		logger:      args.Logger,

     

       54
       -
       		metricsAddr: args.MetricsAddr,

     

       55
       -
       		relayHost:   args.RelayHost,

     

       56
       -
       		wg:          sync.WaitGroup{},

     

       57
       -
       		cursorFile:  args.CursorFile,

     

       58
       -
       	}

     

       59
       -
       

     

       60
        
       	conn, err := clickhouse.Open(&clickhouse.Options{

     

       61
        
       		Addr: []string{args.ClickhouseAddr},

     

       62
        
       		Auth: clickhouse.Auth{

     
···

       69
        
       		return nil, err

     

       70
        
       	}

     

       71
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       72
        
       	insertionsHist := promauto.NewHistogramVec(prometheus.HistogramOpts{

     

       73
        
       		Name:    "photocopy_inserts_time",

     

       74
        
       		Help:    "histogram of photocopy inserts",

     
···

       78
        
       	fi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       79
        
       		PrometheusCounterPrefix: "photocopy_follows",

     

       80
        
       		Histogram:               insertionsHist,

     

       81
       -
       		BatchSize:               1000,

     

       82
        
       		Logger:                  p.logger,

     

       83
        
       		Conn:                    conn,

     

       84
        
       		Query:                   "INSERT INTO follow (uri, did, rkey, created_at, indexed_at, subject)",

     

       0
        
       
     

       85
        
       	})

     

       86
        
       	if err != nil {

     

       87
        
       		return nil, err

     
···

       90
        
       	pi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       91
        
       		PrometheusCounterPrefix: "photocopy_posts",

     

       92
        
       		Histogram:               insertionsHist,

     

       93
       -
       		BatchSize:               100,

     

       94
        
       		Logger:                  p.logger,

     

       95
        
       		Conn:                    conn,

     

       96
       -
       		Query:                   "INSERT INTO post (uri, did, rkey, created_at, indexed_at, root_uri, root_did, parent_uri, parent_did, quote_uri, quote_did)",

     

       0
        
       
     

       97
        
       	})

     

       98
        
       	if err != nil {

     

       99
        
       		return nil, err

     
···

       106
        
       		Logger:                  p.logger,

     

       107
        
       		Conn:                    conn,

     

       108
        
       		Query:                   "INSERT INTO interaction (uri, did, rkey, kind, created_at, indexed_at, subject_uri, subject_did)",

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       109
        
       	})

     

       110
        
       	if err != nil {

     

       111
        
       		return nil, err

     
···

       115
        
       		followsInserter:      fi,

     

       116
        
       		postsInserter:        pi,

     

       117
        
       		interactionsInserter: ii,

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       118
        
       	}

     

       119
        
       

     

       120
        
       	p.inserters = is

     
···

       148
        
       

     

       149
        
       	p.inserters.plcInserter = plci

     

       150
        
       	p.plcScraper = plcs

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       151
        
       

     

       152
        
       	return p, nil

     

       153
        
       }

     

       154
        
       

     

       155
       -
       func (p *Photocopy) Run(baseCtx context.Context) error {

     

       156
        
       	ctx, cancel := context.WithCancel(baseCtx)

     

       157
        
       

     

       158
        
       	metricsServer := http.NewServeMux()

     
···

       178
        
       		}

     

       179
        
       	}(ctx)

     

       180
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       181
        
       	<-ctx.Done()

     

       182
        
       

     

       183
        
       	if p.inserters != nil {

     
···

       194
        
       					return

     

       195
        
       				}

     

       196
        
       				p.logger.Info("follows inserter closed")

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       197
        
       			}()

     

       198
        
       		}

     

       199

···

       9
        
       	"time"

     

       10
        
       

     

       11
        
       	"github.com/ClickHouse/clickhouse-go/v2"

     

       12
       +
       	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"

     

       13
        
       	"github.com/haileyok/photocopy/clickhouse_inserter"

     

       14
       +
       	"github.com/haileyok/photocopy/nervana"

     

       15
        
       	"github.com/prometheus/client_golang/prometheus"

     

       16
        
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       17
        
       	"github.com/prometheus/client_golang/prometheus/promhttp"

     
···

       29
        
       	inserters *Inserters

     

       30
        
       

     

       31
        
       	plcScraper *PLCScraper

     

       32
       +
       

     

       33
       +
       	ratelimitBypassKey string

     

       34
       +
       

     

       35
       +
       	conn driver.Conn

     

       36
       +
       

     

       37
       +
       	nervanaClient   *nervana.Client

     

       38
       +
       	nervanaEndpoint string

     

       39
       +
       	nervanaApiKey   string

     

       40
        
       }

     

       41
        
       

     

       42
        
       type Inserters struct {

     
···

       44
        
       	interactionsInserter *clickhouse_inserter.Inserter

     

       45
        
       	postsInserter        *clickhouse_inserter.Inserter

     

       46
        
       	plcInserter          *clickhouse_inserter.Inserter

     

       47
       +
       	recordsInserter      *clickhouse_inserter.Inserter

     

       48
       +
       	deletesInserter      *clickhouse_inserter.Inserter

     

       49
       +
       	labelsInserter       *clickhouse_inserter.Inserter

     

       50
        
       }

     

       51
        
       

     

       52
        
       type Args struct {

     
···

       59
        
       	ClickhouseDatabase   string

     

       60
        
       	ClickhouseUser       string

     

       61
        
       	ClickhousePass       string

     

       62
       +
       	RatelimitBypassKey   string

     

       63
       +
       	NervanaEndpoint      string

     

       64
       +
       	NervanaApiKey        string

     

       65
        
       }

     

       66
        
       

     

       67
        
       func New(ctx context.Context, args *Args) (*Photocopy, error) {

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       68
        
       	conn, err := clickhouse.Open(&clickhouse.Options{

     

       69
        
       		Addr: []string{args.ClickhouseAddr},

     

       70
        
       		Auth: clickhouse.Auth{

     
···

       77
        
       		return nil, err

     

       78
        
       	}

     

       79
        
       

     

       80
       +
       	p := &Photocopy{

     

       81
       +
       		logger:             args.Logger,

     

       82
       +
       		metricsAddr:        args.MetricsAddr,

     

       83
       +
       		relayHost:          args.RelayHost,

     

       84
       +
       		wg:                 sync.WaitGroup{},

     

       85
       +
       		cursorFile:         args.CursorFile,

     

       86
       +
       		ratelimitBypassKey: args.RatelimitBypassKey,

     

       87
       +
       		conn:               conn,

     

       88
       +
       	}

     

       89
       +
       

     

       90
        
       	insertionsHist := promauto.NewHistogramVec(prometheus.HistogramOpts{

     

       91
        
       		Name:    "photocopy_inserts_time",

     

       92
        
       		Help:    "histogram of photocopy inserts",

     
···

       96
        
       	fi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       97
        
       		PrometheusCounterPrefix: "photocopy_follows",

     

       98
        
       		Histogram:               insertionsHist,

     

       99
       +
       		BatchSize:               500,

     

       100
        
       		Logger:                  p.logger,

     

       101
        
       		Conn:                    conn,

     

       102
        
       		Query:                   "INSERT INTO follow (uri, did, rkey, created_at, indexed_at, subject)",

     

       103
       +
       		RateLimit:               3,

     

       104
        
       	})

     

       105
        
       	if err != nil {

     

       106
        
       		return nil, err

     
···

       109
        
       	pi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       110
        
       		PrometheusCounterPrefix: "photocopy_posts",

     

       111
        
       		Histogram:               insertionsHist,

     

       112
       +
       		BatchSize:               300,

     

       113
        
       		Logger:                  p.logger,

     

       114
        
       		Conn:                    conn,

     

       115
       +
       		Query:                   "INSERT INTO post (uri, did, rkey, created_at, indexed_at, root_uri, root_did, parent_uri, parent_did, quote_uri, quote_did, lang)",

     

       116
       +
       		RateLimit:               3,

     

       117
        
       	})

     

       118
        
       	if err != nil {

     

       119
        
       		return nil, err

     
···

       126
        
       		Logger:                  p.logger,

     

       127
        
       		Conn:                    conn,

     

       128
        
       		Query:                   "INSERT INTO interaction (uri, did, rkey, kind, created_at, indexed_at, subject_uri, subject_did)",

     

       129
       +
       		RateLimit:               3,

     

       130
       +
       	})

     

       131
       +
       	if err != nil {

     

       132
       +
       		return nil, err

     

       133
       +
       	}

     

       134
       +
       

     

       135
       +
       	ri, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       136
       +
       		PrometheusCounterPrefix: "photocopy_records",

     

       137
       +
       		Histogram:               insertionsHist,

     

       138
       +
       		BatchSize:               2500,

     

       139
       +
       		Logger:                  p.logger,

     

       140
       +
       		Conn:                    conn,

     

       141
       +
       		Query:                   "INSERT INTO record (did, rkey, collection, cid, seq, raw, created_at)",

     

       142
       +
       		RateLimit:               3,

     

       143
       +
       	})

     

       144
       +
       	if err != nil {

     

       145
       +
       		return nil, err

     

       146
       +
       	}

     

       147
       +
       

     

       148
       +
       	di, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       149
       +
       		PrometheusCounterPrefix: "photocopy_deletes",

     

       150
       +
       		Histogram:               insertionsHist,

     

       151
       +
       		BatchSize:               500,

     

       152
       +
       		Logger:                  p.logger,

     

       153
       +
       		Conn:                    conn,

     

       154
       +
       		Query:                   "INSERT INTO delete (did, rkey, created_at)",

     

       155
       +
       		RateLimit:               3,

     

       156
       +
       	})

     

       157
       +
       	if err != nil {

     

       158
       +
       		return nil, err

     

       159
       +
       	}

     

       160
       +
       

     

       161
       +
       	li, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       162
       +
       		PrometheusCounterPrefix: "photocopy_labels",

     

       163
       +
       		Histogram:               insertionsHist,

     

       164
       +
       		BatchSize:               100,

     

       165
       +
       		Logger:                  p.logger,

     

       166
       +
       		Conn:                    conn,

     

       167
       +
       		Query:                   "INSERT INTO post_label (did, rkey, text, label, entity_id, description, topic, created_at)",

     

       168
       +
       		RateLimit:               3,

     

       169
        
       	})

     

       170
        
       	if err != nil {

     

       171
        
       		return nil, err

     
···

       175
        
       		followsInserter:      fi,

     

       176
        
       		postsInserter:        pi,

     

       177
        
       		interactionsInserter: ii,

     

       178
       +
       		recordsInserter:      ri,

     

       179
       +
       		deletesInserter:      di,

     

       180
       +
       		labelsInserter:       li,

     

       181
        
       	}

     

       182
        
       

     

       183
        
       	p.inserters = is

     
···

       211
        
       

     

       212
        
       	p.inserters.plcInserter = plci

     

       213
        
       	p.plcScraper = plcs

     

       214
       +
       

     

       215
       +
       	if args.NervanaApiKey != "" && args.NervanaEndpoint != "" {

     

       216
       +
       		p.nervanaClient = nervana.NewClient(args.NervanaEndpoint, args.NervanaApiKey)

     

       217
       +
       	}

     

       218
        
       

     

       219
        
       	return p, nil

     

       220
        
       }

     

       221
        
       

     

       222
       +
       func (p *Photocopy) Run(baseCtx context.Context, withBackfill bool) error {

     

       223
        
       	ctx, cancel := context.WithCancel(baseCtx)

     

       224
        
       

     

       225
        
       	metricsServer := http.NewServeMux()

     
···

       245
        
       		}

     

       246
        
       	}(ctx)

     

       247
        
       

     

       248
       +
       	if withBackfill {

     

       249
       +
       		go func(ctx context.Context) {

     

       250
       +
       			if err := p.runBackfiller(ctx); err != nil {

     

       251
       +
       				panic(fmt.Errorf("error starting backfiller: %w", err))

     

       252
       +
       			}

     

       253
       +
       		}(ctx)

     

       254
       +
       	}

     

       255
       +
       

     

       256
        
       	<-ctx.Done()

     

       257
        
       

     

       258
        
       	if p.inserters != nil {

     
···

       269
        
       					return

     

       270
        
       				}

     

       271
        
       				p.logger.Info("follows inserter closed")

     

       272
       +
       			}()

     

       273
       +
       		}

     

       274
       +
       

     

       275
       +
       		if p.inserters.interactionsInserter != nil {

     

       276
       +
       			p.wg.Add(1)

     

       277
       +
       			go func() {

     

       278
       +
       				defer p.wg.Done()

     

       279
       +
       				if err := p.inserters.interactionsInserter.Close(ctx); err != nil {

     

       280
       +
       					p.logger.Error("failed to close interactions inserter", "error", err)

     

       281
       +
       					return

     

       282
       +
       				}

     

       283
       +
       				p.logger.Info("interactions inserter closed")

     

       284
       +
       			}()

     

       285
       +
       		}

     

       286
       +
       

     

       287
       +
       		if p.inserters.postsInserter != nil {

     

       288
       +
       			p.wg.Add(1)

     

       289
       +
       			go func() {

     

       290
       +
       				defer p.wg.Done()

     

       291
       +
       				if err := p.inserters.postsInserter.Close(ctx); err != nil {

     

       292
       +
       					p.logger.Error("failed to close posts inserter", "error", err)

     

       293
       +
       					return

     

       294
       +
       				}

     

       295
       +
       				p.logger.Info("posts inserter closed")

     

       296
       +
       			}()

     

       297
       +
       		}

     

       298
       +
       

     

       299
       +
       		if p.inserters.recordsInserter != nil {

     

       300
       +
       			p.wg.Add(1)

     

       301
       +
       			go func() {

     

       302
       +
       				defer p.wg.Done()

     

       303
       +
       				if err := p.inserters.recordsInserter.Close(ctx); err != nil {

     

       304
       +
       					p.logger.Error("failed to close records inserter", "error", err)

     

       305
       +
       					return

     

       306
       +
       				}

     

       307
       +
       				p.logger.Info("records inserter closed")

     

       308
       +
       			}()

     

       309
       +
       		}

     

       310
       +
       

     

       311
       +
       		if p.inserters.deletesInserter != nil {

     

       312
       +
       			p.wg.Add(1)

     

       313
       +
       			go func() {

     

       314
       +
       				defer p.wg.Done()

     

       315
       +
       				if err := p.inserters.deletesInserter.Close(ctx); err != nil {

     

       316
       +
       					p.logger.Error("failed to close deletes inserter", "error", err)

     

       317
       +
       					return

     

       318
       +
       				}

     

       319
       +
       				p.logger.Info("deletes inserter closed")

     

       320
        
       			}()

     

       321
        
       		}

     

       322

plc_models.go

···

       137
        
       			che.PlcOpPrev = *pop.Prev

     

       138
        
       		}

     

       139
        
       		che.PlcOpType = pop.Type

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       140
        
       		che.PlcOpAlsoKnownAs = pop.AlsoKnownAs

     

       141
        
       		che.PlcOpRotationKeys = pop.RotationKeys

     

       142
        
       		if e.Operation.PLCOperation.Services == nil {

···

       137
        
       			che.PlcOpPrev = *pop.Prev

     

       138
        
       		}

     

       139
        
       		che.PlcOpType = pop.Type

     

       140
       +
       		services := []string{}

     

       141
       +
       		for _, s := range pop.Services {

     

       142
       +
       			services = append(services, s.Endpoint)

     

       143
       +
       		}

     

       144
       +
       		che.PlcOpServices = services

     

       145
        
       		che.PlcOpAlsoKnownAs = pop.AlsoKnownAs

     

       146
        
       		che.PlcOpRotationKeys = pop.RotationKeys

     

       147
        
       		if e.Operation.PLCOperation.Services == nil {

Compare changes