comparing 3b0bbfa446e0c422398b301787cf0b653138e434 and main on hailey.at/photocopy

Makefile

···

       13
       13
        
       .PHONY: build

     

       14
       14
        
       build: ## Build all executables

     

       15
       15
        
       	go build -ldflags "-X main.Version=$(VERSION)" -o photocopy ./cmd/photocopy

     

       16
       16
       +
       	go build -o bodega ./cmd/bodega

     

       16
       17
        
       

     

       17
       18
        
       .PHONY: run

     

       18
       19
        
       run:

+445

backfiller.go

···

       1
       1
       +
       package photocopy

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"bytes"

     

       5
       5
       +
       	"context"

     

       6
       6
       +
       	"encoding/json"

     

       7
       7
       +
       	"fmt"

     

       8
       8
       +
       	"io"

     

       9
       9
       +
       	"net/http"

     

       10
       10
       +
       	"runtime"

     

       11
       11
       +
       	"strings"

     

       12
       12
       +
       	"sync"

     

       13
       13
       +
       	"time"

     

       14
       14
       +
       

     

       15
       15
       +
       	atproto_repo "github.com/bluesky-social/indigo/atproto/repo"

     

       16
       16
       +
       	"github.com/bluesky-social/indigo/repo"

     

       17
       17
       +
       	"github.com/bluesky-social/indigo/util"

     

       18
       18
       +
       	"github.com/ipfs/go-cid"

     

       19
       19
       +
       	"github.com/ipld/go-car"

     

       20
       20
       +
       	_ "github.com/joho/godotenv/autoload"

     

       21
       21
       +
       	"go.uber.org/ratelimit"

     

       22
       22
       +
       )

     

       23
       23
       +
       

     

       24
       24
       +
       type ProcessJob struct {

     

       25
       25
       +
       	did       string

     

       26
       26
       +
       	repoBytes []byte

     

       27
       27
       +
       }

     

       28
       28
       +
       

     

       29
       29
       +
       type RepoDownloader struct {

     

       30
       30
       +
       	clients     map[string]*http.Client

     

       31
       31
       +
       	rateLimits  map[string]ratelimit.Limiter

     

       32
       32
       +
       	processChan chan ProcessJob

     

       33
       33
       +
       	mu          sync.RWMutex

     

       34
       34
       +
       	p           *Photocopy

     

       35
       35
       +
       }

     

       36
       36
       +
       

     

       37
       37
       +
       func NewRepoDownloader(p *Photocopy) *RepoDownloader {

     

       38
       38
       +
       	return &RepoDownloader{

     

       39
       39
       +
       		clients:     make(map[string]*http.Client),

     

       40
       40
       +
       		rateLimits:  make(map[string]ratelimit.Limiter),

     

       41
       41
       +
       		p:           p,

     

       42
       42
       +
       		processChan: make(chan ProcessJob, 1000),

     

       43
       43
       +
       	}

     

       44
       44
       +
       }

     

       45
       45
       +
       

     

       46
       46
       +
       func (rd *RepoDownloader) getClient(service string) *http.Client {

     

       47
       47
       +
       	rd.mu.RLock()

     

       48
       48
       +
       	client, exists := rd.clients[service]

     

       49
       49
       +
       	rd.mu.RUnlock()

     

       50
       50
       +
       

     

       51
       51
       +
       	if exists {

     

       52
       52
       +
       		return client

     

       53
       53
       +
       	}

     

       54
       54
       +
       

     

       55
       55
       +
       	rd.mu.Lock()

     

       56
       56
       +
       	defer rd.mu.Unlock()

     

       57
       57
       +
       

     

       58
       58
       +
       	if client, exists := rd.clients[service]; exists {

     

       59
       59
       +
       		return client

     

       60
       60
       +
       	}

     

       61
       61
       +
       

     

       62
       62
       +
       	client = util.RobustHTTPClient()

     

       63
       63
       +
       	client.Timeout = 45 * time.Second

     

       64
       64
       +
       	rd.clients[service] = client

     

       65
       65
       +
       	return client

     

       66
       66
       +
       }

     

       67
       67
       +
       

     

       68
       68
       +
       func (rd *RepoDownloader) getRateLimiter(service string) ratelimit.Limiter {

     

       69
       69
       +
       	if !strings.HasSuffix(service, ".bsky.network") {

     

       70
       70
       +
       		service = "third-party"

     

       71
       71
       +
       	}

     

       72
       72
       +
       

     

       73
       73
       +
       	rd.mu.RLock()

     

       74
       74
       +
       	limiter, exists := rd.rateLimits[service]

     

       75
       75
       +
       	rd.mu.RUnlock()

     

       76
       76
       +
       

     

       77
       77
       +
       	if exists {

     

       78
       78
       +
       		return limiter

     

       79
       79
       +
       	}

     

       80
       80
       +
       

     

       81
       81
       +
       	rd.mu.Lock()

     

       82
       82
       +
       	defer rd.mu.Unlock()

     

       83
       83
       +
       

     

       84
       84
       +
       	if limiter, exists := rd.rateLimits[service]; exists {

     

       85
       85
       +
       		return limiter

     

       86
       86
       +
       	}

     

       87
       87
       +
       

     

       88
       88
       +
       	// 3000 per five minutes

     

       89
       89
       +
       	limiter = ratelimit.New(10)

     

       90
       90
       +
       	rd.rateLimits[service] = limiter

     

       91
       91
       +
       	return limiter

     

       92
       92
       +
       }

     

       93
       93
       +
       

     

       94
       94
       +
       func (rd *RepoDownloader) downloadRepo(service, did string) ([]byte, error) {

     

       95
       95
       +
       	dlurl := fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo?did=%s", service, did)

     

       96
       96
       +
       

     

       97
       97
       +
       	req, err := http.NewRequestWithContext(context.TODO(), "GET", dlurl, nil)

     

       98
       98
       +
       	if err != nil {

     

       99
       99
       +
       		return nil, fmt.Errorf("failed to create request: %w", err)

     

       100
       100
       +
       	}

     

       101
       101
       +
       

     

       102
       102
       +
       	if rd.p.ratelimitBypassKey != "" && strings.HasSuffix(service, ".bsky.network") {

     

       103
       103
       +
       		req.Header.Set("x-ratelimit-bypass", rd.p.ratelimitBypassKey)

     

       104
       104
       +
       	}

     

       105
       105
       +
       

     

       106
       106
       +
       	client := rd.getClient(service)

     

       107
       107
       +
       

     

       108
       108
       +
       	resp, err := client.Do(req)

     

       109
       109
       +
       	if err != nil {

     

       110
       110
       +
       		return nil, fmt.Errorf("failed to download repo: %w", err)

     

       111
       111
       +
       	}

     

       112
       112
       +
       	defer resp.Body.Close()

     

       113
       113
       +
       

     

       114
       114
       +
       	if resp.StatusCode != http.StatusOK {

     

       115
       115
       +
       		if resp.StatusCode == 400 {

     

       116
       116
       +
       			return nil, nil

     

       117
       117
       +
       		}

     

       118
       118
       +
       		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)

     

       119
       119
       +
       	}

     

       120
       120
       +
       

     

       121
       121
       +
       	b, err := io.ReadAll(resp.Body)

     

       122
       122
       +
       	if err != nil {

     

       123
       123
       +
       		return nil, fmt.Errorf("could not read bytes from response: %w", err)

     

       124
       124
       +
       	}

     

       125
       125
       +
       

     

       126
       126
       +
       	return b, nil

     

       127
       127
       +
       }

     

       128
       128
       +
       

     

       129
       129
       +
       func (p *Photocopy) processRepo(ctx context.Context, b []byte, did string) error {

     

       130
       130
       +
       	bs := atproto_repo.NewTinyBlockstore()

     

       131
       131
       +
       	cs, err := car.NewCarReader(bytes.NewReader(b))

     

       132
       132
       +
       	if err != nil {

     

       133
       133
       +
       		fmt.Println("error opening car", err)

     

       134
       134
       +
       		return fmt.Errorf("error opening car: %v\n", err)

     

       135
       135
       +
       	}

     

       136
       136
       +
       

     

       137
       137
       +
       	currBlock, _ := cs.Next()

     

       138
       138
       +
       	for currBlock != nil {

     

       139
       139
       +
       		bs.Put(context.TODO(), currBlock)

     

       140
       140
       +
       		next, _ := cs.Next()

     

       141
       141
       +
       		currBlock = next

     

       142
       142
       +
       	}

     

       143
       143
       +
       

     

       144
       144
       +
       	r, err := repo.OpenRepo(context.TODO(), bs, cs.Header.Roots[0])

     

       145
       145
       +
       	if err != nil || r == nil {

     

       146
       146
       +
       		fmt.Println("error opening repo", err)

     

       147
       147
       +
       		fmt.Printf("could not open repo: %v", err)

     

       148
       148
       +
       		return nil

     

       149
       149
       +
       	}

     

       150
       150
       +
       

     

       151
       151
       +
       	if err := r.ForEach(context.TODO(), "", func(key string, cid cid.Cid) error {

     

       152
       152
       +
       		pts := strings.Split(key, "/")

     

       153
       153
       +
       		nsid := pts[0]

     

       154
       154
       +
       		rkey := pts[1]

     

       155
       155
       +
       		cidStr := cid.String()

     

       156
       156
       +
       		b, err := bs.Get(context.TODO(), cid)

     

       157
       157
       +
       		if err != nil {

     

       158
       158
       +
       			return nil

     

       159
       159
       +
       		}

     

       160
       160
       +
       		if err := p.handleCreate(ctx, b.RawData(), time.Now().Format(time.RFC3339Nano), "unk", did, nsid, rkey, cidStr, "unk"); err != nil {

     

       161
       161
       +
       			return err

     

       162
       162
       +
       		}

     

       163
       163
       +
       		return nil

     

       164
       164
       +
       	}); err != nil {

     

       165
       165
       +
       		return fmt.Errorf("erorr traversing records: %v", err)

     

       166
       166
       +
       	}

     

       167
       167
       +
       

     

       168
       168
       +
       	return nil

     

       169
       169
       +
       }

     

       170
       170
       +
       

     

       171
       171
       +
       type ListReposResponse struct {

     

       172
       172
       +
       	Cursor string          `json:"cursor"`

     

       173
       173
       +
       	Repos  []ListReposRepo `json:"repos"`

     

       174
       174
       +
       }

     

       175
       175
       +
       

     

       176
       176
       +
       type ListReposRepo struct {

     

       177
       177
       +
       	Did    string  `json:"did"`

     

       178
       178
       +
       	Head   string  `json:"head"`

     

       179
       179
       +
       	Rev    string  `json:"rev"`

     

       180
       180
       +
       	Active bool    `json:"active"`

     

       181
       181
       +
       	Status *string `json:"status,omitempty"`

     

       182
       182
       +
       }

     

       183
       183
       +
       

     

       184
       184
       +
       func (rd *RepoDownloader) getDidsFromService(ctx context.Context, service string) ([]ListReposRepo, error) {

     

       185
       185
       +
       	var cursor string

     

       186
       186
       +
       	var repos []ListReposRepo

     

       187
       187
       +
       	if service == "https://atproto.brid.gy" {

     

       188
       188
       +
       		return nil, nil

     

       189
       189
       +
       	}

     

       190
       190
       +
       	for {

     

       191
       191
       +
       		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/xrpc/com.atproto.sync.listRepos?limit=1000&cursor=%s", service, cursor), nil)

     

       192
       192
       +
       		if err != nil {

     

       193
       193
       +
       			return nil, err

     

       194
       194
       +
       		}

     

       195
       195
       +
       

     

       196
       196
       +
       		if rd.p.ratelimitBypassKey != "" && strings.HasSuffix(service, ".bsky.network") {

     

       197
       197
       +
       			req.Header.Set("x-ratelimit-bypass", rd.p.ratelimitBypassKey)

     

       198
       198
       +
       		}

     

       199
       199
       +
       

     

       200
       200
       +
       		rl := rd.getRateLimiter(service)

     

       201
       201
       +
       		rl.Take()

     

       202
       202
       +
       

     

       203
       203
       +
       		cli := rd.getClient(service)

     

       204
       204
       +
       		resp, err := cli.Do(req)

     

       205
       205
       +
       		if err != nil {

     

       206
       206
       +
       			return nil, err

     

       207
       207
       +
       		}

     

       208
       208
       +
       		defer resp.Body.Close()

     

       209
       209
       +
       

     

       210
       210
       +
       		if resp.StatusCode != http.StatusOK {

     

       211
       211
       +
       			return nil, fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       212
       212
       +
       		}

     

       213
       213
       +
       

     

       214
       214
       +
       		var reposResp ListReposResponse

     

       215
       215
       +
       		if err := json.NewDecoder(resp.Body).Decode(&reposResp); err != nil {

     

       216
       216
       +
       			return nil, fmt.Errorf("error decoding repos response: %w", err)

     

       217
       217
       +
       		}

     

       218
       218
       +
       

     

       219
       219
       +
       		for _, repo := range reposResp.Repos {

     

       220
       220
       +
       			if repo.Status != nil {

     

       221
       221
       +
       				if *repo.Status == "deleted" || *repo.Status == "takendown" || *repo.Status == "deactivated" {

     

       222
       222
       +
       					continue

     

       223
       223
       +
       				}

     

       224
       224
       +
       			}

     

       225
       225
       +
       

     

       226
       226
       +
       			repos = append(repos, repo)

     

       227
       227
       +
       		}

     

       228
       228
       +
       

     

       229
       229
       +
       		if len(reposResp.Repos) != 1000 || reposResp.Cursor == "" {

     

       230
       230
       +
       			break

     

       231
       231
       +
       		}

     

       232
       232
       +
       

     

       233
       233
       +
       		fmt.Printf("cursor %s service %s\n", reposResp.Cursor, service)

     

       234
       234
       +
       

     

       235
       235
       +
       		cursor = reposResp.Cursor

     

       236
       236
       +
       	}

     

       237
       237
       +
       

     

       238
       238
       +
       	return repos, nil

     

       239
       239
       +
       }

     

       240
       240
       +
       

     

       241
       241
       +
       type ListServicesResponse struct {

     

       242
       242
       +
       	Cursor string                     `json:"cursor"`

     

       243
       243
       +
       	Hosts  []ListServicesResponseItem `json:"hosts"`

     

       244
       244
       +
       }

     

       245
       245
       +
       

     

       246
       246
       +
       type ListServicesResponseItem struct {

     

       247
       247
       +
       	Hostname string `json:"hostname"`

     

       248
       248
       +
       	Status   string `json:"status"`

     

       249
       249
       +
       }

     

       250
       250
       +
       

     

       251
       251
       +
       func (p *Photocopy) runProcessRepoWorker(ctx context.Context, jobs <-chan ProcessJob) {

     

       252
       252
       +
       	for j := range jobs {

     

       253
       253
       +
       		p.processRepo(ctx, j.repoBytes, j.did)

     

       254
       254
       +
       	}

     

       255
       255
       +
       }

     

       256
       256
       +
       

     

       257
       257
       +
       func (p *Photocopy) runBackfiller(ctx context.Context) error {

     

       258
       258
       +
       	startTime := time.Now()

     

       259
       259
       +
       

     

       260
       260
       +
       	fmt.Println("querying clickhouse for dids and services...")

     

       261
       261
       +
       

     

       262
       262
       +
       	type alreadyFetchedItem struct {

     

       263
       263
       +
       		Did string `ch:"did"`

     

       264
       264
       +
       	}

     

       265
       265
       +
       	var alreadyFetched []alreadyFetchedItem

     

       266
       266
       +
       	if err := p.conn.Select(ctx, &alreadyFetched, "SELECT DISTINCT(did) FROM default.record WHERE created_at < '2025-07-01'"); err != nil {

     

       267
       267
       +
       		return err

     

       268
       268
       +
       	}

     

       269
       269
       +
       

     

       270
       270
       +
       	alreadyFetchedMap := map[string]bool{}

     

       271
       271
       +
       	for _, d := range alreadyFetched {

     

       272
       272
       +
       		alreadyFetchedMap[d.Did] = true

     

       273
       273
       +
       	}

     

       274
       274
       +
       

     

       275
       275
       +
       	fmt.Println("getting dids")

     

       276
       276
       +
       

     

       277
       277
       +
       	var hostsCursor string

     

       278
       278
       +
       	var sevs []ListServicesResponseItem

     

       279
       279
       +
       	for {

     

       280
       280
       +
       		if hostsCursor != "" {

     

       281
       281
       +
       			hostsCursor = "&cursor=" + hostsCursor

     

       282
       282
       +
       		}

     

       283
       283
       +
       		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://relay1.us-east.bsky.network/xrpc/com.atproto.sync.listHosts?limit=1000%s", hostsCursor), nil)

     

       284
       284
       +
       		if err != nil {

     

       285
       285
       +
       			return err

     

       286
       286
       +
       		}

     

       287
       287
       +
       

     

       288
       288
       +
       		resp, err := http.DefaultClient.Do(req)

     

       289
       289
       +
       		if err != nil {

     

       290
       290
       +
       			return err

     

       291
       291
       +
       		}

     

       292
       292
       +
       		defer resp.Body.Close()

     

       293
       293
       +
       

     

       294
       294
       +
       		if resp.StatusCode != http.StatusOK {

     

       295
       295
       +
       			return fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       296
       296
       +
       		}

     

       297
       297
       +
       

     

       298
       298
       +
       		var sevsResp ListServicesResponse

     

       299
       299
       +
       		if err := json.NewDecoder(resp.Body).Decode(&sevsResp); err != nil {

     

       300
       300
       +
       			return fmt.Errorf("error decoding sevs response: %w", err)

     

       301
       301
       +
       		}

     

       302
       302
       +
       

     

       303
       303
       +
       		for _, sev := range sevsResp.Hosts {

     

       304
       304
       +
       			if sev.Status != "active" {

     

       305
       305
       +
       				continue

     

       306
       306
       +
       			}

     

       307
       307
       +
       

     

       308
       308
       +
       			sevs = append(sevs, sev)

     

       309
       309
       +
       		}

     

       310
       310
       +
       

     

       311
       311
       +
       		if len(sevsResp.Hosts) != 1000 || sevsResp.Cursor == "" {

     

       312
       312
       +
       			break

     

       313
       313
       +
       		}

     

       314
       314
       +
       

     

       315
       315
       +
       		hostsCursor = sevsResp.Cursor

     

       316
       316
       +
       	}

     

       317
       317
       +
       

     

       318
       318
       +
       	servicesDids := map[string][]string{}

     

       319
       319
       +
       	for _, sev := range sevs {

     

       320
       320
       +
       		servicesDids["https://"+sev.Hostname] = []string{}

     

       321
       321
       +
       	}

     

       322
       322
       +
       

     

       323
       323
       +
       	fmt.Printf("found %d services\n", len(servicesDids))

     

       324
       324
       +
       

     

       325
       325
       +
       	fmt.Printf("collecting dids...\n")

     

       326
       326
       +
       

     

       327
       327
       +
       	fmt.Printf("building download buckets...")

     

       328
       328
       +
       

     

       329
       329
       +
       	skipped := 0

     

       330
       330
       +
       	downloader := NewRepoDownloader(p)

     

       331
       331
       +
       	serviceDids := map[string][]string{}

     

       332
       332
       +
       

     

       333
       333
       +
       	for range runtime.NumCPU() / 2 {

     

       334
       334
       +
       		go p.runProcessRepoWorker(ctx, downloader.processChan)

     

       335
       335
       +
       	}

     

       336
       336
       +
       

     

       337
       337
       +
       	wg := sync.WaitGroup{}

     

       338
       338
       +
       	mplk := sync.Mutex{}

     

       339
       339
       +
       	for s := range servicesDids {

     

       340
       340
       +
       		wg.Add(1)

     

       341
       341
       +
       		go func() {

     

       342
       342
       +
       			defer wg.Done()

     

       343
       343
       +
       			repos, err := downloader.getDidsFromService(context.TODO(), s)

     

       344
       344
       +
       			if err != nil {

     

       345
       345
       +
       				fmt.Printf("error getting dids for services %s: %v", s, err)

     

       346
       346
       +
       				return

     

       347
       347
       +
       			}

     

       348
       348
       +
       			dids := []string{}

     

       349
       349
       +
       			for _, r := range repos {

     

       350
       350
       +
       				if alreadyFetchedMap[r.Did] {

     

       351
       351
       +
       					skipped++

     

       352
       352
       +
       					continue

     

       353
       353
       +
       				}

     

       354
       354
       +
       				dids = append(dids, r.Did)

     

       355
       355
       +
       			}

     

       356
       356
       +
       			mplk.Lock()

     

       357
       357
       +
       			defer mplk.Unlock()

     

       358
       358
       +
       			serviceDids[s] = dids

     

       359
       359
       +
       		}()

     

       360
       360
       +
       	}

     

       361
       361
       +
       

     

       362
       362
       +
       	fmt.Println("getting all the repos...")

     

       363
       363
       +
       	wg.Wait()

     

       364
       364
       +
       

     

       365
       365
       +
       	fmt.Printf("was able to skip %d repos\n", skipped)

     

       366
       366
       +
       

     

       367
       367
       +
       	total := 0

     

       368
       368
       +
       

     

       369
       369
       +
       	for service, dids := range serviceDids {

     

       370
       370
       +
       		if len(dids) < 100 {

     

       371
       371
       +
       			continue

     

       372
       372
       +
       		}

     

       373
       373
       +
       		fmt.Printf("%s: %d jobs\n", service, len(dids))

     

       374
       374
       +
       		total += len(dids)

     

       375
       375
       +
       	}

     

       376
       376
       +
       

     

       377
       377
       +
       	fmt.Printf("Total jobs: %d across %d services \n", total, len(serviceDids))

     

       378
       378
       +
       

     

       379
       379
       +
       	for _, c := range downloader.clients {

     

       380
       380
       +
       		c.Timeout = 10 * time.Minute

     

       381
       381
       +
       	}

     

       382
       382
       +
       

     

       383
       383
       +
       	for s := range downloader.rateLimits {

     

       384
       384
       +
       		if p.ratelimitBypassKey != "" && strings.HasSuffix(s, ".bsky.network") {

     

       385
       385
       +
       			downloader.rateLimits[s] = ratelimit.New(25)

     

       386
       386
       +
       		}

     

       387
       387
       +
       	}

     

       388
       388
       +
       

     

       389
       389
       +
       	processed := 0

     

       390
       390
       +
       	errored := 0

     

       391
       391
       +
       	var errors []error

     

       392
       392
       +
       	for service, dids := range serviceDids {

     

       393
       393
       +
       		go func() {

     

       394
       394
       +
       			for _, did := range dids {

     

       395
       395
       +
       				ratelimiter := downloader.getRateLimiter(service)

     

       396
       396
       +
       				ratelimiter.Take()

     

       397
       397
       +
       

     

       398
       398
       +
       				b, err := downloader.downloadRepo(service, did)

     

       399
       399
       +
       				if err != nil {

     

       400
       400
       +
       					errored++

     

       401
       401
       +
       					processed++

     

       402
       402
       +
       					errors = append(errors, err)

     

       403
       403
       +
       					continue

     

       404
       404
       +
       				}

     

       405
       405
       +
       

     

       406
       406
       +
       				go func(b []byte, did string) {

     

       407
       407
       +
       					downloader.processChan <- ProcessJob{repoBytes: b, did: did}

     

       408
       408
       +
       				}(b, did)

     

       409
       409
       +
       

     

       410
       410
       +
       				processed++

     

       411
       411
       +
       			}

     

       412
       412
       +
       		}()

     

       413
       413
       +
       	}

     

       414
       414
       +
       

     

       415
       415
       +
       	ticker := time.NewTicker(1 * time.Second)

     

       416
       416
       +
       	defer ticker.Stop()

     

       417
       417
       +
       

     

       418
       418
       +
       	for range ticker.C {

     

       419
       419
       +
       		elapsed := time.Since(startTime)

     

       420
       420
       +
       		rate := float64(processed) / elapsed.Seconds()

     

       421
       421
       +
       		remaining := total - processed

     

       422
       422
       +
       

     

       423
       423
       +
       		var eta string

     

       424
       424
       +
       		if rate > 0 {

     

       425
       425
       +
       			etaSeconds := float64(remaining) / rate

     

       426
       426
       +
       			etaDuration := time.Duration(etaSeconds * float64(time.Second))

     

       427
       427
       +
       			eta = fmt.Sprintf(", ETA: %v", etaDuration.Round(time.Second))

     

       428
       428
       +
       		} else {

     

       429
       429
       +
       			eta = ", ETA: calculating..."

     

       430
       430
       +
       		}

     

       431
       431
       +
       

     

       432
       432
       +
       		for _, err := range errors {

     

       433
       433
       +
       			fmt.Printf("%v\n", err)

     

       434
       434
       +
       		}

     

       435
       435
       +
       

     

       436
       436
       +
       		errors = nil

     

       437
       437
       +
       

     

       438
       438
       +
       		fmt.Printf("\rProgress: %d/%d processed (%.1f%%), %d skipped, %d errors, %.1f jobs/sec%s",

     

       439
       439
       +
       			processed, total, float64(processed)/float64(total)*100, skipped, errored, rate, eta)

     

       440
       440
       +
       	}

     

       441
       441
       +
       

     

       442
       442
       +
       	fmt.Printf("\nCompleted: %d processed, %d errors\n", processed, errored)

     

       443
       443
       +
       

     

       444
       444
       +
       	return nil

     

       445
       445
       +
       }

+179

clickhouse_inserter/inserter.go

···

       1
       1
       +
       package clickhouse_inserter

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"context"

     

       5
       5
       +
       	"log/slog"

     

       6
       6
       +
       	"reflect"

     

       7
       7
       +
       	"slices"

     

       8
       8
       +
       	"sync"

     

       9
       9
       +
       	"time"

     

       10
       10
       +
       

     

       11
       11
       +
       	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"

     

       12
       12
       +
       	"github.com/prometheus/client_golang/prometheus"

     

       13
       13
       +
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       14
       14
       +
       	"go.uber.org/ratelimit"

     

       15
       15
       +
       )

     

       16
       16
       +
       

     

       17
       17
       +
       type Inserter struct {

     

       18
       18
       +
       	conn           driver.Conn

     

       19
       19
       +
       	query          string

     

       20
       20
       +
       	mu             sync.Mutex

     

       21
       21
       +
       	queuedEvents   []any

     

       22
       22
       +
       	batchSize      int

     

       23
       23
       +
       	insertsCounter *prometheus.CounterVec

     

       24
       24
       +
       	pendingSends   prometheus.Gauge

     

       25
       25
       +
       	histogram      *prometheus.HistogramVec

     

       26
       26
       +
       	logger         *slog.Logger

     

       27
       27
       +
       	prefix         string

     

       28
       28
       +
       	rateLimit      ratelimit.Limiter

     

       29
       29
       +
       }

     

       30
       30
       +
       

     

       31
       31
       +
       type Args struct {

     

       32
       32
       +
       	Conn                    driver.Conn

     

       33
       33
       +
       	Query                   string

     

       34
       34
       +
       	BatchSize               int

     

       35
       35
       +
       	PrometheusCounterPrefix string

     

       36
       36
       +
       	Logger                  *slog.Logger

     

       37
       37
       +
       	Histogram               *prometheus.HistogramVec

     

       38
       38
       +
       	RateLimit               int

     

       39
       39
       +
       }

     

       40
       40
       +
       

     

       41
       41
       +
       func New(ctx context.Context, args *Args) (*Inserter, error) {

     

       42
       42
       +
       	if args.Logger == nil {

     

       43
       43
       +
       		args.Logger = slog.Default()

     

       44
       44
       +
       	}

     

       45
       45
       +
       

     

       46
       46
       +
       	inserter := &Inserter{

     

       47
       47
       +
       		conn:      args.Conn,

     

       48
       48
       +
       		query:     args.Query,

     

       49
       49
       +
       		mu:        sync.Mutex{},

     

       50
       50
       +
       		batchSize: args.BatchSize,

     

       51
       51
       +
       		histogram: args.Histogram,

     

       52
       52
       +
       		logger:    args.Logger,

     

       53
       53
       +
       		prefix:    args.PrometheusCounterPrefix,

     

       54
       54
       +
       	}

     

       55
       55
       +
       

     

       56
       56
       +
       	if args.RateLimit != 0 {

     

       57
       57
       +
       		rateLimit := ratelimit.New(args.RateLimit)

     

       58
       58
       +
       		inserter.rateLimit = rateLimit

     

       59
       59
       +
       	}

     

       60
       60
       +
       

     

       61
       61
       +
       	if args.PrometheusCounterPrefix != "" {

     

       62
       62
       +
       		inserter.insertsCounter = promauto.NewCounterVec(prometheus.CounterOpts{

     

       63
       63
       +
       			Name:      "clickhouse_inserts",

     

       64
       64
       +
       			Namespace: args.PrometheusCounterPrefix,

     

       65
       65
       +
       			Help:      "total inserts into clickhouse by status",

     

       66
       66
       +
       		}, []string{"status"})

     

       67
       67
       +
       

     

       68
       68
       +
       		inserter.pendingSends = promauto.NewGauge(prometheus.GaugeOpts{

     

       69
       69
       +
       			Name:      "clickhouse_pending_sends",

     

       70
       70
       +
       			Namespace: args.PrometheusCounterPrefix,

     

       71
       71
       +
       			Help:      "total clickhouse insertions that are in progress",

     

       72
       72
       +
       		})

     

       73
       73
       +
       

     

       74
       74
       +
       	} else {

     

       75
       75
       +
       		args.Logger.Info("no prometheus prefix provided, no metrics will be registered for this counter", "query", args.Query)

     

       76
       76
       +
       	}

     

       77
       77
       +
       

     

       78
       78
       +
       	return inserter, nil

     

       79
       79
       +
       }

     

       80
       80
       +
       

     

       81
       81
       +
       func (i *Inserter) Insert(ctx context.Context, e any) error {

     

       82
       82
       +
       	i.mu.Lock()

     

       83
       83
       +
       

     

       84
       84
       +
       	i.queuedEvents = append(i.queuedEvents, e)

     

       85
       85
       +
       

     

       86
       86
       +
       	var toInsert []any

     

       87
       87
       +
       	if len(i.queuedEvents) >= i.batchSize {

     

       88
       88
       +
       		toInsert = slices.Clone(i.queuedEvents)

     

       89
       89
       +
       		i.queuedEvents = nil

     

       90
       90
       +
       	}

     

       91
       91
       +
       

     

       92
       92
       +
       	i.mu.Unlock()

     

       93
       93
       +
       

     

       94
       94
       +
       	if len(toInsert) > 0 {

     

       95
       95
       +
       		i.sendStream(ctx, toInsert)

     

       96
       96
       +
       	}

     

       97
       97
       +
       

     

       98
       98
       +
       	return nil

     

       99
       99
       +
       }

     

       100
       100
       +
       

     

       101
       101
       +
       func (i *Inserter) Close(ctx context.Context) error {

     

       102
       102
       +
       	i.mu.Lock()

     

       103
       103
       +
       

     

       104
       104
       +
       	var toInsert []any

     

       105
       105
       +
       

     

       106
       106
       +
       	if len(i.queuedEvents) > 0 {

     

       107
       107
       +
       		toInsert = slices.Clone(i.queuedEvents)

     

       108
       108
       +
       		i.queuedEvents = nil

     

       109
       109
       +
       	}

     

       110
       110
       +
       

     

       111
       111
       +
       	i.mu.Unlock()

     

       112
       112
       +
       

     

       113
       113
       +
       	if len(toInsert) > 0 {

     

       114
       114
       +
       		i.sendStream(ctx, toInsert)

     

       115
       115
       +
       	}

     

       116
       116
       +
       

     

       117
       117
       +
       	return nil

     

       118
       118
       +
       }

     

       119
       119
       +
       

     

       120
       120
       +
       func (i *Inserter) sendStream(ctx context.Context, toInsert []any) {

     

       121
       121
       +
       	if i.pendingSends != nil {

     

       122
       122
       +
       		i.pendingSends.Inc()

     

       123
       123
       +
       		defer i.pendingSends.Dec()

     

       124
       124
       +
       	}

     

       125
       125
       +
       

     

       126
       126
       +
       	if i.histogram != nil {

     

       127
       127
       +
       		start := time.Now()

     

       128
       128
       +
       		defer func() {

     

       129
       129
       +
       			i.histogram.WithLabelValues(i.prefix).Observe(time.Since(start).Seconds())

     

       130
       130
       +
       		}()

     

       131
       131
       +
       	}

     

       132
       132
       +
       

     

       133
       133
       +
       	if len(toInsert) == 0 {

     

       134
       134
       +
       		return

     

       135
       135
       +
       	}

     

       136
       136
       +
       

     

       137
       137
       +
       	status := "ok"

     

       138
       138
       +
       	if i.insertsCounter != nil {

     

       139
       139
       +
       		defer func() {

     

       140
       140
       +
       			i.insertsCounter.WithLabelValues(status).Add(float64(len(toInsert)))

     

       141
       141
       +
       		}()

     

       142
       142
       +
       	}

     

       143
       143
       +
       

     

       144
       144
       +
       	batch, err := i.conn.PrepareBatch(ctx, i.query)

     

       145
       145
       +
       	if err != nil {

     

       146
       146
       +
       		i.logger.Error("error creating batch", "prefix", i.prefix, "error", err)

     

       147
       147
       +
       		status = "failed"

     

       148
       148
       +
       		return

     

       149
       149
       +
       	}

     

       150
       150
       +
       

     

       151
       151
       +
       	for _, d := range toInsert {

     

       152
       152
       +
       		var structPtr any

     

       153
       153
       +
       		if reflect.TypeOf(d).Kind() == reflect.Ptr {

     

       154
       154
       +
       			structPtr = d

     

       155
       155
       +
       		} else {

     

       156
       156
       +
       			v := reflect.ValueOf(d)

     

       157
       157
       +
       			if v.CanAddr() {

     

       158
       158
       +
       				structPtr = v.Addr().Addr().Interface()

     

       159
       159
       +
       			} else {

     

       160
       160
       +
       				ptr := reflect.New(v.Type())

     

       161
       161
       +
       				ptr.Elem().Set(v)

     

       162
       162
       +
       				structPtr = ptr.Interface()

     

       163
       163
       +
       			}

     

       164
       164
       +
       		}

     

       165
       165
       +
       

     

       166
       166
       +
       		if err := batch.AppendStruct(structPtr); err != nil {

     

       167
       167
       +
       			i.logger.Error("error appending to batch", "prefix", i.prefix, "error", err)

     

       168
       168
       +
       		}

     

       169
       169
       +
       	}

     

       170
       170
       +
       

     

       171
       171
       +
       	if i.rateLimit != nil {

     

       172
       172
       +
       		i.rateLimit.Take()

     

       173
       173
       +
       	}

     

       174
       174
       +
       

     

       175
       175
       +
       	if err := batch.Send(); err != nil {

     

       176
       176
       +
       		status = "failed"

     

       177
       177
       +
       		i.logger.Error("error sending batch", "prefix", i.prefix, "error", err)

     

       178
       178
       +
       	}

     

       179
       179
       +
       }

+54 -5

cmd/photocopy/main.go

···

       14
       14
        
       

     

       15
       15
        
       func main() {

     

       16
       16
        
       	app := cli.App{

     

       17
       17
       -
       		Name:   "photocopy",

     

       18
       18
       -
       		Usage:  "bigquery inserter for firehose events",

     

       19
       19
       -
       		Action: run,

     

       17
       17
       +
       		Name:  "photocopy",

     

       18
       18
       +
       		Usage: "bigquery inserter for firehose events",

     

       20
       19
        
       		Flags: []cli.Flag{

     

       21
       20
        
       			&cli.StringFlag{

     

       22
       21
        
       				Name:    "relay-host",

     
···

       41
       40
        
       			&cli.StringFlag{

     

       42
       41
        
       				Name:     "plc-scraper-cursor-file",

     

       43
       42
        
       				EnvVars:  []string{"PHOTOCOPY_PLC_SCRAPER_CURSOR_FILE"},

     

       43
       43
       +
       				Required: true,

     

       44
       44
       +
       			},

     

       45
       45
       +
       			&cli.StringFlag{

     

       46
       46
       +
       				Name:     "clickhouse-addr",

     

       47
       47
       +
       				EnvVars:  []string{"PHOTOCOPY_CLICKHOUSE_ADDR"},

     

       48
       48
       +
       				Required: true,

     

       49
       49
       +
       			},

     

       50
       50
       +
       			&cli.StringFlag{

     

       51
       51
       +
       				Name:     "clickhouse-database",

     

       52
       52
       +
       				EnvVars:  []string{"PHOTOCOPY_CLICKHOUSE_DATABASE"},

     

       44
       53
        
       				Required: true,

     

       45
       54
        
       			},

     

       46
       55
        
       			&cli.StringFlag{

     
···

       53
       62
        
       				EnvVars:  []string{"PHOTOCOPY_CLICKHOUSE_PASS"},

     

       54
       63
        
       				Required: true,

     

       55
       64
        
       			},

     

       65
       65
       +
       			&cli.StringFlag{

     

       66
       66
       +
       				Name:     "ratelimit-bypass-key",

     

       67
       67
       +
       				EnvVars:  []string{"PHOTOCOPY_RATELIMIT_BYPASS_KEY"},

     

       68
       68
       +
       				Required: false,

     

       69
       69
       +
       			},

     

       70
       70
       +
       			&cli.BoolFlag{

     

       71
       71
       +
       				Name: "with-backfill",

     

       72
       72
       +
       			},

     

       73
       73
       +
       			&cli.StringFlag{

     

       74
       74
       +
       				Name:    "nervana-endpoint",

     

       75
       75
       +
       				EnvVars: []string{"PHOTOCOPY_NERVANA_ENDPOINT"},

     

       76
       76
       +
       			},

     

       77
       77
       +
       			&cli.StringFlag{

     

       78
       78
       +
       				Name:    "nervana-api-key",

     

       79
       79
       +
       				EnvVars: []string{"PHOTOCOPY_NERVANA_API_KEY"},

     

       80
       80
       +
       			},

     

       81
       81
       +
       		},

     

       82
       82
       +
       		Commands: cli.Commands{

     

       83
       83
       +
       			&cli.Command{

     

       84
       84
       +
       				Name:   "run",

     

       85
       85
       +
       				Action: run,

     

       86
       86
       +
       			},

     

       87
       87
       +
       			&cli.Command{

     

       88
       88
       +
       				Name:   "fetch-repos",

     

       89
       89
       +
       				Action: runFetchRepos,

     

       90
       90
       +
       			},

     

       56
       91
        
       		},

     

       57
       92
        
       		ErrWriter: os.Stderr,

     

       58
       93
        
       	}

     
···

       89
       124
        
       		MetricsAddr:          cmd.String("metrics-addr"),

     

       90
       125
        
       		CursorFile:           cmd.String("cursor-file"),

     

       91
       126
        
       		PLCScraperCursorFile: cmd.String("plc-scraper-cursor-file"),

     

       127
       127
       +
       		ClickhouseAddr:       cmd.String("clickhouse-addr"),

     

       128
       128
       +
       		ClickhouseDatabase:   cmd.String("clickhouse-database"),

     

       92
       129
        
       		ClickhouseUser:       cmd.String("clickhouse-user"),

     

       93
       130
        
       		ClickhousePass:       cmd.String("clickhouse-pass"),

     

       131
       131
       +
       		RatelimitBypassKey:   cmd.String("ratelimit-bypass-key"),

     

       132
       132
       +
       		NervanaEndpoint:      cmd.String("nervana-endpoint"),

     

       133
       133
       +
       		NervanaApiKey:        cmd.String("nervana-api-key"),

     

       94
       134
        
       	})

     

       95
       135
        
       	if err != nil {

     

       96
       136
        
       		panic(err)

     
···

       103
       143
        
       		sig := <-exitSignals

     

       104
       144
        
       

     

       105
       145
        
       		l.Info("received os exit signal", "signal", sig)

     

       146
       146
       +
       		cancel()

     

       106
       147
        
       	}()

     

       107
       148
        
       

     

       108
       108
       -
       	if err := p.Run(ctx); err != nil {

     

       109
       109
       -
       		return err

     

       149
       149
       +
       	if err := p.Run(ctx, cmd.Bool("with-backfill")); err != nil {

     

       150
       150
       +
       		panic(err)

     

       110
       151
        
       	}

     

       111
       152
        
       

     

       112
       153
        
       	return nil

     

       113
       154
        
       }

     

       155
       155
       +
       

     

       156
       156
       +
       var runFetchRepos = func(cmd *cli.Context) error {

     

       157
       157
       +
       	ctx := cmd.Context

     

       158
       158
       +
       	ctx, cancel := context.WithCancel(ctx)

     

       159
       159
       +
       	defer cancel()

     

       160
       160
       +
       

     

       161
       161
       +
       	return nil

     

       162
       162
       +
       }

+1 -10

consumer.go

···

       7
       7
        
       	"net/http"

     

       8
       8
        
       	"net/url"

     

       9
       9
        
       	"os"

     

       10
       10
       -
       	"strings"

     

       11
       10
        
       	"time"

     

       12
       11
        
       

     

       13
       12
        
       	"github.com/bluesky-social/indigo/api/atproto"

     
···

       136
       135
        
       				continue

     

       137
       136
        
       			}

     

       138
       137
        
       

     

       139
       139
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       140
       140
       -
       				continue

     

       141
       141
       -
       			}

     

       142
       142
       -
       

     

       143
       143
       -
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String()); err != nil {

     

       138
       138
       +
       			if err := p.handleCreate(ctx, *rec, evt.Time, evt.Rev, did.String(), collection.String(), rkey.String(), reccid.String(), fmt.Sprintf("%d", evt.Seq)); err != nil {

     

       144
       139
        
       				p.logger.Error("error handling create event", "error", err)

     

       145
       140
        
       				continue

     

       146
       141
        
       			}

     

       147
       142
        
       		case repomgr.EvtKindDeleteRecord:

     

       148
       148
       -
       			if !strings.HasPrefix(collection.String(), "app.bsky.") {

     

       149
       149
       -
       				continue

     

       150
       150
       -
       			}

     

       151
       151
       -
       

     

       152
       143
        
       			if err := p.handleDelete(ctx, did.String(), collection.String(), rkey.String()); err != nil {

     

       153
       144
        
       				p.logger.Error("error handling delete event", "error", err)

     

       154
       145
        
       				continue

+12 -24

go.mod

···

       3
       3
        
       go 1.24.4

     

       4
       4
        
       

     

       5
       5
        
       require (

     

       6
       6
       -
       	cloud.google.com/go/bigquery v1.69.0

     

       6
       6
       +
       	github.com/ClickHouse/clickhouse-go/v2 v2.37.2

     

       7
       7
        
       	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de

     

       8
       8
        
       	github.com/bluesky-social/indigo v0.0.0-20250626183556-5641d3c27325

     

       9
       9
        
       	github.com/gorilla/websocket v1.5.1

     

       10
       10
        
       	github.com/ipfs/go-cid v0.5.0

     

       11
       11
       +
       	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4

     

       11
       12
        
       	github.com/joho/godotenv v1.5.1

     

       12
       13
        
       	github.com/prometheus/client_golang v1.22.0

     

       13
       14
        
       	github.com/urfave/cli/v2 v2.25.7

     

       15
       15
       +
       	go.uber.org/ratelimit v0.3.1

     

       14
       16
        
       )

     

       15
       17
        
       

     

       16
       18
        
       require (

     

       17
       17
       -
       	cloud.google.com/go v0.121.0 // indirect

     

       18
       18
       -
       	cloud.google.com/go/auth v0.16.1 // indirect

     

       19
       19
       -
       	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect

     

       20
       20
       -
       	cloud.google.com/go/compute/metadata v0.6.0 // indirect

     

       21
       21
       -
       	cloud.google.com/go/iam v1.5.2 // indirect

     

       19
       19
       +
       	github.com/ClickHouse/ch-go v0.66.1 // indirect

     

       22
       20
        
       	github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect

     

       23
       23
       -
       	github.com/apache/arrow/go/v15 v15.0.2 // indirect

     

       21
       21
       +
       	github.com/andybalholm/brotli v1.1.1 // indirect

     

       22
       22
       +
       	github.com/benbjohnson/clock v1.3.0 // indirect

     

       24
       23
        
       	github.com/beorn7/perks v1.0.1 // indirect

     

       25
       24
        
       	github.com/carlmjohnson/versioninfo v0.22.5 // indirect

     

       26
       25
        
       	github.com/cespare/xxhash/v2 v2.3.0 // indirect

     

       27
       26
        
       	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect

     

       28
       27
        
       	github.com/felixge/httpsnoop v1.0.4 // indirect

     

       28
       28
       +
       	github.com/go-faster/city v1.0.1 // indirect

     

       29
       29
       +
       	github.com/go-faster/errors v0.7.1 // indirect

     

       29
       30
        
       	github.com/go-logr/logr v1.4.2 // indirect

     

       30
       31
        
       	github.com/go-logr/stdr v1.2.2 // indirect

     

       31
       31
       -
       	github.com/goccy/go-json v0.10.2 // indirect

     

       32
       32
        
       	github.com/gocql/gocql v1.7.0 // indirect

     

       33
       33
        
       	github.com/gogo/protobuf v1.3.2 // indirect

     

       34
       34
        
       	github.com/golang/snappy v0.0.4 // indirect

     

       35
       35
       -
       	github.com/google/flatbuffers v23.5.26+incompatible // indirect

     

       36
       36
       -
       	github.com/google/s2a-go v0.1.9 // indirect

     

       37
       35
        
       	github.com/google/uuid v1.6.0 // indirect

     

       38
       38
       -
       	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect

     

       39
       39
       -
       	github.com/googleapis/gax-go/v2 v2.14.1 // indirect

     

       40
       36
        
       	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect

     

       41
       37
        
       	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect

     

       42
       38
        
       	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect

     
···

       59
       55
        
       	github.com/ipfs/go-merkledag v0.11.0 // indirect

     

       60
       56
        
       	github.com/ipfs/go-metrics-interface v0.0.1 // indirect

     

       61
       57
        
       	github.com/ipfs/go-verifcid v0.0.3 // indirect

     

       62
       62
       -
       	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect

     

       63
       58
        
       	github.com/ipld/go-codec-dagpb v1.6.0 // indirect

     

       64
       59
        
       	github.com/ipld/go-ipld-prime v0.21.0 // indirect

     

       65
       60
        
       	github.com/jackc/pgpassfile v1.0.0 // indirect

     
···

       82
       77
        
       	github.com/multiformats/go-varint v0.0.7 // indirect

     

       83
       78
        
       	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

     

       84
       79
        
       	github.com/opentracing/opentracing-go v1.2.0 // indirect

     

       80
       80
       +
       	github.com/paulmach/orb v0.11.1 // indirect

     

       85
       81
        
       	github.com/pierrec/lz4/v4 v4.1.22 // indirect

     

       86
       82
        
       	github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect

     

       87
       83
        
       	github.com/prometheus/client_model v0.6.1 // indirect

     

       88
       84
        
       	github.com/prometheus/common v0.62.0 // indirect

     

       89
       85
        
       	github.com/prometheus/procfs v0.15.1 // indirect

     

       90
       86
        
       	github.com/russross/blackfriday/v2 v2.1.0 // indirect

     

       87
       87
       +
       	github.com/segmentio/asm v1.2.0 // indirect

     

       88
       88
       +
       	github.com/shopspring/decimal v1.4.0 // indirect

     

       91
       89
        
       	github.com/spaolacci/murmur3 v1.1.0 // indirect

     

       92
       90
        
       	github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e // indirect

     

       93
       91
        
       	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

     

       94
       94
       -
       	github.com/zeebo/xxh3 v1.0.2 // indirect

     

       95
       92
        
       	gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect

     

       96
       93
        
       	gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect

     

       97
       94
        
       	go.opentelemetry.io/auto/sdk v1.1.0 // indirect

     

       98
       98
       -
       	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect

     

       99
       95
        
       	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect

     

       100
       96
        
       	go.opentelemetry.io/otel v1.36.0 // indirect

     

       101
       97
        
       	go.opentelemetry.io/otel/metric v1.36.0 // indirect

     

       102
       102
       -
       	go.opentelemetry.io/otel/sdk v1.36.0 // indirect

     

       103
       98
        
       	go.opentelemetry.io/otel/trace v1.36.0 // indirect

     

       104
       99
        
       	go.uber.org/atomic v1.11.0 // indirect

     

       105
       100
        
       	go.uber.org/multierr v1.11.0 // indirect

     

       106
       101
        
       	go.uber.org/zap v1.27.0 // indirect

     

       107
       102
        
       	golang.org/x/crypto v0.39.0 // indirect

     

       108
       103
        
       	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect

     

       109
       109
       -
       	golang.org/x/mod v0.25.0 // indirect

     

       110
       104
        
       	golang.org/x/net v0.41.0 // indirect

     

       111
       111
       -
       	golang.org/x/oauth2 v0.30.0 // indirect

     

       112
       105
        
       	golang.org/x/sync v0.15.0 // indirect

     

       113
       106
        
       	golang.org/x/sys v0.33.0 // indirect

     

       114
       107
        
       	golang.org/x/text v0.26.0 // indirect

     

       115
       108
        
       	golang.org/x/time v0.11.0 // indirect

     

       116
       116
       -
       	golang.org/x/tools v0.33.0 // indirect

     

       117
       109
        
       	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect

     

       118
       118
       -
       	google.golang.org/api v0.232.0 // indirect

     

       119
       119
       -
       	google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect

     

       120
       120
       -
       	google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect

     

       121
       121
       -
       	google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect

     

       122
       122
       -
       	google.golang.org/grpc v1.72.0 // indirect

     

       123
       110
        
       	google.golang.org/protobuf v1.36.6 // indirect

     

       124
       111
        
       	gopkg.in/inf.v0 v0.9.1 // indirect

     

       112
       112
       +
       	gopkg.in/yaml.v3 v3.0.1 // indirect

     

       125
       113
        
       	gorm.io/driver/postgres v1.5.7 // indirect

     

       126
       114
        
       	gorm.io/gorm v1.30.0 // indirect

     

       127
       115
        
       	lukechampine.com/blake3 v1.2.1 // indirect

+43 -81

go.sum

···

       1
       1
       -
       cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI=

     

       2
       2
       -
       cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=

     

       3
       3
       -
       cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg=

     

       4
       4
       -
       cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q=

     

       5
       5
       -
       cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU=

     

       6
       6
       -
       cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI=

     

       7
       7
       -
       cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=

     

       8
       8
       -
       cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=

     

       9
       9
       -
       cloud.google.com/go/bigquery v1.69.0 h1:rZvHnjSUs5sHK3F9awiuFk2PeOaB8suqNuim21GbaTc=

     

       10
       10
       -
       cloud.google.com/go/bigquery v1.69.0/go.mod h1:TdGLquA3h/mGg+McX+GsqG9afAzTAcldMjqhdjHTLew=

     

       11
       11
       -
       cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I=

     

       12
       12
       -
       cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg=

     

       13
       13
       -
       cloud.google.com/go/datacatalog v1.26.0 h1:eFgygb3DTufTWWUB8ARk+dSuXz+aefNJXTlkWlQcWwE=

     

       14
       14
       -
       cloud.google.com/go/datacatalog v1.26.0/go.mod h1:bLN2HLBAwB3kLTFT5ZKLHVPj/weNz6bR0c7nYp0LE14=

     

       15
       15
       -
       cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8=

     

       16
       16
       -
       cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE=

     

       17
       17
       -
       cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE=

     

       18
       18
       -
       cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY=

     

       19
       19
       -
       cloud.google.com/go/monitoring v1.24.0 h1:csSKiCJ+WVRgNkRzzz3BPoGjFhjPY23ZTcaenToJxMM=

     

       20
       20
       -
       cloud.google.com/go/monitoring v1.24.0/go.mod h1:Bd1PRK5bmQBQNnuGwHBfUamAV1ys9049oEPHnn4pcsc=

     

       21
       21
       -
       cloud.google.com/go/storage v1.53.0 h1:gg0ERZwL17pJ+Cz3cD2qS60w1WMDnwcm5YPAIQBHUAw=

     

       22
       22
       -
       cloud.google.com/go/storage v1.53.0/go.mod h1:7/eO2a/srr9ImZW9k5uufcNahT2+fPb8w5it1i5boaA=

     

       23
       1
        
       github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

     

       24
       24
       -
       github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc=

     

       25
       25
       -
       github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY=

     

       26
       26
       -
       github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU=

     

       27
       27
       -
       github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0=

     

       28
       28
       -
       github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI=

     

       29
       29
       -
       github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0=

     

       2
       2
       +
       github.com/ClickHouse/ch-go v0.66.1 h1:LQHFslfVYZsISOY0dnOYOXGkOUvpv376CCm8g7W74A4=

     

       3
       3
       +
       github.com/ClickHouse/ch-go v0.66.1/go.mod h1:NEYcg3aOFv2EmTJfo4m2WF7sHB/YFbLUuIWv9iq76xY=

     

       4
       4
       +
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2 h1:wRLNKoynvHQEN4znnVHNLaYnrqVc9sGJmGYg+GGCfto=

     

       5
       5
       +
       github.com/ClickHouse/clickhouse-go/v2 v2.37.2/go.mod h1:pH2zrBGp5Y438DMwAxXMm1neSXPPjSI7tD4MURVULw8=

     

       30
       6
        
       github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU=

     

       31
       7
        
       github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4=

     

       32
       8
        
       github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=

     

       33
       9
        
       github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=

     

       34
       34
       -
       github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=

     

       35
       35
       -
       github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA=

     

       10
       10
       +
       github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=

     

       11
       11
       +
       github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=

     

       36
       12
        
       github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=

     

       37
       13
        
       github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=

     

       38
       14
        
       github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

     
···

       50
       26
        
       github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=

     

       51
       27
        
       github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

     

       52
       28
        
       github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

     

       53
       53
       -
       github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk=

     

       54
       54
       -
       github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=

     

       55
       29
        
       github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=

     

       56
       30
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=

     

       57
       31
        
       github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

     
···

       62
       36
        
       github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

     

       63
       37
        
       github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=

     

       64
       38
        
       github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=

     

       65
       65
       -
       github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=

     

       66
       66
       -
       github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=

     

       67
       67
       -
       github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=

     

       68
       68
       -
       github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=

     

       69
       69
       -
       github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=

     

       70
       39
        
       github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=

     

       71
       40
        
       github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=

     

       72
       41
        
       github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=

     

       73
       42
        
       github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=

     

       74
       74
       -
       github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=

     

       75
       75
       -
       github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=

     

       43
       43
       +
       github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=

     

       44
       44
       +
       github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=

     

       45
       45
       +
       github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=

     

       46
       46
       +
       github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=

     

       76
       47
        
       github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=

     

       77
       48
        
       github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=

     

       78
       49
        
       github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

     
···

       81
       52
        
       github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=

     

       82
       53
        
       github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=

     

       83
       54
        
       github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=

     

       84
       84
       -
       github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=

     

       85
       85
       -
       github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

     

       86
       55
        
       github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus=

     

       87
       56
        
       github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4=

     

       88
       57
        
       github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=

     

       89
       58
        
       github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=

     

       90
       90
       -
       github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=

     

       91
       91
       -
       github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=

     

       59
       59
       +
       github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=

     

       60
       60
       +
       github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=

     

       92
       61
        
       github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=

     

       93
       62
        
       github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=

     

       94
       63
        
       github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=

     

       95
       95
       -
       github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=

     

       96
       96
       -
       github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=

     

       64
       64
       +
       github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=

     

       65
       65
       +
       github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=

     

       97
       66
        
       github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=

     

       98
       67
        
       github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=

     

       99
       68
        
       github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=

     

       100
       69
        
       github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=

     

       101
       101
       -
       github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=

     

       102
       102
       -
       github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=

     

       103
       70
        
       github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=

     

       104
       104
       -
       github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=

     

       105
       105
       -
       github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=

     

       106
       71
        
       github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=

     

       107
       72
        
       github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

     

       108
       108
       -
       github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=

     

       109
       109
       -
       github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=

     

       110
       110
       -
       github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=

     

       111
       111
       -
       github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=

     

       112
       73
        
       github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=

     

       113
       74
        
       github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=

     

       114
       75
        
       github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=

     
···

       215
       176
        
       github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=

     

       216
       177
        
       github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

     

       217
       178
        
       github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

     

       179
       179
       +
       github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=

     

       218
       180
        
       github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=

     

       219
       181
        
       github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=

     

       220
       182
        
       github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=

     
···

       258
       220
        
       github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=

     

       259
       221
        
       github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=

     

       260
       222
        
       github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=

     

       223
       223
       +
       github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=

     

       261
       224
        
       github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=

     

       262
       225
        
       github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=

     

       263
       226
        
       github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=

     
···

       284
       247
        
       github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=

     

       285
       248
        
       github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=

     

       286
       249
        
       github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=

     

       250
       250
       +
       github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=

     

       251
       251
       +
       github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=

     

       252
       252
       +
       github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=

     

       287
       253
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=

     

       288
       254
        
       github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=

     

       289
       255
        
       github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=

     

       290
       256
        
       github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=

     

       291
       257
        
       github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

     

       292
       292
       -
       github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=

     

       293
       293
       -
       github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=

     

       258
       258
       +
       github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

     

       294
       259
        
       github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

     

       295
       260
        
       github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

     

       296
       261
        
       github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4TCkEuWIckKvdHFeJH/huIFJ9/cXOB0=

     
···

       311
       276
        
       github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=

     

       312
       277
        
       github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

     

       313
       278
        
       github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=

     

       279
       279
       +
       github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=

     

       280
       280
       +
       github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=

     

       281
       281
       +
       github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=

     

       282
       282
       +
       github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=

     

       314
       283
        
       github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=

     

       315
       284
        
       github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=

     

       316
       285
        
       github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=

     
···

       318
       287
        
       github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=

     

       319
       288
        
       github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=

     

       320
       289
        
       github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=

     

       321
       321
       -
       github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=

     

       322
       322
       -
       github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=

     

       323
       290
        
       github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

     

       324
       291
        
       github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

     

       325
       292
        
       github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

     

       326
       293
        
       github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

     

       294
       294
       +
       github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

     

       327
       295
        
       github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

     

       328
       296
        
       github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=

     

       329
       297
        
       github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=

     

       298
       298
       +
       github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

     

       330
       299
        
       github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

     

       331
       300
        
       github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=

     

       332
       301
        
       github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=

     
···

       338
       307
        
       github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ=

     

       339
       308
        
       github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e h1:28X54ciEwwUxyHn9yrZfl5ojgF4CBNLWX7LR0rvBkf4=

     

       340
       309
        
       github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so=

     

       310
       310
       +
       github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=

     

       311
       311
       +
       github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=

     

       312
       312
       +
       github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=

     

       341
       313
        
       github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=

     

       342
       314
        
       github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=

     

       315
       315
       +
       github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=

     

       316
       316
       +
       github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=

     

       317
       317
       +
       github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=

     

       343
       318
        
       github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

     

       344
       319
        
       github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

     

       345
       320
        
       github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=

     

       346
       346
       -
       github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=

     

       347
       347
       -
       github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=

     

       348
       348
       -
       github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=

     

       349
       349
       -
       github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=

     

       350
       350
       -
       github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=

     

       351
       351
       -
       github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=

     

       352
       321
        
       gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA=

     

       353
       322
        
       gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=

     

       354
       323
        
       gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q=

     

       355
       324
        
       gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I=

     

       325
       325
       +
       go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=

     

       356
       326
        
       go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=

     

       357
       327
        
       go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=

     

       358
       358
       -
       go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA=

     

       359
       359
       -
       go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA=

     

       360
       360
       -
       go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw=

     

       361
       361
       -
       go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM=

     

       362
       328
        
       go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=

     

       363
       329
        
       go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=

     

       364
       330
        
       go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=

     
···

       382
       348
        
       go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=

     

       383
       349
        
       go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

     

       384
       350
        
       go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=

     

       351
       351
       +
       go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0=

     

       352
       352
       +
       go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk=

     

       385
       353
        
       go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=

     

       386
       354
        
       go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=

     

       387
       355
        
       go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=

     
···

       391
       359
        
       golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

     

       392
       360
        
       golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

     

       393
       361
        
       golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

     

       362
       362
       +
       golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=

     

       394
       363
        
       golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=

     

       395
       364
        
       golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=

     

       396
       365
        
       golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=

     
···

       408
       377
        
       golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

     

       409
       378
        
       golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=

     

       410
       379
        
       golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=

     

       380
       380
       +
       golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=

     

       411
       381
        
       golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=

     

       412
       382
        
       golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=

     

       413
       413
       -
       golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=

     

       414
       414
       -
       golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=

     

       415
       383
        
       golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

     

       416
       384
        
       golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

     

       417
       385
        
       golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

     
···

       423
       391
        
       golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

     

       424
       392
        
       golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

     

       425
       393
        
       golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

     

       394
       394
       +
       golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

     

       426
       395
        
       golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

     

       396
       396
       +
       golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

     

       427
       397
        
       golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

     

       428
       398
        
       golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

     

       429
       399
        
       golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

     
···

       432
       402
        
       golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

     

       433
       403
        
       golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

     

       434
       404
        
       golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

     

       405
       405
       +
       golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

     

       406
       406
       +
       golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=

     

       435
       407
        
       golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=

     

       436
       408
        
       golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=

     

       437
       409
        
       golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=

     
···

       454
       426
        
       golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

     

       455
       427
        
       golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=

     

       456
       428
        
       golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=

     

       457
       457
       -
       gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o=

     

       458
       458
       -
       gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY=

     

       459
       459
       -
       google.golang.org/api v0.232.0 h1:qGnmaIMf7KcuwHOlF3mERVzChloDYwRfOJOrHt8YC3I=

     

       460
       460
       -
       google.golang.org/api v0.232.0/go.mod h1:p9QCfBWZk1IJETUdbTKloR5ToFdKbYh2fkjsUL6vNoY=

     

       461
       461
       -
       google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE=

     

       462
       462
       -
       google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE=

     

       463
       463
       -
       google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 h1:vPV0tzlsK6EzEDHNNH5sa7Hs9bd7iXR7B1tSiPepkV0=

     

       464
       464
       -
       google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw=

     

       465
       465
       -
       google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 h1:IqsN8hx+lWLqlN+Sc3DoMy/watjofWiU8sRFgQ8fhKM=

     

       466
       466
       -
       google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=

     

       467
       467
       -
       google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM=

     

       468
       468
       -
       google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=

     

       429
       429
       +
       google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=

     

       430
       430
       +
       google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

     

       469
       431
        
       google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=

     

       470
       432
        
       google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=

     

       471
       433
        
       gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

+241 -130

handle_create.go

···

       1
       1
        
       package photocopy

     

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
       +
       	"bytes"

     

       4
       5
        
       	"context"

     

       5
       6
        
       	"fmt"

     

       7
       7
       +
       	"slices"

     

       8
       8
       +
       	"strings"

     

       6
       9
        
       	"time"

     

       7
       10
        
       

     

       8
       11
        
       	"github.com/araddon/dateparse"

     

       9
       12
        
       	"github.com/bluesky-social/indigo/api/bsky"

     

       10
       13
        
       	"github.com/bluesky-social/indigo/atproto/syntax"

     

       14
       14
       +
       	"github.com/haileyok/photocopy/models"

     

       11
       15
        
       )

     

       12
       16
        
       

     

       13
       13
       -
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid string) error {

     

       14
       14
       -
       	_, err := dateparse.ParseAny(indexedAt)

     

       17
       17
       +
       func (p *Photocopy) handleCreate(ctx context.Context, recb []byte, indexedAt, rev, did, collection, rkey, cid, seq string) error {

     

       18
       18
       +
       	iat, err := dateparse.ParseAny(indexedAt)

     

       15
       19
        
       	if err != nil {

     

       16
       20
        
       		return err

     

       21
       21
       +
       	}

     

       22
       22
       +
       

     

       23
       23
       +
       	if err := p.handleCreateRecord(ctx, did, rkey, collection, cid, recb, seq); err != nil {

     

       24
       24
       +
       		p.logger.Error("error creating record", "error", err)

     

       17
       25
        
       	}

     

       18
       26
        
       

     

       19
       27
        
       	switch collection {

     

       28
       28
       +
       	case "app.bsky.feed.post":

     

       29
       29
       +
       		return p.handleCreatePost(ctx, rev, recb, uriFromParts(did, collection, rkey), did, collection, rkey, cid, iat)

     

       30
       30
       +
       	case "app.bsky.graph.follow":

     

       31
       31
       +
       		return p.handleCreateFollow(ctx, recb, uriFromParts(did, collection, rkey), did, rkey, iat)

     

       32
       32
       +
       	case "app.bsky.feed.like", "app.bsky.feed.repost":

     

       33
       33
       +
       		return p.handleCreateInteraction(ctx, recb, uriFromParts(did, collection, rkey), did, collection, rkey, iat)

     

       34
       34
       +
       	default:

     

       35
       35
       +
       		return nil

     

       36
       36
       +
       	}

     

       37
       37
       +
       }

     

       38
       38
       +
       

     

       39
       39
       +
       func (p *Photocopy) handleCreateRecord(ctx context.Context, did, rkey, collection, cid string, raw []byte, seq string) error {

     

       40
       40
       +
       	var cat time.Time

     

       41
       41
       +
       	prkey, err := syntax.ParseTID(rkey)

     

       42
       42
       +
       	if err == nil {

     

       43
       43
       +
       		cat = prkey.Time()

     

       44
       44
       +
       	} else {

     

       45
       45
       +
       		cat = time.Now()

     

       46
       46
       +
       	}

     

       47
       47
       +
       

     

       48
       48
       +
       	rec := models.Record{

     

       49
       49
       +
       		Did:        did,

     

       50
       50
       +
       		Rkey:       rkey,

     

       51
       51
       +
       		Collection: collection,

     

       52
       52
       +
       		Cid:        cid,

     

       53
       53
       +
       		Seq:        seq,

     

       54
       54
       +
       		Raw:        string(raw),

     

       55
       55
       +
       		CreatedAt:  cat,

     

       56
       56
       +
       	}

     

       57
       57
       +
       

     

       58
       58
       +
       	if err := p.inserters.recordsInserter.Insert(ctx, rec); err != nil {

     

       59
       59
       +
       		return err

     

       20
       60
        
       	}

     

       21
       61
        
       

     

       22
       62
        
       	return nil

     

       23
       63
        
       }

     

       24
       64
        
       

     

       25
       25
       -
       // func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

     

       26
       26
       -
       // 	var rec bsky.FeedPost

     

       27
       27
       -
       // 	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       28
       28
       -
       // 		return err

     

       29
       29
       -
       // 	}

     

       30
       30
       -
       //

     

       31
       31
       -
       // 	jb, err := json.Marshal(rec)

     

       32
       32
       -
       // 	if err != nil {

     

       33
       33
       -
       // 		return err

     

       34
       34
       -
       // 	}

     

       35
       35
       -
       //

     

       36
       36
       -
       // 	cat, err := parseTimeFromRecord(rec, rkey)

     

       37
       37
       -
       // 	if err != nil {

     

       38
       38
       -
       // 		return err

     

       39
       39
       -
       // 	}

     

       40
       40
       -
       //

     

       41
       41
       -
       // 	bqrec := &BigQueryRecord{

     

       42
       42
       -
       // 		Uri:        uri,

     

       43
       43
       -
       // 		AuthorDid:  did,

     

       44
       44
       -
       // 		Collection: collection,

     

       45
       45
       -
       // 		Rkey:       rkey,

     

       46
       46
       -
       // 		CreatedAt:  *cat,

     

       47
       47
       -
       // 		IndexedAt:  indexedAt,

     

       48
       48
       -
       // 		Raw:        recb,

     

       49
       49
       -
       // 		Json:       string(jb),

     

       50
       50
       -
       // 		Cid:        cid,

     

       51
       51
       -
       // 		Rev:        rev,

     

       52
       52
       -
       // 	}

     

       53
       53
       -
       //

     

       54
       54
       -
       // 	if rec.Reply != nil {

     

       55
       55
       -
       // 		if rec.Reply.Parent != nil {

     

       56
       56
       -
       // 			bqrec.ReplyToUri = rec.Reply.Parent.Uri

     

       57
       57
       -
       // 		}

     

       58
       58
       -
       // 		if rec.Reply.Root != nil {

     

       59
       59
       -
       // 			bqrec.InThreadUri = rec.Reply.Root.Uri

     

       60
       60
       -
       // 		}

     

       61
       61
       -
       // 	}

     

       62
       62
       -
       //

     

       63
       63
       -
       // 	if err := p.inserters.genericInserter.Insert(ctx, bqrec); err != nil {

     

       64
       64
       -
       // 		return err

     

       65
       65
       -
       // 	}

     

       66
       66
       -
       //

     

       67
       67
       -
       // 	return nil

     

       68
       68
       -
       // }

     

       65
       65
       +
       func (p *Photocopy) handleCreatePost(ctx context.Context, rev string, recb []byte, uri, did, collection, rkey, cid string, indexedAt time.Time) error {

     

       66
       66
       +
       	var rec bsky.FeedPost

     

       67
       67
       +
       	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       68
       68
       +
       		return err

     

       69
       69
       +
       	}

     

       70
       70
       +
       

     

       71
       71
       +
       	cat, err := parseTimeFromRecord(rec, rkey)

     

       72
       72
       +
       	if err != nil {

     

       73
       73
       +
       		return err

     

       74
       74
       +
       	}

     

       75
       75
       +
       

     

       76
       76
       +
       	lang := ""

     

       77
       77
       +
       	if len(rec.Langs) != 0 {

     

       78
       78
       +
       		lang = rec.Langs[0]

     

       79
       79
       +
       	}

     

       80
       80
       +
       

     

       81
       81
       +
       	post := models.Post{

     

       82
       82
       +
       		Uri:       uri,

     

       83
       83
       +
       		Rkey:      rkey,

     

       84
       84
       +
       		CreatedAt: *cat,

     

       85
       85
       +
       		IndexedAt: indexedAt,

     

       86
       86
       +
       		Did:       did,

     

       87
       87
       +
       		Lang:      lang,

     

       88
       88
       +
       		Text:      rec.Text,

     

       89
       89
       +
       	}

     

       90
       90
       +
       

     

       91
       91
       +
       	if rec.Reply != nil {

     

       92
       92
       +
       		if rec.Reply.Parent != nil {

     

       93
       93
       +
       			aturi, err := syntax.ParseATURI(rec.Reply.Parent.Uri)

     

       94
       94
       +
       			if err != nil {

     

       95
       95
       +
       				return fmt.Errorf("error parsing at-uri: %w", err)

     

       96
       96
       +
       

     

       97
       97
       +
       			}

     

       98
       98
       +
       			post.ParentDid = aturi.Authority().String()

     

       99
       99
       +
       			post.ParentUri = rec.Reply.Parent.Uri

     

       100
       100
       +
       		}

     

       101
       101
       +
       		if rec.Reply.Root != nil {

     

       102
       102
       +
       			aturi, err := syntax.ParseATURI(rec.Reply.Root.Uri)

     

       103
       103
       +
       			if err != nil {

     

       104
       104
       +
       				return fmt.Errorf("error parsing at-uri: %w", err)

     

       105
       105
       +
       

     

       106
       106
       +
       			}

     

       107
       107
       +
       			post.RootDid = aturi.Authority().String()

     

       108
       108
       +
       			post.RootUri = rec.Reply.Root.Uri

     

       109
       109
       +
       		}

     

       110
       110
       +
       	}

     

       111
       111
       +
       

     

       112
       112
       +
       	if rec.Embed != nil && rec.Embed.EmbedRecord != nil && rec.Embed.EmbedRecord.Record != nil {

     

       113
       113
       +
       		aturi, err := syntax.ParseATURI(rec.Embed.EmbedRecord.Record.Uri)

     

       114
       114
       +
       		if err != nil {

     

       115
       115
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       116
       116
       +
       

     

       117
       117
       +
       		}

     

       118
       118
       +
       		post.QuoteDid = aturi.Authority().String()

     

       119
       119
       +
       		post.QuoteUri = rec.Embed.EmbedRecord.Record.Uri

     

       120
       120
       +
       	} else if rec.Embed != nil && rec.Embed.EmbedRecordWithMedia != nil && rec.Embed.EmbedRecordWithMedia.Record != nil && rec.Embed.EmbedRecordWithMedia.Record.Record != nil {

     

       121
       121
       +
       		aturi, err := syntax.ParseATURI(rec.Embed.EmbedRecordWithMedia.Record.Record.Uri)

     

       122
       122
       +
       		if err != nil {

     

       123
       123
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       124
       124
       +
       

     

       125
       125
       +
       		}

     

       126
       126
       +
       		post.QuoteDid = aturi.Authority().String()

     

       127
       127
       +
       		post.QuoteUri = rec.Embed.EmbedRecordWithMedia.Record.Record.Uri

     

       128
       128
       +
       	}

     

       129
       129
       +
       

     

       130
       130
       +
       	if err := p.inserters.postsInserter.Insert(ctx, post); err != nil {

     

       131
       131
       +
       		return err

     

       132
       132
       +
       	}

     

       133
       133
       +
       

     

       134
       134
       +
       	isEn := slices.Contains(rec.Langs, "en")

     

       135
       135
       +
       	if rec.Text != "" && rec.Reply == nil && isEn && p.nervanaClient != nil {

     

       136
       136
       +
       		go func(ctx context.Context, rec bsky.FeedPost, did, rkey string) {

     

       137
       137
       +
       			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)

     

       138
       138
       +
       			defer cancel()

     

       139
       139
       +
       

     

       140
       140
       +
       			nervanaItems, err := p.nervanaClient.MakeRequest(ctx, rec.Text)

     

       141
       141
       +
       			if err != nil {

     

       142
       142
       +
       				p.logger.Error("error making nervana items request", "error", err)

     

       143
       143
       +
       				return

     

       144
       144
       +
       			}

     

       145
       145
       +
       

     

       146
       146
       +
       			for _, ni := range nervanaItems {

     

       147
       147
       +
       				postLabel := models.PostLabel{

     

       148
       148
       +
       					Did:         did,

     

       149
       149
       +
       					Rkey:        rkey,

     

       150
       150
       +
       					Text:        ni.Text,

     

       151
       151
       +
       					Label:       ni.Label,

     

       152
       152
       +
       					EntityId:    ni.EntityId,

     

       153
       153
       +
       					Description: ni.Description,

     

       154
       154
       +
       					Topic:       "",

     

       155
       155
       +
       					CreatedAt:   time.Now(),

     

       156
       156
       +
       				}

     

       157
       157
       +
       				p.inserters.labelsInserter.Insert(ctx, postLabel)

     

       158
       158
       +
       			}

     

       159
       159
       +
       		}(ctx, rec, did, rkey)

     

       160
       160
       +
       	}

     

       161
       161
       +
       

     

       162
       162
       +
       	return nil

     

       163
       163
       +
       }

     

       164
       164
       +
       

     

       165
       165
       +
       func (p *Photocopy) handleCreateFollow(ctx context.Context, recb []byte, uri, did, rkey string, indexedAt time.Time) error {

     

       166
       166
       +
       	var rec bsky.GraphFollow

     

       167
       167
       +
       	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       168
       168
       +
       		return err

     

       169
       169
       +
       	}

     

       170
       170
       +
       

     

       171
       171
       +
       	cat, err := parseTimeFromRecord(rec, rkey)

     

       172
       172
       +
       	if err != nil {

     

       173
       173
       +
       		return err

     

       174
       174
       +
       	}

     

       175
       175
       +
       

     

       176
       176
       +
       	follow := models.Follow{

     

       177
       177
       +
       		Uri:       uri,

     

       178
       178
       +
       		Did:       did,

     

       179
       179
       +
       		Rkey:      rkey,

     

       180
       180
       +
       		CreatedAt: *cat,

     

       181
       181
       +
       		IndexedAt: indexedAt,

     

       182
       182
       +
       		Subject:   rec.Subject,

     

       183
       183
       +
       	}

     

       184
       184
       +
       

     

       185
       185
       +
       	if err := p.inserters.followsInserter.Insert(ctx, follow); err != nil {

     

       186
       186
       +
       		return err

     

       187
       187
       +
       	}

     

       188
       188
       +
       

     

       189
       189
       +
       	return nil

     

       190
       190
       +
       }

     

       191
       191
       +
       

     

       192
       192
       +
       func (p *Photocopy) handleCreateInteraction(ctx context.Context, recb []byte, uri, did, collection, rkey string, indexedAt time.Time) error {

     

       193
       193
       +
       	colPts := strings.Split(collection, ".")

     

       194
       194
       +
       	if len(colPts) < 4 {

     

       195
       195
       +
       		return fmt.Errorf("invalid collection type %s", collection)

     

       196
       196
       +
       	}

     

       197
       197
       +
       

     

       198
       198
       +
       	interaction := models.Interaction{

     

       199
       199
       +
       		Uri:        uri,

     

       200
       200
       +
       		Kind:       colPts[3],

     

       201
       201
       +
       		Rkey:       rkey,

     

       202
       202
       +
       		IndexedAt:  indexedAt,

     

       203
       203
       +
       		Did:        did,

     

       204
       204
       +
       		SubjectUri: uri,

     

       205
       205
       +
       		SubjectDid: did,

     

       206
       206
       +
       	}

     

       207
       207
       +
       

     

       208
       208
       +
       	switch collection {

     

       209
       209
       +
       	case "app.bsky.feed.like":

     

       210
       210
       +
       		var rec bsky.FeedLike

     

       211
       211
       +
       		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       212
       212
       +
       			return err

     

       213
       213
       +
       		}

     

       214
       214
       +
       

     

       215
       215
       +
       		cat, err := parseTimeFromRecord(rec, rkey)

     

       216
       216
       +
       		if err != nil {

     

       217
       217
       +
       			return err

     

       218
       218
       +
       		}

     

       219
       219
       +
       

     

       220
       220
       +
       		if rec.Subject == nil {

     

       221
       221
       +
       			return fmt.Errorf("invalid subject in like")

     

       222
       222
       +
       		}

     

       223
       223
       +
       

     

       224
       224
       +
       		aturi, err := syntax.ParseATURI(rec.Subject.Uri)

     

       225
       225
       +
       		if err != nil {

     

       226
       226
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       227
       227
       +
       

     

       228
       228
       +
       		}

     

       229
       229
       +
       

     

       230
       230
       +
       		interaction.SubjectDid = aturi.Authority().String()

     

       231
       231
       +
       		interaction.SubjectUri = rec.Subject.Uri

     

       232
       232
       +
       		interaction.CreatedAt = *cat

     

       233
       233
       +
       	case "app.bsky.feed.repost":

     

       234
       234
       +
       		var rec bsky.FeedRepost

     

       235
       235
       +
       		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       236
       236
       +
       			return err

     

       237
       237
       +
       		}

     

       238
       238
       +
       

     

       239
       239
       +
       		cat, err := parseTimeFromRecord(rec, rkey)

     

       240
       240
       +
       		if err != nil {

     

       241
       241
       +
       			return err

     

       242
       242
       +
       		}

     

       243
       243
       +
       

     

       244
       244
       +
       		if rec.Subject == nil {

     

       245
       245
       +
       			return fmt.Errorf("invalid subject in repost")

     

       246
       246
       +
       		}

     

       247
       247
       +
       

     

       248
       248
       +
       		aturi, err := syntax.ParseATURI(rec.Subject.Uri)

     

       249
       249
       +
       		if err != nil {

     

       250
       250
       +
       			return fmt.Errorf("error parsing at-uri: %w", err)

     

       251
       251
       +
       

     

       252
       252
       +
       		}

     

       253
       253
       +
       

     

       254
       254
       +
       		interaction.SubjectDid = aturi.Authority().String()

     

       255
       255
       +
       		interaction.SubjectUri = rec.Subject.Uri

     

       256
       256
       +
       		interaction.CreatedAt = *cat

     

       257
       257
       +
       	}

     

       258
       258
       +
       

     

       259
       259
       +
       	if err := p.inserters.interactionsInserter.Insert(ctx, interaction); err != nil {

     

       260
       260
       +
       		return err

     

       261
       261
       +
       	}

     

       69
       262
        
       

     

       70
       70
       -
       // func (p *Photocopy) handleCreateFollow(ctx context.Context, recb []byte, uri, did, rkey string, indexedAt time.Time) error {

     

       71
       71
       -
       // 	var rec bsky.GraphFollow

     

       72
       72
       -
       // 	if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       73
       73
       -
       // 		return err

     

       74
       74
       -
       // 	}

     

       75
       75
       -
       //

     

       76
       76
       -
       // 	cat, err := parseTimeFromRecord(rec, rkey)

     

       77
       77
       -
       // 	if err != nil {

     

       78
       78
       -
       // 		return err

     

       79
       79
       -
       // 	}

     

       80
       80
       -
       //

     

       81
       81
       -
       // 	bqrec := &BigQueryFollow{

     

       82
       82
       -
       // 		Uri:        uri,

     

       83
       83
       -
       // 		AuthorDid:  did,

     

       84
       84
       -
       // 		Rkey:       rkey,

     

       85
       85
       -
       // 		CreatedAt:  *cat,

     

       86
       86
       -
       // 		IndexedAt:  indexedAt,

     

       87
       87
       -
       // 		SubjectDid: rec.Subject,

     

       88
       88
       -
       // 	}

     

       89
       89
       -
       //

     

       90
       90
       -
       // 	if err := p.inserters.followsInserter.Insert(ctx, bqrec); err != nil {

     

       91
       91
       -
       // 		return err

     

       92
       92
       -
       // 	}

     

       93
       93
       -
       //

     

       94
       94
       -
       // 	return nil

     

       95
       95
       -
       // }

     

       96
       96
       -
       //

     

       97
       97
       -
       // func (p *Photocopy) handleCreateInteraction(ctx context.Context, recb []byte, uri, did, collection, rkey string, indexedAt time.Time) error {

     

       98
       98
       -
       // 	colPts := strings.Split(collection, ".")

     

       99
       99
       -
       // 	if len(colPts) < 4 {

     

       100
       100
       -
       // 		return fmt.Errorf("invalid collection type %s", collection)

     

       101
       101
       -
       // 	}

     

       102
       102
       -
       //

     

       103
       103
       -
       // 	bqi := &BigQueryInteraction{

     

       104
       104
       -
       // 		Uri:       uri,

     

       105
       105
       -
       // 		Kind:      colPts[3],

     

       106
       106
       -
       // 		AuthorDid: did,

     

       107
       107
       -
       // 		Rkey:      rkey,

     

       108
       108
       -
       // 		IndexedAt: indexedAt,

     

       109
       109
       -
       // 	}

     

       110
       110
       -
       //

     

       111
       111
       -
       // 	switch collection {

     

       112
       112
       -
       // 	case "app.bsky.feed.like":

     

       113
       113
       -
       // 		var rec bsky.FeedLike

     

       114
       114
       -
       // 		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       115
       115
       -
       // 			return err

     

       116
       116
       -
       // 		}

     

       117
       117
       -
       //

     

       118
       118
       -
       // 		cat, err := parseTimeFromRecord(rec, rkey)

     

       119
       119
       -
       // 		if err != nil {

     

       120
       120
       -
       // 			return err

     

       121
       121
       -
       // 		}

     

       122
       122
       -
       //

     

       123
       123
       -
       // 		if rec.Subject == nil {

     

       124
       124
       -
       // 			return fmt.Errorf("invalid subject in like")

     

       125
       125
       -
       // 		}

     

       126
       126
       -
       //

     

       127
       127
       -
       // 		bqi.SubjectUri = rec.Subject.Uri

     

       128
       128
       -
       // 		bqi.CreatedAt = *cat

     

       129
       129
       -
       // 	case "app.bsky.feed.repost":

     

       130
       130
       -
       // 		var rec bsky.FeedRepost

     

       131
       131
       -
       // 		if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {

     

       132
       132
       -
       // 			return err

     

       133
       133
       -
       // 		}

     

       134
       134
       -
       //

     

       135
       135
       -
       // 		cat, err := parseTimeFromRecord(rec, rkey)

     

       136
       136
       -
       // 		if err != nil {

     

       137
       137
       -
       // 			return err

     

       138
       138
       -
       // 		}

     

       139
       139
       -
       //

     

       140
       140
       -
       // 		if rec.Subject == nil {

     

       141
       141
       -
       // 			return fmt.Errorf("invalid subject in repost")

     

       142
       142
       -
       // 		}

     

       143
       143
       -
       //

     

       144
       144
       -
       // 		bqi.SubjectUri = rec.Subject.Uri

     

       145
       145
       -
       // 		bqi.CreatedAt = *cat

     

       146
       146
       -
       // 	}

     

       147
       147
       -
       //

     

       148
       148
       -
       // 	if err := p.inserters.interactionsInserter.Insert(ctx, bqi); err != nil {

     

       149
       149
       -
       // 		return err

     

       150
       150
       -
       // 	}

     

       151
       151
       -
       //

     

       152
       152
       -
       // 	return nil

     

       153
       153
       -
       // }

     

       263
       263
       +
       	return nil

     

       264
       264
       +
       }

     

       154
       265
        
       

     

       155
       266
        
       func parseTimeFromRecord(rec any, rkey string) (*time.Time, error) {

     

       156
       267
        
       	var rkeyTime time.Time

+13

handle_delete.go

···

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
        
       	"context"

     

       5
       5
       +
       	"time"

     

       6
       6
       +
       

     

       7
       7
       +
       	"github.com/haileyok/photocopy/models"

     

       5
       8
        
       )

     

       6
       9
        
       

     

       7
       10
        
       func (p *Photocopy) handleDelete(ctx context.Context, did, collection, rkey string) error {

     

       11
       11
       +
       	del := models.Delete{

     

       12
       12
       +
       		Did:       did,

     

       13
       13
       +
       		Rkey:      rkey,

     

       14
       14
       +
       		CreatedAt: time.Now(),

     

       15
       15
       +
       	}

     

       16
       16
       +
       

     

       17
       17
       +
       	if err := p.inserters.deletesInserter.Insert(ctx, del); err != nil {

     

       18
       18
       +
       		return err

     

       19
       19
       +
       	}

     

       20
       20
       +
       

     

       8
       21
        
       	return nil

     

       9
       22
        
       }

-131

internal/inserter.go

···

       1
       1
       -
       package inserter

     

       2
       2
       -
       

     

       3
       3
       -
       import (

     

       4
       4
       -
       	"context"

     

       5
       5
       -
       	"database/sql/driver"

     

       6
       6
       -
       	"log/slog"

     

       7
       7
       -
       	"slices"

     

       8
       8
       -
       	"sync"

     

       9
       9
       -
       	"time"

     

       10
       10
       -
       

     

       11
       11
       -
       	"github.com/prometheus/client_golang/prometheus"

     

       12
       12
       -
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       13
       13
       -
       )

     

       14
       14
       -
       

     

       15
       15
       -
       type Inserter struct {

     

       16
       16
       -
       	db             driver.Conn

     

       17
       17
       -
       	mu             sync.Mutex

     

       18
       18
       -
       	queuedEvents   []any

     

       19
       19
       -
       	batchSize      int

     

       20
       20
       -
       	insertsCounter *prometheus.CounterVec

     

       21
       21
       -
       	pendingSends   prometheus.Gauge

     

       22
       22
       -
       	histogram      *prometheus.HistogramVec

     

       23
       23
       -
       	logger         *slog.Logger

     

       24
       24
       -
       	prefix         string

     

       25
       25
       -
       }

     

       26
       26
       -
       

     

       27
       27
       -
       type BaseArgs struct {

     

       28
       28
       -
       	CredentialsPath string

     

       29
       29
       -
       	ProjectID       string

     

       30
       30
       -
       	DatasetID       string

     

       31
       31
       -
       	TableID         string

     

       32
       32
       -
       }

     

       33
       33
       -
       

     

       34
       34
       -
       type Args struct {

     

       35
       35
       -
       	BaseArgs

     

       36
       36
       -
       	BatchSize               int

     

       37
       37
       -
       	PrometheusCounterPrefix string

     

       38
       38
       -
       	Logger                  *slog.Logger

     

       39
       39
       -
       	Histogram               *prometheus.HistogramVec

     

       40
       40
       -
       }

     

       41
       41
       -
       

     

       42
       42
       -
       func New(ctx context.Context, args *Args) (*Inserter, error) {

     

       43
       43
       -
       	if args.Logger == nil {

     

       44
       44
       -
       		args.Logger = slog.Default()

     

       45
       45
       -
       	}

     

       46
       46
       -
       

     

       47
       47
       -
       	// dataset := bqc.Dataset(args.DatasetID)

     

       48
       48
       -
       	// table := dataset.Table(args.TableID)

     

       49
       49
       -
       

     

       50
       50
       -
       	inserter := &Inserter{}

     

       51
       51
       -
       

     

       52
       52
       -
       	if args.PrometheusCounterPrefix != "" {

     

       53
       53
       -
       		inserter.insertsCounter = promauto.NewCounterVec(prometheus.CounterOpts{

     

       54
       54
       -
       			Name:      "clickhouse_inserts",

     

       55
       55
       -
       			Namespace: args.PrometheusCounterPrefix,

     

       56
       56
       -
       			Help:      "total inserts into clickhouse by status",

     

       57
       57
       -
       		}, []string{"status"})

     

       58
       58
       -
       

     

       59
       59
       -
       		inserter.pendingSends = promauto.NewGauge(prometheus.GaugeOpts{

     

       60
       60
       -
       			Name:      "clickhouse_pending_sends",

     

       61
       61
       -
       			Namespace: args.PrometheusCounterPrefix,

     

       62
       62
       -
       			Help:      "total clickhouse insertions that are in progress",

     

       63
       63
       -
       		})

     

       64
       64
       -
       

     

       65
       65
       -
       	} else {

     

       66
       66
       -
       		args.Logger.Info("no prometheus prefix provided, no metrics will be registered for this counter", "dataset", args.DatasetID, "table", args.TableID)

     

       67
       67
       -
       	}

     

       68
       68
       -
       

     

       69
       69
       -
       	return inserter, nil

     

       70
       70
       -
       }

     

       71
       71
       -
       

     

       72
       72
       -
       func (i *Inserter) Insert(ctx context.Context, e any) error {

     

       73
       73
       -
       	i.mu.Lock()

     

       74
       74
       -
       

     

       75
       75
       -
       	i.queuedEvents = append(i.queuedEvents, e)

     

       76
       76
       -
       

     

       77
       77
       -
       	var toInsert []any

     

       78
       78
       -
       	if len(i.queuedEvents) >= i.batchSize {

     

       79
       79
       -
       		toInsert = slices.Clone(i.queuedEvents)

     

       80
       80
       -
       		i.queuedEvents = nil

     

       81
       81
       -
       	}

     

       82
       82
       -
       

     

       83
       83
       -
       	i.mu.Unlock()

     

       84
       84
       -
       

     

       85
       85
       -
       	if len(toInsert) > 0 {

     

       86
       86
       -
       		i.sendStream(ctx, toInsert)

     

       87
       87
       -
       	}

     

       88
       88
       -
       

     

       89
       89
       -
       	return nil

     

       90
       90
       -
       }

     

       91
       91
       -
       

     

       92
       92
       -
       func (i *Inserter) Close(ctx context.Context) error {

     

       93
       93
       -
       	i.mu.Lock()

     

       94
       94
       -
       

     

       95
       95
       -
       	var toInsert []any

     

       96
       96
       -
       

     

       97
       97
       -
       	if len(i.queuedEvents) > 0 {

     

       98
       98
       -
       		toInsert = slices.Clone(i.queuedEvents)

     

       99
       99
       -
       		i.queuedEvents = nil

     

       100
       100
       -
       	}

     

       101
       101
       -
       

     

       102
       102
       -
       	i.mu.Unlock()

     

       103
       103
       -
       

     

       104
       104
       -
       	if len(toInsert) > 0 {

     

       105
       105
       -
       		i.sendStream(ctx, toInsert)

     

       106
       106
       -
       	}

     

       107
       107
       -
       

     

       108
       108
       -
       	return nil

     

       109
       109
       -
       }

     

       110
       110
       -
       

     

       111
       111
       -
       func (i *Inserter) sendStream(ctx context.Context, toInsert []any) {

     

       112
       112
       -
       	i.pendingSends.Inc()

     

       113
       113
       -
       	defer i.pendingSends.Dec()

     

       114
       114
       -
       

     

       115
       115
       -
       	if i.histogram != nil {

     

       116
       116
       -
       		start := time.Now()

     

       117
       117
       -
       		defer func() {

     

       118
       118
       -
       			i.histogram.WithLabelValues(i.prefix).Observe(time.Since(start).Seconds())

     

       119
       119
       -
       		}()

     

       120
       120
       -
       	}

     

       121
       121
       -
       

     

       122
       122
       -
       	if len(toInsert) == 0 {

     

       123
       123
       -
       		return

     

       124
       124
       -
       	}

     

       125
       125
       -
       

     

       126
       126
       -
       	status := "ok"

     

       127
       127
       -
       

     

       128
       128
       -
       	// TODO: do the insert

     

       129
       129
       -
       

     

       130
       130
       -
       	i.insertsCounter.WithLabelValues(status).Add(float64(len(toInsert)))

     

       131
       131
       -
       }

models/delete.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Delete struct {

     

       6
       6
       +
       	Did       string    `ch:"did"`

     

       7
       7
       +
       	Rkey      string    `ch:"rkey"`

     

       8
       8
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       9
       9
       +
       }

+12

models/follow.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Follow struct {

     

       6
       6
       +
       	Uri       string    `ch:"uri"`

     

       7
       7
       +
       	Did       string    `ch:"did"`

     

       8
       8
       +
       	Rkey      string    `ch:"rkey"`

     

       9
       9
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       10
       10
       +
       	IndexedAt time.Time `ch:"indexed_at"`

     

       11
       11
       +
       	Subject   string    `ch:"subject"`

     

       12
       12
       +
       }

+14

models/interaction.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Interaction struct {

     

       6
       6
       +
       	Uri        string    `ch:"uri"`

     

       7
       7
       +
       	Did        string    `ch:"did"`

     

       8
       8
       +
       	Rkey       string    `ch:"rkey"`

     

       9
       9
       +
       	Kind       string    `ch:"kind"`

     

       10
       10
       +
       	CreatedAt  time.Time `ch:"created_at"`

     

       11
       11
       +
       	IndexedAt  time.Time `ch:"indexed_at"`

     

       12
       12
       +
       	SubjectUri string    `ch:"subject_uri"`

     

       13
       13
       +
       	SubjectDid string    `ch:"subject_did"`

     

       14
       14
       +
       }

+19

models/post.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Post struct {

     

       6
       6
       +
       	Uri       string    `ch:"uri"`

     

       7
       7
       +
       	Did       string    `ch:"did"`

     

       8
       8
       +
       	Rkey      string    `ch:"rkey"`

     

       9
       9
       +
       	CreatedAt time.Time `ch:"created_at"`

     

       10
       10
       +
       	IndexedAt time.Time `ch:"indexed_at"`

     

       11
       11
       +
       	RootUri   string    `ch:"root_uri"`

     

       12
       12
       +
       	RootDid   string    `ch:"root_did"`

     

       13
       13
       +
       	ParentUri string    `ch:"parent_uri"`

     

       14
       14
       +
       	ParentDid string    `ch:"parent_did"`

     

       15
       15
       +
       	QuoteUri  string    `ch:"quote_uri"`

     

       16
       16
       +
       	QuoteDid  string    `ch:"quote_did"`

     

       17
       17
       +
       	Lang      string    `ch:"lang"`

     

       18
       18
       +
       	Text      string    `ch:"text"`

     

       19
       19
       +
       }

+14

models/post_label.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type PostLabel struct {

     

       6
       6
       +
       	Did         string    `ch:"did"`

     

       7
       7
       +
       	Rkey        string    `ch:"rkey"`

     

       8
       8
       +
       	CreatedAt   time.Time `ch:"created_at"`

     

       9
       9
       +
       	Text        string    `ch:"text"`

     

       10
       10
       +
       	Label       string    `ch:"label"`

     

       11
       11
       +
       	EntityId    string    `ch:"entity_id"`

     

       12
       12
       +
       	Description string    `ch:"description"`

     

       13
       13
       +
       	Topic       string    `ch:"topic"`

     

       14
       14
       +
       }

+13

models/record.go

···

       1
       1
       +
       package models

     

       2
       2
       +
       

     

       3
       3
       +
       import "time"

     

       4
       4
       +
       

     

       5
       5
       +
       type Record struct {

     

       6
       6
       +
       	Did        string    `ch:"did"`

     

       7
       7
       +
       	Rkey       string    `ch:"rkey"`

     

       8
       8
       +
       	Collection string    `ch:"collection"`

     

       9
       9
       +
       	Cid        string    `ch:"cid"`

     

       10
       10
       +
       	Seq        string    `ch:"seq"`

     

       11
       11
       +
       	Raw        string    `ch:"raw"`

     

       12
       12
       +
       	CreatedAt  time.Time `ch:"created_at"`

     

       13
       13
       +
       }

-1

models.go

···

       1
       1
       -
       package photocopy

+77

nervana/client.go

···

       1
       1
       +
       package nervana

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"bytes"

     

       5
       5
       +
       	"context"

     

       6
       6
       +
       	"encoding/json"

     

       7
       7
       +
       	"fmt"

     

       8
       8
       +
       	"io"

     

       9
       9
       +
       	"net/http"

     

       10
       10
       +
       	"time"

     

       11
       11
       +
       )

     

       12
       12
       +
       

     

       13
       13
       +
       type Client struct {

     

       14
       14
       +
       	cli      *http.Client

     

       15
       15
       +
       	endpoint string

     

       16
       16
       +
       	apiKey   string

     

       17
       17
       +
       }

     

       18
       18
       +
       

     

       19
       19
       +
       func NewClient(endpoint string, apiKey string) *Client {

     

       20
       20
       +
       	return &Client{

     

       21
       21
       +
       		cli: &http.Client{

     

       22
       22
       +
       			Timeout: 5 * time.Second,

     

       23
       23
       +
       		},

     

       24
       24
       +
       		endpoint: endpoint,

     

       25
       25
       +
       		apiKey:   apiKey,

     

       26
       26
       +
       	}

     

       27
       27
       +
       }

     

       28
       28
       +
       

     

       29
       29
       +
       type NervanaItem struct {

     

       30
       30
       +
       	Text        string `json:"text"`

     

       31
       31
       +
       	Label       string `json:"label"`

     

       32
       32
       +
       	EntityId    string `json:"entityId"`

     

       33
       33
       +
       	Description string `json:"description"`

     

       34
       34
       +
       }

     

       35
       35
       +
       

     

       36
       36
       +
       func (c *Client) newRequest(ctx context.Context, text string) (*http.Request, error) {

     

       37
       37
       +
       	payload := map[string]string{

     

       38
       38
       +
       		"text":     text,

     

       39
       39
       +
       		"language": "en",

     

       40
       40
       +
       	}

     

       41
       41
       +
       

     

       42
       42
       +
       	b, err := json.Marshal(payload)

     

       43
       43
       +
       	if err != nil {

     

       44
       44
       +
       		return nil, err

     

       45
       45
       +
       	}

     

       46
       46
       +
       

     

       47
       47
       +
       	req, err := http.NewRequestWithContext(ctx, "POST", c.endpoint, bytes.NewReader(b))

     

       48
       48
       +
       

     

       49
       49
       +
       	req.Header.Set("Authorization", "Bearer "+c.apiKey)

     

       50
       50
       +
       

     

       51
       51
       +
       	return req, err

     

       52
       52
       +
       }

     

       53
       53
       +
       

     

       54
       54
       +
       func (c *Client) MakeRequest(ctx context.Context, text string) ([]NervanaItem, error) {

     

       55
       55
       +
       	req, err := c.newRequest(ctx, text)

     

       56
       56
       +
       	if err != nil {

     

       57
       57
       +
       		return nil, err

     

       58
       58
       +
       	}

     

       59
       59
       +
       

     

       60
       60
       +
       	resp, err := c.cli.Do(req)

     

       61
       61
       +
       	if err != nil {

     

       62
       62
       +
       		return nil, err

     

       63
       63
       +
       	}

     

       64
       64
       +
       	defer resp.Body.Close()

     

       65
       65
       +
       

     

       66
       66
       +
       	if resp.StatusCode != 200 {

     

       67
       67
       +
       		io.Copy(io.Discard, resp.Body)

     

       68
       68
       +
       		return nil, fmt.Errorf("received non-200 response code: %d", resp.StatusCode)

     

       69
       69
       +
       	}

     

       70
       70
       +
       

     

       71
       71
       +
       	var nervanaResp []NervanaItem

     

       72
       72
       +
       	if err := json.NewDecoder(resp.Body).Decode(&nervanaResp); err != nil {

     

       73
       73
       +
       		return nil, err

     

       74
       74
       +
       	}

     

       75
       75
       +
       

     

       76
       76
       +
       	return nervanaResp, nil

     

       77
       77
       +
       }

+188 -12

photocopy.go

···

       8
       8
        
       	"sync"

     

       9
       9
        
       	"time"

     

       10
       10
        
       

     

       11
       11
       -
       	inserter "github.com/haileyok/photocopy/internal"

     

       11
       11
       +
       	"github.com/ClickHouse/clickhouse-go/v2"

     

       12
       12
       +
       	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"

     

       13
       13
       +
       	"github.com/haileyok/photocopy/clickhouse_inserter"

     

       14
       14
       +
       	"github.com/haileyok/photocopy/nervana"

     

       12
       15
        
       	"github.com/prometheus/client_golang/prometheus"

     

       13
       16
        
       	"github.com/prometheus/client_golang/prometheus/promauto"

     

       14
       17
        
       	"github.com/prometheus/client_golang/prometheus/promhttp"

     
···

       26
       29
        
       	inserters *Inserters

     

       27
       30
        
       

     

       28
       31
        
       	plcScraper *PLCScraper

     

       32
       32
       +
       

     

       33
       33
       +
       	ratelimitBypassKey string

     

       34
       34
       +
       

     

       35
       35
       +
       	conn driver.Conn

     

       36
       36
       +
       

     

       37
       37
       +
       	nervanaClient   *nervana.Client

     

       38
       38
       +
       	nervanaEndpoint string

     

       39
       39
       +
       	nervanaApiKey   string

     

       29
       40
        
       }

     

       30
       41
        
       

     

       31
       42
        
       type Inserters struct {

     

       32
       32
       -
       	followsInserter *inserter.Inserter

     

       33
       33
       -
       	plcInserter     *inserter.Inserter

     

       43
       43
       +
       	followsInserter      *clickhouse_inserter.Inserter

     

       44
       44
       +
       	interactionsInserter *clickhouse_inserter.Inserter

     

       45
       45
       +
       	postsInserter        *clickhouse_inserter.Inserter

     

       46
       46
       +
       	plcInserter          *clickhouse_inserter.Inserter

     

       47
       47
       +
       	recordsInserter      *clickhouse_inserter.Inserter

     

       48
       48
       +
       	deletesInserter      *clickhouse_inserter.Inserter

     

       49
       49
       +
       	labelsInserter       *clickhouse_inserter.Inserter

     

       34
       50
        
       }

     

       35
       51
        
       

     

       36
       52
        
       type Args struct {

     
···

       39
       55
        
       	MetricsAddr          string

     

       40
       56
        
       	CursorFile           string

     

       41
       57
        
       	PLCScraperCursorFile string

     

       58
       58
       +
       	ClickhouseAddr       string

     

       59
       59
       +
       	ClickhouseDatabase   string

     

       42
       60
        
       	ClickhouseUser       string

     

       43
       61
        
       	ClickhousePass       string

     

       62
       62
       +
       	RatelimitBypassKey   string

     

       63
       63
       +
       	NervanaEndpoint      string

     

       64
       64
       +
       	NervanaApiKey        string

     

       44
       65
        
       }

     

       45
       66
        
       

     

       46
       67
        
       func New(ctx context.Context, args *Args) (*Photocopy, error) {

     

       68
       68
       +
       	conn, err := clickhouse.Open(&clickhouse.Options{

     

       69
       69
       +
       		Addr: []string{args.ClickhouseAddr},

     

       70
       70
       +
       		Auth: clickhouse.Auth{

     

       71
       71
       +
       			Database: args.ClickhouseDatabase,

     

       72
       72
       +
       			Username: args.ClickhouseUser,

     

       73
       73
       +
       			Password: args.ClickhousePass,

     

       74
       74
       +
       		},

     

       75
       75
       +
       	})

     

       76
       76
       +
       	if err != nil {

     

       77
       77
       +
       		return nil, err

     

       78
       78
       +
       	}

     

       79
       79
       +
       

     

       47
       80
        
       	p := &Photocopy{

     

       48
       48
       -
       		logger:      args.Logger,

     

       49
       49
       -
       		metricsAddr: args.MetricsAddr,

     

       50
       50
       -
       		relayHost:   args.RelayHost,

     

       51
       51
       -
       		wg:          sync.WaitGroup{},

     

       52
       52
       -
       		cursorFile:  args.CursorFile,

     

       81
       81
       +
       		logger:             args.Logger,

     

       82
       82
       +
       		metricsAddr:        args.MetricsAddr,

     

       83
       83
       +
       		relayHost:          args.RelayHost,

     

       84
       84
       +
       		wg:                 sync.WaitGroup{},

     

       85
       85
       +
       		cursorFile:         args.CursorFile,

     

       86
       86
       +
       		ratelimitBypassKey: args.RatelimitBypassKey,

     

       87
       87
       +
       		conn:               conn,

     

       53
       88
        
       	}

     

       54
       89
        
       

     

       55
       90
        
       	insertionsHist := promauto.NewHistogramVec(prometheus.HistogramOpts{

     
···

       58
       93
        
       		Buckets: prometheus.ExponentialBucketsRange(0.0001, 30, 20),

     

       59
       94
        
       	}, []string{"type"})

     

       60
       95
        
       

     

       61
       61
       -
       	fi, err := inserter.New(ctx, &inserter.Args{

     

       96
       96
       +
       	fi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       62
       97
        
       		PrometheusCounterPrefix: "photocopy_follows",

     

       63
       98
        
       		Histogram:               insertionsHist,

     

       99
       99
       +
       		BatchSize:               500,

     

       100
       100
       +
       		Logger:                  p.logger,

     

       101
       101
       +
       		Conn:                    conn,

     

       102
       102
       +
       		Query:                   "INSERT INTO follow (uri, did, rkey, created_at, indexed_at, subject)",

     

       103
       103
       +
       		RateLimit:               3,

     

       104
       104
       +
       	})

     

       105
       105
       +
       	if err != nil {

     

       106
       106
       +
       		return nil, err

     

       107
       107
       +
       	}

     

       108
       108
       +
       

     

       109
       109
       +
       	pi, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       110
       110
       +
       		PrometheusCounterPrefix: "photocopy_posts",

     

       111
       111
       +
       		Histogram:               insertionsHist,

     

       112
       112
       +
       		BatchSize:               300,

     

       113
       113
       +
       		Logger:                  p.logger,

     

       114
       114
       +
       		Conn:                    conn,

     

       115
       115
       +
       		Query:                   "INSERT INTO post (uri, did, rkey, created_at, indexed_at, root_uri, root_did, parent_uri, parent_did, quote_uri, quote_did, lang)",

     

       116
       116
       +
       		RateLimit:               3,

     

       117
       117
       +
       	})

     

       118
       118
       +
       	if err != nil {

     

       119
       119
       +
       		return nil, err

     

       120
       120
       +
       	}

     

       121
       121
       +
       

     

       122
       122
       +
       	ii, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       123
       123
       +
       		PrometheusCounterPrefix: "photocopy_interactions",

     

       124
       124
       +
       		Histogram:               insertionsHist,

     

       125
       125
       +
       		BatchSize:               1000,

     

       126
       126
       +
       		Logger:                  p.logger,

     

       127
       127
       +
       		Conn:                    conn,

     

       128
       128
       +
       		Query:                   "INSERT INTO interaction (uri, did, rkey, kind, created_at, indexed_at, subject_uri, subject_did)",

     

       129
       129
       +
       		RateLimit:               3,

     

       130
       130
       +
       	})

     

       131
       131
       +
       	if err != nil {

     

       132
       132
       +
       		return nil, err

     

       133
       133
       +
       	}

     

       134
       134
       +
       

     

       135
       135
       +
       	ri, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       136
       136
       +
       		PrometheusCounterPrefix: "photocopy_records",

     

       137
       137
       +
       		Histogram:               insertionsHist,

     

       138
       138
       +
       		BatchSize:               2500,

     

       139
       139
       +
       		Logger:                  p.logger,

     

       140
       140
       +
       		Conn:                    conn,

     

       141
       141
       +
       		Query:                   "INSERT INTO record (did, rkey, collection, cid, seq, raw, created_at)",

     

       142
       142
       +
       		RateLimit:               3,

     

       143
       143
       +
       	})

     

       144
       144
       +
       	if err != nil {

     

       145
       145
       +
       		return nil, err

     

       146
       146
       +
       	}

     

       147
       147
       +
       

     

       148
       148
       +
       	di, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       149
       149
       +
       		PrometheusCounterPrefix: "photocopy_deletes",

     

       150
       150
       +
       		Histogram:               insertionsHist,

     

       151
       151
       +
       		BatchSize:               500,

     

       152
       152
       +
       		Logger:                  p.logger,

     

       153
       153
       +
       		Conn:                    conn,

     

       154
       154
       +
       		Query:                   "INSERT INTO delete (did, rkey, created_at)",

     

       155
       155
       +
       		RateLimit:               3,

     

       156
       156
       +
       	})

     

       157
       157
       +
       	if err != nil {

     

       158
       158
       +
       		return nil, err

     

       159
       159
       +
       	}

     

       160
       160
       +
       

     

       161
       161
       +
       	li, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       162
       162
       +
       		PrometheusCounterPrefix: "photocopy_labels",

     

       163
       163
       +
       		Histogram:               insertionsHist,

     

       64
       164
        
       		BatchSize:               100,

     

       65
       165
        
       		Logger:                  p.logger,

     

       166
       166
       +
       		Conn:                    conn,

     

       167
       167
       +
       		Query:                   "INSERT INTO post_label (did, rkey, text, label, entity_id, description, topic, created_at)",

     

       168
       168
       +
       		RateLimit:               3,

     

       66
       169
        
       	})

     

       67
       170
        
       	if err != nil {

     

       68
       171
        
       		return nil, err

     

       69
       172
        
       	}

     

       70
       173
        
       

     

       71
       174
        
       	is := &Inserters{

     

       72
       72
       -
       		followsInserter: fi,

     

       175
       175
       +
       		followsInserter:      fi,

     

       176
       176
       +
       		postsInserter:        pi,

     

       177
       177
       +
       		interactionsInserter: ii,

     

       178
       178
       +
       		recordsInserter:      ri,

     

       179
       179
       +
       		deletesInserter:      di,

     

       180
       180
       +
       		labelsInserter:       li,

     

       73
       181
        
       	}

     

       74
       182
        
       

     

       75
       183
        
       	p.inserters = is

     

       76
       184
        
       

     

       77
       77
       -
       	plci, err := inserter.New(ctx, &inserter.Args{

     

       185
       185
       +
       	plci, err := clickhouse_inserter.New(ctx, &clickhouse_inserter.Args{

     

       78
       186
        
       		PrometheusCounterPrefix: "photocopy_plc_entries",

     

       79
       187
        
       		Histogram:               insertionsHist,

     

       80
       188
        
       		BatchSize:               100,

     

       81
       189
        
       		Logger:                  args.Logger,

     

       190
       190
       +
       		Conn:                    conn,

     

       191
       191
       +
       		Query: `INSERT INTO plc (

     

       192
       192
       +
       			did, cid, nullified, created_at, plc_op_sig, plc_op_prev, plc_op_type,

     

       193
       193
       +
       			plc_op_services, plc_op_also_known_as, plc_op_rotation_keys,

     

       194
       194
       +
       			 plc_tomb_sig, plc_tomb_prev, plc_tomb_type,

     

       195
       195
       +
       			legacy_op_sig, legacy_op_prev, legacy_op_type, legacy_op_handle,

     

       196
       196
       +
       			legacy_op_service, legacy_op_signing_key, legacy_op_recovery_key

     

       197
       197
       +
       		)`,

     

       82
       198
        
       	})

     

       83
       199
        
       	if err != nil {

     

       84
       200
        
       		return nil, err

     
···

       96
       212
        
       	p.inserters.plcInserter = plci

     

       97
       213
        
       	p.plcScraper = plcs

     

       98
       214
        
       

     

       215
       215
       +
       	if args.NervanaApiKey != "" && args.NervanaEndpoint != "" {

     

       216
       216
       +
       		p.nervanaClient = nervana.NewClient(args.NervanaEndpoint, args.NervanaApiKey)

     

       217
       217
       +
       	}

     

       218
       218
       +
       

     

       99
       219
        
       	return p, nil

     

       100
       220
        
       }

     

       101
       221
        
       

     

       102
       102
       -
       func (p *Photocopy) Run(baseCtx context.Context) error {

     

       222
       222
       +
       func (p *Photocopy) Run(baseCtx context.Context, withBackfill bool) error {

     

       103
       223
        
       	ctx, cancel := context.WithCancel(baseCtx)

     

       104
       224
        
       

     

       105
       225
        
       	metricsServer := http.NewServeMux()

     
···

       125
       245
        
       		}

     

       126
       246
        
       	}(ctx)

     

       127
       247
        
       

     

       248
       248
       +
       	if withBackfill {

     

       249
       249
       +
       		go func(ctx context.Context) {

     

       250
       250
       +
       			if err := p.runBackfiller(ctx); err != nil {

     

       251
       251
       +
       				panic(fmt.Errorf("error starting backfiller: %w", err))

     

       252
       252
       +
       			}

     

       253
       253
       +
       		}(ctx)

     

       254
       254
       +
       	}

     

       255
       255
       +
       

     

       128
       256
        
       	<-ctx.Done()

     

       129
       257
        
       

     

       130
       258
        
       	if p.inserters != nil {

     
···

       141
       269
        
       					return

     

       142
       270
        
       				}

     

       143
       271
        
       				p.logger.Info("follows inserter closed")

     

       272
       272
       +
       			}()

     

       273
       273
       +
       		}

     

       274
       274
       +
       

     

       275
       275
       +
       		if p.inserters.interactionsInserter != nil {

     

       276
       276
       +
       			p.wg.Add(1)

     

       277
       277
       +
       			go func() {

     

       278
       278
       +
       				defer p.wg.Done()

     

       279
       279
       +
       				if err := p.inserters.interactionsInserter.Close(ctx); err != nil {

     

       280
       280
       +
       					p.logger.Error("failed to close interactions inserter", "error", err)

     

       281
       281
       +
       					return

     

       282
       282
       +
       				}

     

       283
       283
       +
       				p.logger.Info("interactions inserter closed")

     

       284
       284
       +
       			}()

     

       285
       285
       +
       		}

     

       286
       286
       +
       

     

       287
       287
       +
       		if p.inserters.postsInserter != nil {

     

       288
       288
       +
       			p.wg.Add(1)

     

       289
       289
       +
       			go func() {

     

       290
       290
       +
       				defer p.wg.Done()

     

       291
       291
       +
       				if err := p.inserters.postsInserter.Close(ctx); err != nil {

     

       292
       292
       +
       					p.logger.Error("failed to close posts inserter", "error", err)

     

       293
       293
       +
       					return

     

       294
       294
       +
       				}

     

       295
       295
       +
       				p.logger.Info("posts inserter closed")

     

       296
       296
       +
       			}()

     

       297
       297
       +
       		}

     

       298
       298
       +
       

     

       299
       299
       +
       		if p.inserters.recordsInserter != nil {

     

       300
       300
       +
       			p.wg.Add(1)

     

       301
       301
       +
       			go func() {

     

       302
       302
       +
       				defer p.wg.Done()

     

       303
       303
       +
       				if err := p.inserters.recordsInserter.Close(ctx); err != nil {

     

       304
       304
       +
       					p.logger.Error("failed to close records inserter", "error", err)

     

       305
       305
       +
       					return

     

       306
       306
       +
       				}

     

       307
       307
       +
       				p.logger.Info("records inserter closed")

     

       308
       308
       +
       			}()

     

       309
       309
       +
       		}

     

       310
       310
       +
       

     

       311
       311
       +
       		if p.inserters.deletesInserter != nil {

     

       312
       312
       +
       			p.wg.Add(1)

     

       313
       313
       +
       			go func() {

     

       314
       314
       +
       				defer p.wg.Done()

     

       315
       315
       +
       				if err := p.inserters.deletesInserter.Close(ctx); err != nil {

     

       316
       316
       +
       					p.logger.Error("failed to close deletes inserter", "error", err)

     

       317
       317
       +
       					return

     

       318
       318
       +
       				}

     

       319
       319
       +
       				p.logger.Info("deletes inserter closed")

     

       144
       320
        
       			}()

     

       145
       321
        
       		}

     

       146
       322

+78 -58

plc_models.go

···

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
        
       	"encoding/json"

     

       5
       5
       -
       	"errors"

     

       6
       5
        
       	"fmt"

     

       7
       6
        
       	"time"

     

       8
       8
       -
       

     

       9
       9
       -
       	"cloud.google.com/go/bigquery"

     

       10
       7
        
       )

     

       11
       8
        
       

     

       12
       9
        
       type PLCEntry struct {

     

       13
       13
       -
       	Did       string           `json:"did" bigquery:"did"`

     

       14
       14
       -
       	Operation PLCOperationType `json:"operation" bigquery:"operation"`

     

       15
       15
       -
       	Cid       string           `json:"cid" bigquery:"cid"`

     

       16
       16
       -
       	Nullified bool             `json:"nullified" bigquery:"nullified"`

     

       17
       17
       -
       	CreatedAt time.Time        `json:"createdAt" bigquery:"created_at"`

     

       10
       10
       +
       	Did       string           `json:"did"`

     

       11
       11
       +
       	Operation PLCOperationType `json:"operation"`

     

       12
       12
       +
       	Cid       string           `json:"cid"`

     

       13
       13
       +
       	Nullified bool             `json:"nullified"`

     

       14
       14
       +
       	CreatedAt time.Time        `json:"createdAt"`

     

       18
       15
        
       }

     

       19
       16
        
       

     

       20
       17
        
       type PLCOperation struct {

     

       21
       21
       -
       	Sig                     string                `json:"sig" bigquery:"sig"`

     

       22
       22
       -
       	Prev                    bigquery.NullString   `json:"prev" bigquery:"prev"`

     

       23
       23
       -
       	Type                    string                `json:"type" bigquery:"type"`

     

       24
       24
       -
       	Services                map[string]PLCService `json:"services" bigquery:"-"`

     

       25
       25
       -
       	ServicesJSON            string                `json:"-" bigquery:"services"`

     

       26
       26
       -
       	AlsoKnownAs             []string              `json:"alsoKnownAs" bigquery:"also_known_as"`

     

       27
       27
       -
       	RotationKeys            []string              `json:"rotationKeys" bigquery:"rotation_keys"`

     

       28
       28
       -
       	VerificationMethods     map[string]string     `json:"verificationMethods" bigquery:"-"`

     

       29
       29
       -
       	VerificationMethodsJSON string                `json:"-" bigquery:"verification_methods"`

     

       18
       18
       +
       	Sig                 string                `json:"sig" `

     

       19
       19
       +
       	Prev                *string               `json:"prev"`

     

       20
       20
       +
       	Type                string                `json:"type"`

     

       21
       21
       +
       	Services            map[string]PLCService `json:"services"`

     

       22
       22
       +
       	AlsoKnownAs         []string              `json:"alsoKnownAs"`

     

       23
       23
       +
       	RotationKeys        []string              `json:"rotationKeys"`

     

       24
       24
       +
       	VerificationMethods map[string]string     `json:"verificationMethods"`

     

       30
       25
        
       }

     

       31
       26
        
       

     

       32
       27
        
       type PLCTombstone struct {

     
···

       51
       46
        
       }

     

       52
       47
        
       

     

       53
       48
        
       type PLCOperationType struct {

     

       54
       54
       -
       	OperationType      string              `json:"-" bigquery:"operation_type"`

     

       55
       55
       -
       	PLCOperation       *PLCOperation       `json:"-" bigquery:"plc_operation"`

     

       56
       56
       -
       	PLCTombstone       *PLCTombstone       `json:"-" bigquery:"plc_tombstone"`

     

       57
       57
       -
       	LegacyPLCOperation *LegacyPLCOperation `json:"-" bigquery:"legacy_plc_operation"`

     

       49
       49
       +
       	OperationType      string

     

       50
       50
       +
       	PLCOperation       *PLCOperation

     

       51
       51
       +
       	PLCTombstone       *PLCTombstone

     

       52
       52
       +
       	LegacyPLCOperation *LegacyPLCOperation

     

       53
       53
       +
       }

     

       54
       54
       +
       

     

       55
       55
       +
       type ClickhousePLCEntry struct {

     

       56
       56
       +
       	Did                 string    `ch:"did"`

     

       57
       57
       +
       	Cid                 string    `ch:"cid"`

     

       58
       58
       +
       	Nullified           bool      `ch:"nullified"`

     

       59
       59
       +
       	CreatedAt           time.Time `ch:"created_at"`

     

       60
       60
       +
       	PlcOpSig            string    `ch:"plc_op_sig"`

     

       61
       61
       +
       	PlcOpPrev           string    `ch:"plc_op_prev"`

     

       62
       62
       +
       	PlcOpType           string    `ch:"plc_op_type"`

     

       63
       63
       +
       	PlcOpServices       []string  `ch:"plc_op_services"`

     

       64
       64
       +
       	PlcOpAlsoKnownAs    []string  `ch:"plc_op_also_known_as"`

     

       65
       65
       +
       	PlcOpRotationKeys   []string  `ch:"plc_op_rotation_keys"`

     

       66
       66
       +
       	PlcTombSig          string    `ch:"plc_tomb_sig"`

     

       67
       67
       +
       	PlcTombPrev         string    `ch:"plc_tomb_prev"`

     

       68
       68
       +
       	PlcTombType         string    `ch:"plc_tomb_type"`

     

       69
       69
       +
       	LegacyOpSig         string    `ch:"legacy_op_sig"`

     

       70
       70
       +
       	LegacyOpPrev        string    `ch:"legacy_op_prev"`

     

       71
       71
       +
       	LegacyOpType        string    `ch:"legacy_op_type"`

     

       72
       72
       +
       	LegacyOpHandle      string    `ch:"legacy_op_handle"`

     

       73
       73
       +
       	LegacyOpService     string    `ch:"legacy_op_service"`

     

       74
       74
       +
       	LegacyOpSigningKey  string    `ch:"legacy_op_signing_key"`

     

       75
       75
       +
       	LegacyOpRecoveryKey string    `ch:"legacy_op_recovery_key"`

     

       58
       76
        
       }

     

       59
       77
        
       

     

       60
       78
        
       func (o *PLCOperationType) UnmarshalJSON(data []byte) error {

     
···

       105
       123
        
       	return nil

     

       106
       124
        
       }

     

       107
       125
        
       

     

       108
       108
       -
       func (o *PLCOperationType) MarshalJSON() ([]byte, error) {

     

       109
       109
       -
       	if o.PLCOperation != nil {

     

       110
       110
       -
       		return json.Marshal(o.PLCOperation)

     

       111
       111
       -
       	}

     

       112
       112
       -
       	if o.PLCTombstone != nil {

     

       113
       113
       -
       		return json.Marshal(o.PLCTombstone)

     

       114
       114
       -
       	}

     

       115
       115
       -
       	if o.LegacyPLCOperation != nil {

     

       116
       116
       -
       		return json.Marshal(o.LegacyPLCOperation)

     

       117
       117
       -
       	}

     

       118
       118
       -
       	return nil, errors.New("no valid operation type found")

     

       119
       119
       -
       }

     

       120
       120
       -
       

     

       121
       121
       -
       func (o *PLCOperationType) Value() (any, error) {

     

       122
       122
       -
       	return json.Marshal(o)

     

       123
       123
       -
       }

     

       124
       124
       -
       

     

       125
       125
       -
       func (o *PLCOperationType) Scan(value any) error {

     

       126
       126
       -
       	bytes, ok := value.([]byte)

     

       127
       127
       -
       	if !ok {

     

       128
       128
       -
       		return errors.New("could not scan PLCOperationType")

     

       126
       126
       +
       func (e *PLCEntry) prepareForClickhouse() (*ClickhousePLCEntry, error) {

     

       127
       127
       +
       	che := &ClickhousePLCEntry{

     

       128
       128
       +
       		Did:       e.Did,

     

       129
       129
       +
       		Cid:       e.Cid,

     

       130
       130
       +
       		Nullified: e.Nullified,

     

       131
       131
       +
       		CreatedAt: e.CreatedAt,

     

       129
       132
        
       	}

     

       130
       130
       -
       	return json.Unmarshal(bytes, o)

     

       131
       131
       -
       }

     

       132
       132
       -
       

     

       133
       133
       -
       func (e *PLCEntry) prepareForBigQuery() (map[string]bigquery.Value, string, error) {

     

       134
       133
        
       	if e.Operation.PLCOperation != nil {

     

       135
       135
       -
       		if e.Operation.PLCOperation.Services != nil {

     

       136
       136
       -
       			b, err := json.Marshal(e.Operation.PLCOperation.Services)

     

       137
       137
       -
       			if err != nil {

     

       138
       138
       -
       				return nil, "", fmt.Errorf("error marshaling services: %w", err)

     

       139
       139
       -
       			}

     

       140
       140
       -
       			e.Operation.PLCOperation.ServicesJSON = string(b)

     

       134
       134
       +
       		pop := e.Operation.PLCOperation

     

       135
       135
       +
       		che.PlcOpSig = pop.Sig

     

       136
       136
       +
       		if pop.Prev != nil {

     

       137
       137
       +
       			che.PlcOpPrev = *pop.Prev

     

       141
       138
        
       		}

     

       142
       142
       -
       		if e.Operation.PLCOperation.VerificationMethods != nil {

     

       143
       143
       -
       			b, err := json.Marshal(e.Operation.PLCOperation.VerificationMethods)

     

       144
       144
       -
       			if err != nil {

     

       145
       145
       -
       				return nil, "", fmt.Errorf("error marshaling verification methods: %w", err)

     

       139
       139
       +
       		che.PlcOpType = pop.Type

     

       140
       140
       +
       		services := []string{}

     

       141
       141
       +
       		for _, s := range pop.Services {

     

       142
       142
       +
       			services = append(services, s.Endpoint)

     

       143
       143
       +
       		}

     

       144
       144
       +
       		che.PlcOpServices = services

     

       145
       145
       +
       		che.PlcOpAlsoKnownAs = pop.AlsoKnownAs

     

       146
       146
       +
       		che.PlcOpRotationKeys = pop.RotationKeys

     

       147
       147
       +
       		if e.Operation.PLCOperation.Services == nil {

     

       148
       148
       +
       			for _, s := range e.Operation.PLCOperation.Services {

     

       149
       149
       +
       				che.PlcOpServices = append(che.PlcOpServices, s.Endpoint)

     

       146
       150
        
       			}

     

       147
       147
       -
       			e.Operation.PLCOperation.VerificationMethodsJSON = string(b)

     

       148
       151
        
       		}

     

       152
       152
       +
       		return che, nil

     

       153
       153
       +
       	} else if e.Operation.PLCTombstone != nil {

     

       154
       154
       +
       		che.PlcTombSig = e.Operation.PLCTombstone.Sig

     

       155
       155
       +
       		che.PlcTombPrev = e.Operation.PLCTombstone.Prev

     

       156
       156
       +
       		che.PlcTombType = e.Operation.PLCTombstone.Type

     

       157
       157
       +
       		return che, nil

     

       158
       158
       +
       	} else if e.Operation.LegacyPLCOperation != nil {

     

       159
       159
       +
       		lop := e.Operation.LegacyPLCOperation

     

       160
       160
       +
       		che.LegacyOpSig = lop.Sig

     

       161
       161
       +
       		che.LegacyOpPrev = lop.Prev

     

       162
       162
       +
       		che.LegacyOpType = lop.Type

     

       163
       163
       +
       		che.LegacyOpService = lop.Service

     

       164
       164
       +
       		che.LegacyOpHandle = lop.Handle

     

       165
       165
       +
       		che.LegacyOpSigningKey = lop.SigningKey

     

       166
       166
       +
       		che.LegacyOpRecoveryKey = lop.RecoveryKey

     

       167
       167
       +
       		return che, nil

     

       149
       168
        
       	}

     

       150
       150
       -
       	return nil, "", nil

     

       169
       169
       +
       

     

       170
       170
       +
       	return nil, fmt.Errorf("no valid plc operation type")

     

       151
       171
        
       }

+10 -6

plc_scraper.go

···

       11
       11
        
       	"strings"

     

       12
       12
        
       	"time"

     

       13
       13
        
       

     

       14
       14
       -
       	inserter "github.com/haileyok/photocopy/internal"

     

       14
       14
       +
       	"github.com/haileyok/photocopy/clickhouse_inserter"

     

       15
       15
        
       )

     

       16
       16
        
       

     

       17
       17
        
       type PLCScraper struct {

     
···

       19
       19
        
       	logger     *slog.Logger

     

       20
       20
        
       	cursor     string

     

       21
       21
        
       	cursorFile string

     

       22
       22
       -
       	inserter   *inserter.Inserter

     

       22
       22
       +
       	inserter   *clickhouse_inserter.Inserter

     

       23
       23
        
       }

     

       24
       24
        
       

     

       25
       25
        
       type PLCScraperArgs struct {

     

       26
       26
        
       	Logger     *slog.Logger

     

       27
       27
       -
       	Inserter   *inserter.Inserter

     

       27
       27
       +
       	Inserter   *clickhouse_inserter.Inserter

     

       28
       28
        
       	CursorFile string

     

       29
       29
        
       }

     

       30
       30
        
       

     
···

       73
       73
        
       			ustr += "&after=" + s.cursor

     

       74
       74
        
       			t, _ := time.Parse(time.RFC3339Nano, s.cursor)

     

       75
       75
        
       			if time.Since(t) > 1*time.Hour {

     

       76
       76
       -
       				setTickerDuration(600 * time.Millisecond)

     

       76
       76
       +
       				setTickerDuration(800 * time.Millisecond)

     

       77
       77
        
       			} else {

     

       78
       78
        
       				setTickerDuration(3 * time.Second)

     

       79
       79
        
       			}

     
···

       129
       129
        
       				s.saveCursor(s.cursor)

     

       130
       130
        
       			}

     

       131
       131
        
       

     

       132
       132
       -
       			entry.prepareForBigQuery()

     

       132
       132
       +
       			chEntry, err := entry.prepareForClickhouse()

     

       133
       133
       +
       			if err != nil {

     

       134
       134
       +
       				s.logger.Error("error getting clickhouse entry from plc entry", "error", err)

     

       135
       135
       +
       				continue

     

       136
       136
       +
       			}

     

       133
       137
        
       

     

       134
       134
       -
       			s.inserter.Insert(ctx, entry)

     

       138
       138
       +
       			s.inserter.Insert(ctx, *chEntry)

     

       135
       139
        
       		}

     

       136
       140
        
       	}

     

       137
       141

Compare changes