···

       
7
       
 
       
	"fmt"

     

       
8
       
 
       
	"io"

     

       
9
       
 
       
	"net/http"

     

       
0
       
 
       

     

       
10
       
 
       
	"strings"

     

       
11
       
 
       
	"sync"

     

       
12
       
 
       
	"time"

     
···

       
20
       
 
       
	"go.uber.org/ratelimit"

     

       
21
       
 
       
)

     

       
22
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
23
       
 
       
type RepoDownloader struct {

     

       
24
       
-
       
	clients    map[string]*http.Client

     

       
25
       
-
       
	rateLimits map[string]ratelimit.Limiter

     

       
26
       
-
       
	mu         sync.RWMutex

     

       
27
       
-
       
	p          *Photocopy

     

       
0
       
 
       

     

       
28
       
 
       
}

     

       
29
       
 
       


     

       
30
       
 
       
func NewRepoDownloader(p *Photocopy) *RepoDownloader {

     

       
31
       
 
       
	return &RepoDownloader{

     

       
32
       
-
       
		clients:    make(map[string]*http.Client),

     

       
33
       
-
       
		rateLimits: make(map[string]ratelimit.Limiter),

     

       
34
       
-
       
		p:          p,

     

       
0
       
 
       

     

       
35
       
 
       
	}

     

       
36
       
 
       
}

     

       
37
       
 
       


     
···

       
122
       
 
       
	bs := atproto_repo.NewTinyBlockstore()

     

       
123
       
 
       
	cs, err := car.NewCarReader(bytes.NewReader(b))

     

       
124
       
 
       
	if err != nil {

     

       
0
       
 
       

     

       
125
       
 
       
		return fmt.Errorf("error opening car: %v\n", err)

     

       
126
       
 
       
	}

     

       
127
       
 
       


     
···

       
134
       
 
       


     

       
135
       
 
       
	r, err := repo.OpenRepo(context.TODO(), bs, cs.Header.Roots[0])

     

       
136
       
 
       
	if err != nil || r == nil {

     

       
0
       
 
       

     

       
137
       
 
       
		fmt.Printf("could not open repo: %v", err)

     

       
138
       
 
       
		return nil

     

       
139
       
 
       
	}

     
···

       
238
       
 
       
	Status   string `json:"status"`

     

       
239
       
 
       
}

     

       
240
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
241
       
 
       
func (p *Photocopy) runBackfiller(ctx context.Context) error {

     

       
242
       
 
       
	startTime := time.Now()

     

       
243
       
 
       


     

       
244
       
 
       
	fmt.Println("querying clickhouse for dids and services...")

     

       
245
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
246
       
 
       
	var hostsCursor string

     

       
247
       
 
       
	var sevs []ListServicesResponseItem

     

       
248
       
 
       
	for {

     
···

       
299
       
 
       
	downloader := NewRepoDownloader(p)

     

       
300
       
 
       
	serviceDids := map[string][]string{}

     

       
301
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
302
       
 
       
	wg := sync.WaitGroup{}

     

       
303
       
 
       
	mplk := sync.Mutex{}

     

       
304
       
 
       
	for s := range servicesDids {

     
···

       
312
       
 
       
			}

     

       
313
       
 
       
			dids := []string{}

     

       
314
       
 
       
			for _, r := range repos {

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
315
       
 
       
				dids = append(dids, r.Did)

     

       
316
       
 
       
			}

     

       
317
       
 
       
			mplk.Lock()

     
···

       
365
       
 
       
				}

     

       
366
       
 
       


     

       
367
       
 
       
				go func(b []byte, did string) {

     

       
368
       
-
       
					if err := p.processRepo(ctx, b, did); err != nil {

     

       
369
       
-
       
						fmt.Printf("error processing backfill record: %v\n", err)

     

       
370
       
-
       
					}

     

       
371
       
 
       
				}(b, did)

     

       
372
       
 
       


     

       
373
       
 
       
				processed++

     

···

       
7
       
 
       
	"fmt"

     

       
8
       
 
       
	"io"

     

       
9
       
 
       
	"net/http"

     

       
10
       
+
       
	"runtime"

     

       
11
       
 
       
	"strings"

     

       
12
       
 
       
	"sync"

     

       
13
       
 
       
	"time"

     
···

       
21
       
 
       
	"go.uber.org/ratelimit"

     

       
22
       
 
       
)

     

       
23
       
 
       


     

       
24
       
+
       
type ProcessJob struct {

     

       
25
       
+
       
	did       string

     

       
26
       
+
       
	repoBytes []byte

     

       
27
       
+
       
}

     

       
28
       
+
       


     

       
29
       
 
       
type RepoDownloader struct {

     

       
30
       
+
       
	clients     map[string]*http.Client

     

       
31
       
+
       
	rateLimits  map[string]ratelimit.Limiter

     

       
32
       
+
       
	processChan chan ProcessJob

     

       
33
       
+
       
	mu          sync.RWMutex

     

       
34
       
+
       
	p           *Photocopy

     

       
35
       
 
       
}

     

       
36
       
 
       


     

       
37
       
 
       
func NewRepoDownloader(p *Photocopy) *RepoDownloader {

     

       
38
       
 
       
	return &RepoDownloader{

     

       
39
       
+
       
		clients:     make(map[string]*http.Client),

     

       
40
       
+
       
		rateLimits:  make(map[string]ratelimit.Limiter),

     

       
41
       
+
       
		p:           p,

     

       
42
       
+
       
		processChan: make(chan ProcessJob, 1000),

     

       
43
       
 
       
	}

     

       
44
       
 
       
}

     

       
45
       
 
       


     
···

       
130
       
 
       
	bs := atproto_repo.NewTinyBlockstore()

     

       
131
       
 
       
	cs, err := car.NewCarReader(bytes.NewReader(b))

     

       
132
       
 
       
	if err != nil {

     

       
133
       
+
       
		fmt.Println("error opening car", err)

     

       
134
       
 
       
		return fmt.Errorf("error opening car: %v\n", err)

     

       
135
       
 
       
	}

     

       
136
       
 
       


     
···

       
143
       
 
       


     

       
144
       
 
       
	r, err := repo.OpenRepo(context.TODO(), bs, cs.Header.Roots[0])

     

       
145
       
 
       
	if err != nil || r == nil {

     

       
146
       
+
       
		fmt.Println("error opening repo", err)

     

       
147
       
 
       
		fmt.Printf("could not open repo: %v", err)

     

       
148
       
 
       
		return nil

     

       
149
       
 
       
	}

     
···

       
248
       
 
       
	Status   string `json:"status"`

     

       
249
       
 
       
}

     

       
250
       
 
       


     

       
251
       
+
       
func (p *Photocopy) runProcessRepoWorker(ctx context.Context, jobs <-chan ProcessJob) {

     

       
252
       
+
       
	for j := range jobs {

     

       
253
       
+
       
		p.processRepo(ctx, j.repoBytes, j.did)

     

       
254
       
+
       
	}

     

       
255
       
+
       
}

     

       
256
       
+
       


     

       
257
       
 
       
func (p *Photocopy) runBackfiller(ctx context.Context) error {

     

       
258
       
 
       
	startTime := time.Now()

     

       
259
       
 
       


     

       
260
       
 
       
	fmt.Println("querying clickhouse for dids and services...")

     

       
261
       
 
       


     

       
262
       
+
       
	var alreadyFetched []string

     

       
263
       
+
       
	if err := p.conn.Select(ctx, &alreadyFetched, "SELECT DISTINCT(did) FROM default.record WHERE created_at < '2025-07-01'"); err != nil {

     

       
264
       
+
       
		return err

     

       
265
       
+
       
	}

     

       
266
       
+
       


     

       
267
       
+
       
	alreadyFetchedMap := map[string]bool{}

     

       
268
       
+
       
	for _, d := range alreadyFetched {

     

       
269
       
+
       
		alreadyFetchedMap[d] = true

     

       
270
       
+
       
	}

     

       
271
       
+
       


     

       
272
       
+
       
	fmt.Println("getting dids")

     

       
273
       
+
       


     

       
274
       
 
       
	var hostsCursor string

     

       
275
       
 
       
	var sevs []ListServicesResponseItem

     

       
276
       
 
       
	for {

     
···

       
327
       
 
       
	downloader := NewRepoDownloader(p)

     

       
328
       
 
       
	serviceDids := map[string][]string{}

     

       
329
       
 
       


     

       
330
       
+
       
	for range runtime.NumCPU() / 2 {

     

       
331
       
+
       
		go p.runProcessRepoWorker(ctx, downloader.processChan)

     

       
332
       
+
       
	}

     

       
333
       
+
       


     

       
334
       
 
       
	wg := sync.WaitGroup{}

     

       
335
       
 
       
	mplk := sync.Mutex{}

     

       
336
       
 
       
	for s := range servicesDids {

     
···

       
344
       
 
       
			}

     

       
345
       
 
       
			dids := []string{}

     

       
346
       
 
       
			for _, r := range repos {

     

       
347
       
+
       
				if alreadyFetchedMap[r.Did] {

     

       
348
       
+
       
					skipped++

     

       
349
       
+
       
					continue

     

       
350
       
+
       
				}

     

       
351
       
 
       
				dids = append(dids, r.Did)

     

       
352
       
 
       
			}

     

       
353
       
 
       
			mplk.Lock()

     
···

       
401
       
 
       
				}

     

       
402
       
 
       


     

       
403
       
 
       
				go func(b []byte, did string) {

     

       
404
       
+
       
					downloader.processChan <- ProcessJob{repoBytes: b, did: did}

     

       
0
       
 
       

     

       
0
       
 
       

     

       
405
       
 
       
				}(b, did)

     

       
406
       
 
       


     

       
407
       
 
       
				processed++