···

       
3
       
 
       
import (

     

       
4
       
 
       
	"context"

     

       
5
       
 
       
	"log/slog"

     

       
6
       
-
       
	"sync"

     

       
7
       
 
       
	"time"

     

       
8
       
 
       


     

       
9
       
 
       
	"gorm.io/gorm"

     

       
10
       
 
       
)

     

       
11
       
 
       


     

       
12
       
 
       
type CursorService struct {

     

       
13
       
-
       
	store *gorm.DB

     

       
14
       
-
       


     

       
15
       
-
       
	firehoseLk  sync.Mutex

     

       
16
       
 
       
	firehoseSeq int64

     

       
17
       
 
       
}

     

       
18
       
 
       


     

···

       
3
       
 
       
import (

     

       
4
       
 
       
	"context"

     

       
5
       
 
       
	"log/slog"

     

       
0
       
 
       

     

       
6
       
 
       
	"time"

     

       
7
       
 
       


     

       
8
       
 
       
	"gorm.io/gorm"

     

       
9
       
 
       
)

     

       
10
       
 
       


     

       
11
       
 
       
type CursorService struct {

     

       
12
       
+
       
	store       *gorm.DB

     

       
0
       
 
       

     

       
0
       
 
       

     

       
13
       
 
       
	firehoseSeq int64

     

       
14
       
 
       
}

     

       
15
       
 
       


     

···

       
13
       
 
       
func NewFirehoseConnection(ctx context.Context, cctx *cli.Context, cursorSvc *CursorService) (*websocket.Conn, error) {

     

       
14
       
 
       
	url := fmt.Sprintf("wss://%s/xrpc/com.atproto.sync.subscribeRepos", cctx.String("relay-host"))

     

       
15
       
 
       
	curs, err := cursorSvc.GetFirehoseCursor()

     

       
16
       
-
       
	if err == nil { // reversed

     

       
0
       
 
       

     

       
0
       
 
       

     

       
17
       
 
       
		url += fmt.Sprintf("?cursor=%d", curs)

     

       
18
       
 
       
	}

     

       
19
       
 
       


     

···

       
13
       
 
       
func NewFirehoseConnection(ctx context.Context, cctx *cli.Context, cursorSvc *CursorService) (*websocket.Conn, error) {

     

       
14
       
 
       
	url := fmt.Sprintf("wss://%s/xrpc/com.atproto.sync.subscribeRepos", cctx.String("relay-host"))

     

       
15
       
 
       
	curs, err := cursorSvc.GetFirehoseCursor()

     

       
16
       
+
       
	if err != nil {

     

       
17
       
+
       
		slog.Error("error getting firehose cursor", "err", err)

     

       
18
       
+
       
	} else if curs > 0 {

     

       
19
       
 
       
		url += fmt.Sprintf("?cursor=%d", curs)

     

       
20
       
 
       
	}

     

       
21
       
 
       


     

···

       
7
       
 
       
	"time"

     

       
8
       
 
       


     

       
9
       
 
       
	"github.com/bluesky-social/indigo/api/atproto"

     

       
10
       
-
       
	"github.com/bluesky-social/indigo/backfill"

     

       
11
       
 
       
	"github.com/bluesky-social/indigo/xrpc"

     

       
12
       
 
       
	"github.com/urfave/cli/v2"

     

       
13
       
 
       
	"golang.org/x/sync/semaphore"

     
···

       
19
       
 
       
	seenHosts map[string]bool

     

       
20
       
 
       
}

     

       
21
       
 
       


     

       
22
       
-
       
type jobMaker interface {

     

       
23
       
-
       
	GetOrCreateJob(context.Context, string, string) (backfill.Job, error)

     

       
24
       
-
       
}

     

       
25
       
-
       


     

       
26
       
 
       
func NewCensusService(cursorSvc *CursorService, backfillSvc *backfill.Backfiller) *CensusService {

     

       
27
       
 
       
	return &CensusService{

     

       
28
       
 
       
		cursor:    cursorSvc,

     
···

       
102
       
 
       
		Host: "https://" + host,

     

       
103
       
 
       
	}

     

       
104
       
 
       


     

       
105
       
-
       
	jmstore, ok := cs.backfill.Store.(jobMaker)

     

       
106
       
-
       
	if !ok {

     

       
107
       
-
       
		slog.Error("configured job store doesn't support random job creation")

     

       
108
       
-
       
		return

     

       
109
       
-
       
	}

     

       
110
       
-
       


     

       
111
       
 
       
	hcur, err := cs.cursor.GetHostCursor(host)

     

       
112
       
 
       
	if err != nil {

     

       
113
       
 
       
		slog.Error("error fetching host cursor", "err", err)

     
···

       
131
       
 
       
		}

     

       
132
       
 
       


     

       
133
       
 
       
		for _, repo := range res.Repos {

     

       
134
       
-
       
			_, err := jmstore.GetOrCreateJob(ctx, repo.Did, backfill.StateEnqueued)

     

       
135
       
 
       
			if err != nil {

     

       
136
       
 
       
				slog.Error("error adding repo to backfiller", "err", err)

     

       
137
       
 
       
			} else {

     

···

       
7
       
 
       
	"time"

     

       
8
       
 
       


     

       
9
       
 
       
	"github.com/bluesky-social/indigo/api/atproto"

     

       
10
       
+
       
	backfill "github.com/bluesky-social/indigo/backfill/next"

     

       
11
       
 
       
	"github.com/bluesky-social/indigo/xrpc"

     

       
12
       
 
       
	"github.com/urfave/cli/v2"

     

       
13
       
 
       
	"golang.org/x/sync/semaphore"

     
···

       
19
       
 
       
	seenHosts map[string]bool

     

       
20
       
 
       
}

     

       
21
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
22
       
 
       
func NewCensusService(cursorSvc *CursorService, backfillSvc *backfill.Backfiller) *CensusService {

     

       
23
       
 
       
	return &CensusService{

     

       
24
       
 
       
		cursor:    cursorSvc,

     
···

       
98
       
 
       
		Host: "https://" + host,

     

       
99
       
 
       
	}

     

       
100
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
101
       
 
       
	hcur, err := cs.cursor.GetHostCursor(host)

     

       
102
       
 
       
	if err != nil {

     

       
103
       
 
       
		slog.Error("error fetching host cursor", "err", err)

     
···

       
121
       
 
       
		}

     

       
122
       
 
       


     

       
123
       
 
       
		for _, repo := range res.Repos {

     

       
124
       
+
       
			err := cs.backfill.EnqueueJob(ctx, host, repo.Did)

     

       
125
       
 
       
			if err != nil {

     

       
126
       
 
       
				slog.Error("error adding repo to backfiller", "err", err)

     

       
127
       
 
       
			} else {