···

       
5
       
 
       
	"errors"

     

       
6
       
 
       
	"fmt"

     

       
7
       
 
       
	"log"

     

       
0
       
 
       

     

       
8
       
 
       
	"net/http"

     

       
9
       
 
       
	"os"

     

       
10
       
 
       
	"os/signal"

     
···

       
41
       
 
       
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGINT, syscall.SIGTERM)

     

       
42
       
 
       
	defer stop()

     

       
43
       
 
       


     

       
0
       
 
       

     

       
0
       
 
       

     

       
44
       
 
       
	newLogger := logger.New(

     

       
45
       
 
       
		log.New(os.Stdout, "\n", log.LstdFlags),

     

       
46
       
 
       
		logger.Config{

     
···

       
48
       
 
       
			Colorful: false,

     

       
49
       
 
       
		},

     

       
50
       
 
       
	)

     

       
51
       
-
       


     

       
52
       
 
       
	db, err := gorm.Open(sqlite.Open("state.db"), &gorm.Config{

     

       
53
       
 
       
		Logger: newLogger,

     

       
54
       
 
       
	})

     

       
55
       
 
       
	if err != nil {

     

       
56
       
-
       
		log.Fatalf("failed to connect database: %s", err)

     

       
57
       
 
       
	}

     

       
58
       
 
       
	db.AutoMigrate(&backfill.GormDBJob{})

     

       
59
       
 
       
	store := backfill.NewGormstore(db)

     
···

       
63
       
 
       
		"User-Agent": []string{"backfiller/0.1 (@edavis.dev)"},

     

       
64
       
 
       
	})

     

       
65
       
 
       
	if err != nil {

     

       
66
       
-
       
		log.Fatalf("failed to connect to relay: %s", err)

     

       
67
       
 
       
	}

     

       
68
       
 
       


     

       
69
       
 
       
	// start backfilling

     
···

       
79
       
 
       
	// pump repos

     

       
80
       
 
       
	go func(bf *backfill.Backfiller) {

     

       
81
       
 
       
		if err := pumpRepos(context.TODO(), bf); err != nil {

     

       
82
       
-
       
			log.Printf("failed pumping repos: %s", err)

     

       
83
       
 
       
		}

     

       
84
       
 
       
	}(bf)

     

       
85
       
 
       


     
···

       
100
       
 
       
	}

     

       
101
       
 
       
	sched := parallel.NewScheduler(16, 100, "firehose", rsc.EventHandler)

     

       
102
       
 
       
	if err := events.HandleRepoStream(ctx, con, sched, nil); err != nil {

     

       
103
       
-
       
		log.Fatalf("failed to start scheduler: %s", err)

     

       
104
       
 
       
	}

     

       
105
       
 
       


     

       
106
       
 
       
	<-ctx.Done()

     
···

       
112
       
 
       
}

     

       
113
       
 
       


     

       
114
       
 
       
func pumpRepos(ctx context.Context, bf *backfill.Backfiller) error {

     

       
0
       
 
       

     

       
0
       
 
       

     

       
115
       
 
       
	xrpcc := &xrpc.Client{

     

       
116
       
 
       
		Host: "https://bsky.network",

     

       
117
       
 
       
	}

     
···

       
123
       
 
       


     

       
124
       
 
       
	var curs string

     

       
125
       
 
       
	for {

     

       
126
       
-
       
		log.Printf("listing repos with cursor = %v", curs)

     

       
127
       
 
       
		res, err := atproto.SyncListRepos(ctx, xrpcc, curs, 1000)

     

       
128
       
 
       
		if err != nil {

     

       
129
       
 
       
			return fmt.Errorf("error listing repos: %w", err)

     
···

       
132
       
 
       
		for _, repo := range res.Repos {

     

       
133
       
 
       
			_, err := jmstore.GetOrCreateJob(ctx, repo.Did, backfill.StateEnqueued)

     

       
134
       
 
       
			if err != nil {

     

       
135
       
-
       
				log.Printf("failed to create backfill job: %s", err)

     

       
136
       
 
       
				continue

     

       
137
       
 
       
			}

     

       
138
       
 
       
		}

     

···

       
5
       
 
       
	"errors"

     

       
6
       
 
       
	"fmt"

     

       
7
       
 
       
	"log"

     

       
8
       
+
       
	"log/slog"

     

       
9
       
 
       
	"net/http"

     

       
10
       
 
       
	"os"

     

       
11
       
 
       
	"os/signal"

     
···

       
42
       
 
       
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGINT, syscall.SIGTERM)

     

       
43
       
 
       
	defer stop()

     

       
44
       
 
       


     

       
45
       
+
       
	sl := slog.With("source", "backfiller")

     

       
46
       
+
       


     

       
47
       
 
       
	newLogger := logger.New(

     

       
48
       
 
       
		log.New(os.Stdout, "\n", log.LstdFlags),

     

       
49
       
 
       
		logger.Config{

     
···

       
51
       
 
       
			Colorful: false,

     

       
52
       
 
       
		},

     

       
53
       
 
       
	)

     

       
0
       
 
       

     

       
54
       
 
       
	db, err := gorm.Open(sqlite.Open("state.db"), &gorm.Config{

     

       
55
       
 
       
		Logger: newLogger,

     

       
56
       
 
       
	})

     

       
57
       
 
       
	if err != nil {

     

       
58
       
+
       
		sl.Error("failed to connect to database", "err", err)

     

       
59
       
 
       
	}

     

       
60
       
 
       
	db.AutoMigrate(&backfill.GormDBJob{})

     

       
61
       
 
       
	store := backfill.NewGormstore(db)

     
···

       
65
       
 
       
		"User-Agent": []string{"backfiller/0.1 (@edavis.dev)"},

     

       
66
       
 
       
	})

     

       
67
       
 
       
	if err != nil {

     

       
68
       
+
       
		sl.Error("failed to connect to relay", "err", err)

     

       
69
       
 
       
	}

     

       
70
       
 
       


     

       
71
       
 
       
	// start backfilling

     
···

       
81
       
 
       
	// pump repos

     

       
82
       
 
       
	go func(bf *backfill.Backfiller) {

     

       
83
       
 
       
		if err := pumpRepos(context.TODO(), bf); err != nil {

     

       
84
       
+
       
			sl.Error("failed pumping repos", "err", err)

     

       
85
       
 
       
		}

     

       
86
       
 
       
	}(bf)

     

       
87
       
 
       


     
···

       
102
       
 
       
	}

     

       
103
       
 
       
	sched := parallel.NewScheduler(16, 100, "firehose", rsc.EventHandler)

     

       
104
       
 
       
	if err := events.HandleRepoStream(ctx, con, sched, nil); err != nil {

     

       
105
       
+
       
		sl.Error("failed to start scheduler", "err", err)

     

       
106
       
 
       
	}

     

       
107
       
 
       


     

       
108
       
 
       
	<-ctx.Done()

     
···

       
114
       
 
       
}

     

       
115
       
 
       


     

       
116
       
 
       
func pumpRepos(ctx context.Context, bf *backfill.Backfiller) error {

     

       
117
       
+
       
	sl := slog.With("source", "pumpRepos")

     

       
118
       
+
       


     

       
119
       
 
       
	xrpcc := &xrpc.Client{

     

       
120
       
 
       
		Host: "https://bsky.network",

     

       
121
       
 
       
	}

     
···

       
127
       
 
       


     

       
128
       
 
       
	var curs string

     

       
129
       
 
       
	for {

     

       
130
       
+
       
		sl.Info("listing repos", "cursor", curs)

     

       
131
       
 
       
		res, err := atproto.SyncListRepos(ctx, xrpcc, curs, 1000)

     

       
132
       
 
       
		if err != nil {

     

       
133
       
 
       
			return fmt.Errorf("error listing repos: %w", err)

     
···

       
136
       
 
       
		for _, repo := range res.Repos {

     

       
137
       
 
       
			_, err := jmstore.GetOrCreateJob(ctx, repo.Did, backfill.StateEnqueued)

     

       
138
       
 
       
			if err != nil {

     

       
139
       
+
       
				sl.Warn("failed to create backfill job", "err", err)

     

       
140
       
 
       
				continue

     

       
141
       
 
       
			}

     

       
142
       
 
       
		}