···
"github.com/bluesky-social/jetstream/pkg/client"
"github.com/bluesky-social/jetstream/pkg/client/schedulers/sequential"
"github.com/bluesky-social/jetstream/pkg/models"
type HandleEvent func(context.Context, *models.Event) error
-
func startJetstreamLoop(logger *slog.Logger, outStream **client.Client, name string, handleEvent HandleEvent, optsFn func() models.SubscriberOptionsUpdatePayload) {
-
stream, startFn, err := startJetstreamClient(name, optsFn(), handleEvent)
-
logger.Error("stream failed", "name", name, "error", err)
-
func startJetstreamClient(name string, opts models.SubscriberOptionsUpdatePayload, handleEvent HandleEvent) (*client.Client, func() error, error) {
-
ctx := context.Background()
config := client.DefaultClientConfig()
config.WebsocketURL = "wss://jetstream1.us-west.bsky.network/subscribe"
-
config.WantedCollections = opts.WantedCollections
-
config.WantedDids = opts.WantedDIDs
-
config.RequireHello = false
scheduler := sequential.NewScheduler(name, logger, handleEvent)
c, err := client.NewClient(config, logger, scheduler)
-
logger.Error("failed to create jetstream client", "name", name, "error", err)
startFn := func() error {
-
logger.Info("starting jetstream client", "name", name, "collections", opts.WantedCollections, "wanted_dids", len(opts.WantedDIDs))
if err := c.ConnectAndRead(ctx, nil); err != nil {
-
logger.Error("jetstream client failed", "name", name, "error", err)
···
"github.com/bluesky-social/jetstream/pkg/client"
"github.com/bluesky-social/jetstream/pkg/client/schedulers/sequential"
"github.com/bluesky-social/jetstream/pkg/models"
+
func Chunk[T any](slice []T, chunkSize int) [][]T {
+
for i := 0; i < len(slice); i += chunkSize {
+
end := min(i+chunkSize, len(slice))
+
chunks = append(chunks, slice[i:end])
+
cancel context.CancelFunc
+
type StreamManager struct {
+
handleEvent HandleEvent
+
func NewStreamManager(logger *slog.Logger, name string, handleEvent HandleEvent, optsFn OptsFn) StreamManager {
+
logger: logger.With("stream", name),
+
streamsLock: sync.Mutex{},
+
streams: make(map[int]Stream),
+
handleEvent: handleEvent,
+
// doesnt lock streams!!!
+
func (manager *StreamManager) startSingle(id int, opts models.SubscriberOptionsUpdatePayload) {
+
ctx, cancel := context.WithCancel(manager.ctx)
+
stream := Stream{inner: nil, cancel: cancel}
+
// add to streams and put on wait group
+
manager.streams[id] = stream
+
go startJetstreamLoop(ctx, manager.logger.With("streamId", id), &stream.inner, fmt.Sprintf("%s_%d", manager.name, id), manager.handleEvent, opts)
+
func (manager *StreamManager) chunkedOpts() ([]models.SubscriberOptionsUpdatePayload, int) {
+
results := make([]models.SubscriberOptionsUpdatePayload, 0)
+
opts := manager.optsFn()
+
for _, wantedDidsChunk := range Chunk(opts.WantedDIDs, 9999) {
+
results = append(results, models.SubscriberOptionsUpdatePayload{
+
WantedCollections: opts.WantedCollections,
+
WantedDIDs: wantedDidsChunk,
+
MaxMessageSizeBytes: opts.MaxMessageSizeBytes,
+
return results, len(opts.WantedDIDs)
+
func (manager *StreamManager) updateOpts() {
+
chunks, userCount := manager.chunkedOpts()
+
manager.streamsLock.Lock()
+
idsSeen := make(map[int]struct{}, 0)
+
// update existing streams or create new ones
+
for id, opts := range chunks {
+
idsSeen[id] = struct{}{}
+
if len(manager.streams) > id {
+
stream := manager.streams[id]
+
if stream.inner == nil {
+
if err := stream.inner.SendOptionsUpdate(opts); err != nil {
+
manager.logger.Error("couldnt update follow stream opts", "error", err, "streamId", id)
+
manager.startSingle(id, opts)
+
// cancel and delete unused streams
+
for k := range manager.streams {
+
if _, exists := idsSeen[k]; !exists {
+
manager.streams[k].cancel()
+
delete(manager.streams, k)
+
manager.streamsLock.Unlock()
+
manager.logger.Info("updated opts", "userCount", userCount)
type HandleEvent func(context.Context, *models.Event) error
+
type OptsFn func() models.SubscriberOptionsUpdatePayload
+
func startJetstreamLoop(ctx context.Context, logger *slog.Logger, outStream **client.Client, name string, handleEvent HandleEvent, opts models.SubscriberOptionsUpdatePayload) {
+
done := make(chan struct{})
+
stream, startFn, err := startJetstreamClient(ctx, logger, name, handleEvent)
+
logger.Info("starting jetstream client", "collections", opts.WantedCollections, "userCount", len(opts.WantedDIDs))
+
// HACK: we need to wait for the websocket connection to start here. so we do
+
// need to upstream something to jetstream client
+
time.Sleep(time.Second * 2)
+
err = stream.SendOptionsUpdate(opts)
+
logger.Error("stream failed", "error", err, "backoff", backoff)
+
func startJetstreamClient(ctx context.Context, logger *slog.Logger, name string, handleEvent HandleEvent) (*client.Client, func() error, error) {
config := client.DefaultClientConfig()
config.WebsocketURL = "wss://jetstream1.us-west.bsky.network/subscribe"
+
config.RequireHello = true
scheduler := sequential.NewScheduler(name, logger, handleEvent)
c, err := client.NewClient(config, logger, scheduler)
+
logger.Error("failed to create jetstream client", "error", err)
startFn := func() error {
if err := c.ConnectAndRead(ctx, nil); err != nil {
+
logger.Error("jetstream client failed", "error", err)
ptr.pet