knotclient: introduce CursorStore #235

merged

opened by

oppi.li 6 months ago targeting master from push-qoplqnlvlqqo

consumers can configure a cursor-store, where cursors of individual event sources are stored. the module provides an in-memory store and a redis-backed store.

Signed-off-by: oppiliappan me@oppi.li

options

unified split

Changed files

+77 -9

cmd

eventconsumer

main.go

knotclient

events.go

+11 -7

cmd/eventconsumer/main.go

···

       11
       11
        
       )

     

       12
       12
        
       

     

       13
       13
        
       func main() {

     

       14
       14
       -
       	sourcesFlag := flag.String("sources", "", "list of wss sources")

     

       14
       14
       +
       	knots := flag.String("knots", "", "list of knots to connect to")

     

       15
       15
        
       	retryFlag := flag.Duration("retry", 1*time.Minute, "retry interval")

     

       16
       16
        
       	maxRetryFlag := flag.Duration("max-retry", 30*time.Minute, "max retry interval")

     

       17
       17
        
       	workerCount := flag.Int("workers", 10, "goroutine pool size")

     

       18
       18
        
       

     

       19
       19
        
       	flag.Parse()

     

       20
       20
        
       

     

       21
       21
       -
       	if *sourcesFlag == "" {

     

       22
       22
       -
       		fmt.Println("error: -sources is required")

     

       21
       21
       +
       	if *knots == "" {

     

       22
       22
       +
       		fmt.Println("error: -knots is required")

     

       23
       23
        
       		flag.Usage()

     

       24
       24
        
       		return

     

       25
       25
        
       	}

     

       26
       26
        
       

     

       27
       27
       -
       	sources := strings.Split(*sourcesFlag, ",")

     

       27
       27
       +
       	var srcs []knotclient.EventSource

     

       28
       28
       +
       	for k := range strings.SplitSeq(*knots, ",") {

     

       29
       29
       +
       		srcs = append(srcs, knotclient.EventSource{k})

     

       30
       30
       +
       	}

     

       28
       31
        
       

     

       29
       32
        
       	consumer := knotclient.NewEventConsumer(knotclient.ConsumerConfig{

     

       30
       30
       -
       		Sources:          sources,

     

       33
       33
       +
       		Sources:          srcs,

     

       31
       34
        
       		ProcessFunc:      processEvent,

     

       32
       35
        
       		RetryInterval:    *retryFlag,

     

       33
       36
        
       		MaxRetryInterval: *maxRetryFlag,

     

       34
       37
        
       		WorkerCount:      *workerCount,

     

       38
       38
       +
       		Dev:              true,

     

       35
       39
        
       	})

     

       36
       40
        
       

     

       37
       41
        
       	ctx, cancel := context.WithCancel(context.Background())

     
···

       41
       45
        
       	consumer.Stop()

     

       42
       46
        
       }

     

       43
       47
        
       

     

       44
       44
       -
       func processEvent(source string, msg []byte) error {

     

       45
       45
       -
       	fmt.Printf("From %s: %s\n", source, string(msg))

     

       48
       48
       +
       func processEvent(source knotclient.EventSource, msg knotclient.Message) error {

     

       49
       49
       +
       	fmt.Printf("From %s (%s, %s): %s\n", source.Knot, msg.Rkey, msg.Nsid, string(msg.EventJson))

     

       46
       50
        
       	return nil

     

       47
       51
        
       }

+66 -2

knotclient/events.go

···

       3
       3
        
       import (

     

       4
       4
        
       	"context"

     

       5
       5
        
       	"encoding/json"

     

       6
       6
       +
       	"fmt"

     

       6
       7
        
       	"log/slog"

     

       7
       8
        
       	"math/rand"

     

       8
       9
        
       	"net/url"

     

       9
       10
        
       	"sync"

     

       10
       11
        
       	"time"

     

       11
       12
        
       

     

       13
       13
       +
       	"tangled.sh/tangled.sh/core/appview/cache"

     

       12
       14
        
       	"tangled.sh/tangled.sh/core/log"

     

       13
       15
        
       

     

       14
       16
        
       	"github.com/gorilla/websocket"

     
···

       20
       22
        
       	Rkey string

     

       21
       23
        
       	Nsid string

     

       22
       24
        
       	// do not full deserialize this portion of the message, processFunc can do that

     

       23
       23
       -
       	EventJson json.RawMessage

     

       25
       25
       +
       	EventJson json.RawMessage `json:"event"`

     

       24
       26
        
       }

     

       25
       27
        
       

     

       26
       28
        
       type ConsumerConfig struct {

     
···

       33
       35
        
       	QueueSize         int

     

       34
       36
        
       	Logger            *slog.Logger

     

       35
       37
        
       	Dev               bool

     

       38
       38
       +
       	CursorStore       CursorStore

     

       36
       39
        
       }

     

       37
       40
        
       

     

       38
       41
        
       type EventSource struct {

     
···

       58
       61
        
       	mu sync.RWMutex

     

       59
       62
        
       }

     

       60
       63
        
       

     

       64
       64
       +
       type CursorStore interface {

     

       65
       65
       +
       	Set(knot, cursor string)

     

       66
       66
       +
       	Get(knot string) (cursor string)

     

       67
       67
       +
       }

     

       68
       68
       +
       

     

       69
       69
       +
       type RedisCursorStore struct {

     

       70
       70
       +
       	rdb *cache.Cache

     

       71
       71
       +
       }

     

       72
       72
       +
       

     

       73
       73
       +
       func NewRedisCursorStore(cache *cache.Cache) RedisCursorStore {

     

       74
       74
       +
       	return RedisCursorStore{

     

       75
       75
       +
       		rdb: cache,

     

       76
       76
       +
       	}

     

       77
       77
       +
       }

     

       78
       78
       +
       

     

       79
       79
       +
       const (

     

       80
       80
       +
       	cursorKey = "cursor:%s"

     

       81
       81
       +
       )

     

       82
       82
       +
       

     

       83
       83
       +
       func (r *RedisCursorStore) Set(knot, cursor string) {

     

       84
       84
       +
       	key := fmt.Sprintf(cursorKey, knot)

     

       85
       85
       +
       	r.rdb.Set(context.Background(), key, cursor, 0)

     

       86
       86
       +
       }

     

       87
       87
       +
       

     

       88
       88
       +
       func (r *RedisCursorStore) Get(knot string) (cursor string) {

     

       89
       89
       +
       	key := fmt.Sprintf(cursorKey, knot)

     

       90
       90
       +
       	val, err := r.rdb.Get(context.Background(), key).Result()

     

       91
       91
       +
       	if err != nil {

     

       92
       92
       +
       		return ""

     

       93
       93
       +
       	}

     

       94
       94
       +
       

     

       95
       95
       +
       	return val

     

       96
       96
       +
       }

     

       97
       97
       +
       

     

       98
       98
       +
       type MemoryCursorStore struct {

     

       99
       99
       +
       	store sync.Map

     

       100
       100
       +
       }

     

       101
       101
       +
       

     

       102
       102
       +
       func (m *MemoryCursorStore) Set(knot, cursor string) {

     

       103
       103
       +
       	m.store.Store(knot, cursor)

     

       104
       104
       +
       }

     

       105
       105
       +
       

     

       106
       106
       +
       func (m *MemoryCursorStore) Get(knot string) (cursor string) {

     

       107
       107
       +
       	if result, ok := m.store.Load(knot); ok {

     

       108
       108
       +
       		if val, ok := result.(string); ok {

     

       109
       109
       +
       			return val

     

       110
       110
       +
       		}

     

       111
       111
       +
       	}

     

       112
       112
       +
       

     

       113
       113
       +
       	return ""

     

       114
       114
       +
       }

     

       115
       115
       +
       

     

       61
       116
        
       func (e *EventConsumer) buildUrl(s EventSource, cursor string) (*url.URL, error) {

     

       62
       117
        
       	scheme := "wss"

     

       63
       118
        
       	if e.cfg.Dev {

     
···

       101
       156
        
       	if cfg.QueueSize == 0 {

     

       102
       157
        
       		cfg.QueueSize = 100

     

       103
       158
        
       	}

     

       159
       159
       +
       	if cfg.CursorStore == nil {

     

       160
       160
       +
       		cfg.CursorStore = &MemoryCursorStore{}

     

       161
       161
       +
       	}

     

       104
       162
        
       	return &EventConsumer{

     

       105
       163
        
       		cfg:        cfg,

     

       106
       164
        
       		dialer:     websocket.DefaultDialer,

     
···

       111
       169
        
       }

     

       112
       170
        
       

     

       113
       171
        
       func (c *EventConsumer) Start(ctx context.Context) {

     

       172
       172
       +
       	c.cfg.Logger.Info("starting consumer", "config", c.cfg)

     

       173
       173
       +
       

     

       114
       174
        
       	// start workers

     

       115
       175
        
       	for range c.cfg.WorkerCount {

     

       116
       176
        
       		c.wg.Add(1)

     
···

       160
       220
        
       				c.logger.Error("error deserializing message", "source", j.source.Knot, "err", err)

     

       161
       221
        
       				return

     

       162
       222
        
       			}

     

       223
       223
       +
       

     

       224
       224
       +
       			// update cursor

     

       225
       225
       +
       			c.cfg.CursorStore.Set(j.source.Knot, msg.Rkey)

     

       226
       226
       +
       

     

       163
       227
        
       			if err := c.cfg.ProcessFunc(j.source, msg); err != nil {

     

       164
       228
        
       				c.logger.Error("error processing message", "source", j.source, "err", err)

     

       165
       229
        
       			}

     
···

       204
       268
        
       	connCtx, cancel := context.WithTimeout(ctx, c.cfg.ConnectionTimeout)

     

       205
       269
        
       	defer cancel()

     

       206
       270
        
       

     

       207
       207
       -
       	u, err := url.Parse(source)

     

       271
       271
       +
       	cursor := c.cfg.CursorStore.Get(source.Knot)

     

       208
       272
        
       

     

       209
       273
        
       	u, err := c.buildUrl(source, cursor)

     

       210
       274
        
       	if err != nil {