commit 77446847d200c4790b5750840af95663886f9213 · edavis.dev/monarch

+19 -2
cmd/monarch/census.go
···

       17
       17
        
       	cursor   *CursorService

     

       18
       18
        
       	backfill *backfill.Backfiller

     

       19
       19
        
       

     

       20
       20
       +
       	seenHosts map[string]bool

     

       21
       21
       +
       	seenLk    sync.Mutex

     

       22
       22
       +
       

     

       20
       23
        
       	storeLk sync.Mutex

     

       21
       24
        
       }

     

       22
       25
        
       

     
···

       26
       29
        
       

     

       27
       30
        
       func NewCensusService(cursorSvc *CursorService, backfillSvc *backfill.Backfiller) *CensusService {

     

       28
       31
        
       	return &CensusService{

     

       29
       29
       -
       		cursor:   cursorSvc,

     

       30
       30
       -
       		backfill: backfillSvc,

     

       32
       32
       +
       		cursor:    cursorSvc,

     

       33
       33
       +
       		backfill:  backfillSvc,

     

       34
       34
       +
       		seenHosts: make(map[string]bool),

     

       31
       35
        
       	}

     

       32
       36
        
       }

     

       33
       37
        
       

     
···

       65
       69
        
       		}

     

       66
       70
        
       

     

       67
       71
        
       		for _, host := range res.Hosts {

     

       72
       72
       +
       			// don't reprocess hosts already handled

     

       73
       73
       +
       			cs.seenLk.Lock()

     

       74
       74
       +
       			_, ok := cs.seenHosts[host.Hostname]

     

       75
       75
       +
       			cs.seenLk.Unlock()

     

       76
       76
       +
       			if ok {

     

       77
       77
       +
       				slog.Info("already processed host, skipping")

     

       78
       78
       +
       				continue

     

       79
       79
       +
       			}

     

       80
       80
       +
       

     

       68
       81
        
       			sem.Acquire(ctx, 1)

     

       69
       82
        
       			wg.Add(1) // TODO wg.Go

     

       70
       83
        
       			go func() {

     
···

       147
       160
        
       			break

     

       148
       161
        
       		}

     

       149
       162
        
       	}

     

       163
       163
       +
       

     

       164
       164
       +
       	cs.seenLk.Lock()

     

       165
       165
       +
       	defer cs.seenLk.Unlock()

     

       166
       166
       +
       	cs.seenHosts[host] = true

     

       150
       167
        
       

     

       151
       168
        
       	slog.Info("finished listing repos", "host", host)

     

       152
       169
        
       }