commit aad37bbebfa60e27e450f5bb1779af1835fa070a · edavis.dev/bsky-tools

+12 -15
bsky-activity.py
···

       2
       2
        
       

     

       3
       3
        
       import asyncio

     

       4
       4
        
       from datetime import datetime, timezone

     

       5
       5
       -
       from io import BytesIO

     

       6
       5
        
       import json

     

       7
       6
        
       import os

     

       8
       7
        
       import sqlite3

     

       9
       8
        
       import sys

     

       10
       9
        
       

     

       11
       11
       -
       from atproto import CAR

     

       12
       10
        
       import redis

     

       13
       13
       -
       import dag_cbor

     

       14
       11
        
       import websockets

     

       15
       12
        
       

     

       16
       13
        
       app_bsky_allowlist = set([

     
···

       39
       36
        
       

     

       40
       37
        
           async with websockets.connect(relay_url, ping_timeout=60) as firehose:

     

       41
       38
        
               while True:

     

       42
       42
       -
                   payload = BytesIO(await firehose.recv())

     

       43
       43
       -
       

     

       44
       44
       -
                   yield json.load(payload)

     

       39
       39
       +
                   yield json.loads(await firehose.recv())

     

       45
       40
        
       

     

       46
       41
        
       async def main():

     

       47
       42
        
           redis_cnx = redis.Redis()

     
···

       66
       61
        
           sys.stdout.flush()

     

       67
       62
        
       

     

       68
       63
        
           op_count = 0

     

       69
       69
       -
           async for payload in bsky_activity():

     

       70
       70
       -
               if payload['opType'] != 'c':

     

       64
       64
       +
           async for event in bsky_activity():

     

       65
       65
       +
               if event['type'] != 'com':

     

       66
       66
       +
                   continue

     

       67
       67
       +
       

     

       68
       68
       +
               payload = event['commit']

     

       69
       69
       +
               if payload['type'] != 'c':

     

       71
       70
        
                   continue

     

       72
       71
        
       

     

       73
       72
        
               collection = payload['collection']

     

       74
       73
        
               if collection not in app_bsky_allowlist:

     

       75
       74
        
                   continue

     

       76
       75
        
       

     

       77
       77
       -
               repo_did = payload['did']

     

       76
       76
       +
               repo_did = event['did']

     

       78
       77
        
               repo_update_time = datetime.now(timezone.utc)

     

       79
       78
        
               db_cnx.execute(

     

       80
       79
        
                   'insert into users values (:did, :ts) on conflict (did) do update set ts = :ts',

     
···

       93
       92
        
       

     

       94
       93
        
               op_count += 1

     

       95
       94
        
               if op_count % 500 == 0:

     

       96
       96
       -
                   now = datetime.now(timezone.utc)

     

       97
       97
       -
                   payload_seq = payload['seq']

     

       98
       98
       -
                   payload_lag = now - repo_update_time

     

       99
       99
       -
       

     

       100
       100
       -
                   sys.stdout.write(f'seq: {payload_seq}, lag: {payload_lag.total_seconds()}\n')

     

       101
       101
       -
                   redis_pipe.set('dev.edavis.muninsky.seq', payload_seq)

     

       95
       95
       +
                   current_time_ms = datetime.now(timezone.utc).timestamp()

     

       96
       96
       +
                   event_time_ms = event['time_us'] / 1_000_000

     

       97
       97
       +
                   current_lag = current_time_ms - event_time_ms

     

       98
       98
       +
                   sys.stdout.write(f'lag: {current_lag:.2f}\n')

     

       102
       99
        
                   redis_pipe.execute()

     

       103
       100
        
                   db_cnx.commit()

     

       104
       101
        
                   sys.stdout.flush()