commit 3d890d58689b69bcdb21a44e13064afcb90bdbb4 · edavis.dev/bsky-tools

+1 -1

feedgen.py

···

       53
       53
        
       

     

       54
       54
        
           feed_manager = FeedManager()

     

       55
       55
        
           feed_manager.register(RapidFireFeed)

     

       56
       56
       -
           feed_manager.register(PopularFeed)

     

       56
       56
       +
           # feed_manager.register(PopularFeed)

     

       57
       57
        
       

     

       58
       58
        
           current_minute = None

     

       59
       59
        
           async for commit in firehose_events(firehose_manager):

+22 -20

feeds/rapidfire.py

···

       1
       1
        
       import os

     

       2
       2
        
       import sys

     

       3
       3
       -
       import sqlite3

     

       3
       3
       +
       import apsw

     

       4
       4
        
       

     

       5
       5
        
       from . import BaseFeed

     

       6
       6
        
       

     
···

       10
       10
        
           FEED_URI = 'at://did:plc:4nsduwlpivpuur4mqkbfvm6a/app.bsky.feed.generator/rapidfire'

     

       11
       11
        
       

     

       12
       12
        
           def __init__(self):

     

       13
       13
       +
               db_fname = ''

     

       13
       14
        
               if os.path.isdir('/dev/shm/'):

     

       14
       15
        
                   os.makedirs('/dev/shm/feedgens/', exist_ok=True)

     

       15
       15
       -
                   self.db_cnx = sqlite3.connect('/dev/shm/feedgens/rapidfire.db')

     

       16
       16
       +
                   db_fname = '/dev/shm/feedgens/rapidfire.db'

     

       16
       17
        
               else:

     

       17
       17
       -
                   self.db_cnx = sqlite3.connect('db/rapidfire.db')

     

       18
       18
       +
                   db_fname = 'db/rapidfire.db'

     

       19
       19
       +
       

     

       20
       20
       +
               self.db_cnx = apsw.Connection(db_fname)

     

       21
       21
       +
               self.db_cnx.pragma('journal_mode', 'WAL')

     

       22
       22
       +
               self.db_cnx.pragma('synchronous', 'OFF')

     

       23
       23
       +
               self.db_cnx.pragma('wal_autocheckpoint', '0')

     

       18
       24
        
       

     

       19
       25
        
               with self.db_cnx:

     

       20
       20
       -
                   self.db_cnx.executescript(

     

       21
       21
       -
                       "pragma journal_mode = WAL;"

     

       22
       22
       -
                       "pragma synchronous = OFF;"

     

       23
       23
       -
                       "pragma wal_autocheckpoint = 0;"

     

       24
       24
       -
                       "create table if not exists posts (uri text, create_ts timestamp, lang text);"

     

       25
       25
       -
                       "create index if not exists create_ts_idx on posts(create_ts);"

     

       26
       26
       -
                   )

     

       27
       27
       -
       

     

       28
       28
       -
               self.checkpoint = 0

     

       26
       26
       +
                   self.db_cnx.execute("""

     

       27
       27
       +
                   create table if not exists posts (uri text, create_ts timestamp, lang text);

     

       28
       28
       +
                   create index if not exists create_ts_idx on posts(create_ts);

     

       29
       29
       +
                   """)

     

       29
       30
        
       

     

       30
       31
        
           def process_commit(self, commit):

     

       31
       32
        
               op = commit['op']

     
···

       48
       49
        
                   path = op['path']

     

       49
       50
        
                   post_uri = f'at://{repo}/{path}'

     

       50
       51
        
                   ts = record['createdAt']

     

       51
       51
       -
                   self.checkpoint += 1

     

       52
       52
        
       

     

       53
       53
        
                   with self.db_cnx:

     

       54
       54
        
                       langs = record.get('langs') or ['']

     
···

       58
       58
        
                               dict(uri=post_uri, ts=ts, lang=lang)

     

       59
       59
        
                           )

     

       60
       60
        
       

     

       61
       61
       -
                       self.db_cnx.execute(

     

       62
       62
       -
                           "delete from posts where strftime('%s', create_ts) < strftime('%s', 'now', '-15 minutes')"

     

       63
       63
       -
                       )

     

       61
       61
       +
           def run_tasks_minute(self):

     

       62
       62
       +
               sys.stdout.write('rapidfire: running minute tasks\n')

     

       63
       63
       +
               sys.stdout.flush()

     

       64
       64
       +
       

     

       65
       65
       +
               with self.db_cnx:

     

       66
       66
       +
                   self.db_cnx.execute(

     

       67
       67
       +
                       "delete from posts where strftime('%s', create_ts) < strftime('%s', 'now', '-15 minutes')"

     

       68
       68
       +
                   )

     

       64
       69
        
       

     

       65
       65
       -
                   if self.checkpoint % 100 == 0:

     

       66
       66
       -
                       sys.stdout.write('rapidfire: checkpoint\n')

     

       67
       67
       -
                       sys.stdout.flush()

     

       68
       68
       -
                       self.db_cnx.execute("pragma wal_checkpoint(TRUNCATE)")

     

       70
       70
       +
               self.db_cnx.pragma('wal_checkpoint(TRUNCATE)')

     

       69
       71
        
       

     

       70
       72
        
           def serve_feed(self, limit, offset, langs):

     

       71
       73
        
               if '*' in langs:

+1 -1

feedweb.py

···

       11
       11
        
       def get_feed_skeleton():

     

       12
       12
        
           manager = FeedManager()

     

       13
       13
        
           manager.register(RapidFireFeed)

     

       14
       14
       -
           manager.register(PopularFeed)

     

       14
       14
       +
           # manager.register(PopularFeed)

     

       15
       15
        
       

     

       16
       16
        
           try:

     

       17
       17
        
               limit = int(request.args.get('limit', 50))