commit 0e600c94be7e0ef18fcb0068f26f4e8aca943e17 · edavis.dev/bsky-tools

+24

feed_manager.py

···

       1
       1
       +
       class FeedManager:

     

       2
       2
       +
           def __init__(self):

     

       3
       3
       +
               self.feeds = {}

     

       4
       4
       +
       

     

       5
       5
       +
           def register(self, feed):

     

       6
       6
       +
               self.feeds[feed.FEED_URI] = feed()

     

       7
       7
       +
       

     

       8
       8
       +
           def process_commit(self, commit):

     

       9
       9
       +
               for feed in self.feeds.values():

     

       10
       10
       +
                   feed.process_commit(commit)

     

       11
       11
       +
       

     

       12
       12
       +
           def serve_feed(self, feed_uri, limit, offset, langs):

     

       13
       13
       +
               feed = self.feeds.get(feed_uri)

     

       14
       14
       +
               if feed is not None:

     

       15
       15
       +
                   return feed.serve_feed(limit, offset, langs)

     

       16
       16
       +
       

     

       17
       17
       +
           def serve_feed_debug(self, feed_uri, limit, offset, langs):

     

       18
       18
       +
               feed = self.feeds.get(feed_uri)

     

       19
       19
       +
               if feed is not None:

     

       20
       20
       +
                   return feed.serve_feed_debug(limit, offset, langs)

     

       21
       21
       +
       

     

       22
       22
       +
           def run_tasks_minute(self):

     

       23
       23
       +
               for feed in self.feeds.values():

     

       24
       24
       +
                   feed.run_tasks_minute()

+9 -9

feedgen.py

···

       1
       1
        
       #!/usr/bin/env python3

     

       2
       2
        
       

     

       3
       3
        
       import asyncio

     

       4
       4
       -
       import dag_cbor

     

       4
       4
       +
       from datetime import datetime, timezone

     

       5
       5
       +
       from io import BytesIO

     

       5
       6
        
       import logging

     

       6
       6
       -
       import sys

     

       7
       7
       -
       import websockets

     

       8
       7
        
       

     

       9
       8
        
       from atproto import CAR

     

       10
       10
       -
       from io import BytesIO

     

       11
       11
       -
       from datetime import datetime, timezone

     

       9
       9
       +
       import dag_cbor

     

       10
       10
       +
       import websockets

     

       12
       11
        
       

     

       13
       13
       -
       from feeds import FeedManager

     

       12
       12
       +
       from feed_manager import FeedManager

     

       14
       13
        
       from feeds.rapidfire import RapidFireFeed

     

       15
       14
        
       from feeds.popular import PopularFeed

     

       16
       15
        
       from firehose_manager import FirehoseManager

     
···

       28
       27
        
           if seq:

     

       29
       28
        
               relay_url += f'?cursor={seq}'

     

       30
       29
        
       

     

       31
       31
       -
           sys.stdout.write(f'opening websocket connection to {relay_url}\n')

     

       32
       32
       -
           sys.stdout.flush()

     

       30
       30
       +
           logger = logging.getLogger('feeds.events')

     

       31
       31
       +
           logger.info(f'opening websocket connection to {relay_url}')

     

       33
       32
        
       

     

       34
       33
        
           async with websockets.connect(relay_url, ping_timeout=None) as firehose:

     

       35
       34
        
               while True:

     
···

       52
       51
        
                       repo_op = op.copy()

     

       53
       52
        
                       if op['cid'] is not None:

     

       54
       53
        
                           repo_op['cid'] = repo_op['cid'].encode('base32')

     

       55
       55
       -
                           repo_op['record'] = car_parsed.blocks[repo_op['cid']]

     

       54
       54
       +
                           repo_op['record'] = car_parsed.blocks.get(repo_op['cid'])

     

       55
       55
       +
       

     

       56
       56
        
                       message['op'] = repo_op

     

       57
       57
        
                       yield message

     

       58
       58

-25

feeds/__init__.py

···

       46
       46
        
                   return parsed

     

       47
       47
        
               elif parsed > utc_now:

     

       48
       48
        
                   return utc_now

     

       49
       49
       -
       

     

       50
       50
       -
       class FeedManager:

     

       51
       51
       -
           def __init__(self):

     

       52
       52
       -
               self.feeds = {}

     

       53
       53
       -
       

     

       54
       54
       -
           def register(self, feed):

     

       55
       55
       -
               self.feeds[feed.FEED_URI] = feed()

     

       56
       56
       -
       

     

       57
       57
       -
           def process_commit(self, commit):

     

       58
       58
       -
               for feed in self.feeds.values():

     

       59
       59
       -
                   feed.process_commit(commit)

     

       60
       60
       -
       

     

       61
       61
       -
           def serve_feed(self, feed_uri, limit, offset, langs):

     

       62
       62
       -
               feed = self.feeds.get(feed_uri)

     

       63
       63
       -
               if feed is not None:

     

       64
       64
       -
                   return feed.serve_feed(limit, offset, langs)

     

       65
       65
       -
       

     

       66
       66
       -
           def serve_feed_debug(self, feed_uri, limit, offset, langs):

     

       67
       67
       -
               feed = self.feeds.get(feed_uri)

     

       68
       68
       -
               if feed is not None:

     

       69
       69
       -
                   return feed.serve_feed_debug(limit, offset, langs)

     

       70
       70
       -
       

     

       71
       71
       -
           def run_tasks_minute(self):

     

       72
       72
       -
               for feed in self.feeds.values():

     

       73
       73
       -
                   feed.run_tasks_minute()

+2 -1

feeds/popular.py

···

       1
       1
       +
       import logging

     

       1
       2
        
       import os

     

       2
       2
       -
       import logging

     

       3
       3
       +
       

     

       3
       4
        
       import apsw

     

       4
       5
        
       

     

       5
       6
        
       from . import BaseFeed

+2 -1

feeds/rapidfire.py

···

       1
       1
       +
       import logging

     

       1
       2
        
       import os

     

       3
       3
       +
       

     

       2
       4
        
       import apsw

     

       3
       5
        
       import apsw.ext

     

       4
       4
       -
       import logging

     

       5
       6
        
       

     

       6
       7
        
       from . import BaseFeed

     

       7
       8

+4 -2

feedweb.py

···

       1
       1
        
       #!/usr/bin/env python3

     

       2
       2
        
       

     

       3
       3
       -
       from feeds import FeedManager

     

       3
       3
       +
       from flask import Flask, request, jsonify

     

       4
       4
       +
       

     

       5
       5
       +
       from feed_manager import FeedManager

     

       4
       6
        
       from feeds.rapidfire import RapidFireFeed

     

       5
       7
        
       from feeds.popular import PopularFeed

     

       6
       6
       -
       from flask import Flask, request, jsonify

     

       7
       8
        
       

     

       8
       9
        
       app = Flask(__name__)

     

       9
       10
        
       

     
···

       43
       44
        
       if __name__ == '__main__':

     

       44
       45
        
           from feedweb_utils import did_doc

     

       45
       46
        
           app.add_url_rule('/.well-known/did.json', view_func=did_doc)

     

       47
       47
       +
       

     

       46
       48
        
           app.run(debug=True)

+2 -1

firehose_manager.py

···

       1
       1
       +
       import logging

     

       2
       2
       +
       

     

       1
       3
        
       import apsw

     

       2
       2
       -
       import logging

     

       3
       4
        
       

     

       4
       5
        
       class FirehoseManager:

     

       5
       6
        
           def __init__(self, fname='firehose.db'):