···
-
from expiringdict import ExpiringDict
-
# store post in database once it has this many likes
-
class DatabaseWorker(threading.Thread):
-
def __init__(self, name, db_path, task_queue):
-
self.db_cnx = apsw.Connection(db_path)
-
self.db_cnx.pragma('foreign_keys', True)
-
self.db_cnx.pragma('journal_mode', 'WAL')
-
self.db_cnx.pragma('wal_autocheckpoint', '0')
-
self.stop_signal = False
-
self.task_queue = task_queue
-
self.logger = logging.getLogger(f'feeds.db.{name}')
-
task = self.task_queue.get(block=True)
-
self.logger.debug('received STOP, breaking now')
-
self.logger.debug(f'committing {self.changes} changes')
-
if self.db_cnx.in_transaction:
-
self.db_cnx.execute('COMMIT')
-
checkpoint = self.db_cnx.execute('PRAGMA wal_checkpoint(PASSIVE)')
-
self.logger.debug(f'checkpoint: {checkpoint.fetchall()!r}')
-
self.logger.debug(f'qsize: {self.task_queue.qsize()}')
-
if not self.db_cnx.in_transaction:
-
self.db_cnx.execute('BEGIN')
-
self.db_cnx.execute(sql, bindings)
-
self.changes += self.db_cnx.changes()
-
self.task_queue.task_done()
-
self.logger.debug('closing database connection')
-
self.task_queue.put('STOP')
class MostLikedFeed(BaseFeed):
FEED_URI = 'at://did:plc:4nsduwlpivpuur4mqkbfvm6a/app.bsky.feed.generator/most-liked'
-
DELETE_OLD_POSTS_QUERY = """
-
delete from posts where create_ts < unixepoch('now', '-24 hours');
self.db_cnx = apsw.Connection('db/mostliked.db')
self.db_cnx.pragma('foreign_keys', True)
self.db_cnx.pragma('journal_mode', 'WAL')
-
self.db_cnx.pragma('wal_autocheckpoint', '0')
-
self.db_cnx.execute("""
-
create table if not exists posts (
-
create table if not exists langs (
-
foreign key(uri) references posts(uri) on delete cascade
-
create index if not exists ts_idx on posts(create_ts);
-
self.logger = logging.getLogger('feeds.mostliked')
-
self.drafts = ExpiringDict(max_len=50_000, max_age_seconds=5*60)
-
self.db_writes = queue.Queue()
-
self.db_worker = DatabaseWorker('mostliked', 'db/mostliked.db', self.db_writes)
-
def stop_db_worker(self):
-
self.logger.debug('sending STOP')
-
self.db_writes.put('STOP')
-
def process_commit(self, commit):
-
if commit['opType'] != 'c':
-
if commit['collection'] == 'app.bsky.feed.post':
-
record = commit.get('record')
-
post_uri = f"at://{commit['did']}/app.bsky.feed.post/{commit['rkey']}"
-
# to keep the DB in check, instead of adding every post right away
-
# we make note of it but only add to DB once it gets some likes
-
self.drafts[post_uri] = {
-
'ts': self.safe_timestamp(record.get('createdAt')).timestamp(),
-
'langs': record.get('langs', []),
-
elif commit['collection'] == 'app.bsky.feed.like':
-
record = commit.get('record')
-
subject_uri = record['subject']['uri']
-
if subject_uri in self.drafts:
-
record_info = self.drafts.pop(subject_uri).copy()
-
record_info['likes'] += 1
-
if record_info['likes'] < MIN_LIKES:
-
self.drafts[subject_uri] = record_info
-
self.logger.debug(f'graduating {subject_uri}')
-
'insert or ignore into posts (uri, create_ts, likes) values (:uri, :ts, :likes)',
-
{'uri': subject_uri, 'ts': record_info['ts'], 'likes': record_info['likes']}
-
self.db_writes.put(task)
-
for lang in record_info['langs']:
-
'insert or ignore into langs (uri, lang) values (:uri, :lang)',
-
{'uri': subject_uri, 'lang': lang}
-
self.db_writes.put(task)
-
subject_exists = self.db_cnx.execute('select 1 from posts where uri = ?', [subject_uri])
-
if subject_exists.fetchone() is None:
-
'update posts set likes = likes + 1 where uri = :uri',
-
self.db_writes.put(task)
-
def commit_changes(self):
-
self.db_writes.put((self.DELETE_OLD_POSTS_QUERY, {}))
-
self.db_writes.put('COMMIT')
-
self.logger.debug(f'there are {len(self.drafts)} drafts')
def generate_sql(self, limit, offset, langs):
edavis.dev