commit 0bfdb8f9ef629fd06f278c9b773d0fa6dfc6abb2 · edavis.dev/bsky-feeds

+1 -1

Makefile

···

       3
       3
        
       bin/mostliked: cmd/mostliked/main.go pkg/mostliked/handler.go db/mostliked/*.go pkg/feeds/*.go

     

       4
       4
        
       	go build -o $@ ./cmd/mostliked

     

       5
       5
        
       

     

       6
       6
       -
       bin/feedweb: cmd/feedweb/main.go pkg/*/view.go db/*/*.go pkg/feeds/*.go

     

       6
       6
       +
       bin/feedweb: cmd/feedweb/main.go pkg/*/generator.go db/*/*.go pkg/feeds/*.go

     

       7
       7
        
       	go build -o $@ ./cmd/feedweb

     

       8
       8
        
       

     

       9
       9
        
       .PHONY: clean

+8 -9

cmd/feedweb/main.go

···

       1
       1
        
       package main

     

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
       +
       	"golang.org/x/text/language"

     

       4
       5
        
       	"net/http"

     

       5
       6
        
       

     

       6
       7
        
       	appbsky "github.com/bluesky-social/indigo/api/bsky"

     
···

       16
       17
        
       	Offset string `query:"offset"`

     

       17
       18
        
       }

     

       18
       19
        
       

     

       19
       19
       -
       type SkeletonHeader struct {

     

       20
       20
       -
       	Langs []string `header:"Accept-Language"`

     

       20
       20
       +
       type FeedLookup map[string]func(feeds.FeedgenParams) appbsky.FeedGetFeedSkeleton_Output

     

       21
       21
       +
       

     

       22
       22
       +
       func parseLangs(userPrefs string) []language.Tag {

     

       23
       23
       +
       	t, _, _ := language.ParseAcceptLanguage(userPrefs)

     

       24
       24
       +
       	return t

     

       21
       25
        
       }

     

       22
       26
        
       

     

       23
       23
       -
       type FeedLookup map[string]func(feeds.FeedgenParams) appbsky.FeedGetFeedSkeleton_Output

     

       24
       24
       -
       

     

       25
       27
        
       func getFeedSkeleton(c echo.Context) error {

     

       26
       28
        
       	var req SkeletonRequest

     

       27
       29
        
       	if err := c.Bind(&req); err != nil {

     

       28
       30
        
       		return c.String(http.StatusBadRequest, "bad request")

     

       29
       31
        
       	}

     

       30
       30
       -
       	var hdr SkeletonHeader

     

       31
       31
       -
       	if err := (&echo.DefaultBinder{}).BindHeaders(c, &hdr); err != nil {

     

       32
       32
       -
       		return c.String(http.StatusBadRequest, "bad request")

     

       33
       33
       -
       	}

     

       34
       32
        
       

     

       35
       33
        
       	generators := FeedLookup{

     

       36
       34
        
       		"at://did:plc:4nsduwlpivpuur4mqkbfvm6a/app.bsky.feed.generator/most-liked": mostliked.Feed,

     

       37
       35
        
       	}

     

       36
       36
       +
       

     

       38
       37
        
       	params := feeds.FeedgenParams{

     

       39
       38
        
       		Feed:   req.Feed,

     

       40
       39
        
       		Limit:  req.Limit,

     

       41
       40
        
       		Offset: req.Offset,

     

       42
       42
       -
       		Langs:  hdr.Langs,

     

       41
       41
       +
       		Langs:  parseLangs(c.Request().Header.Get("Accept-Language")),

     

       43
       42
        
       	}

     

       44
       43
        
       	feedFunc, ok := generators[req.Feed]

     

       45
       44
        
       	if !ok {

+3 -1

go.mod

···

       10
       10
        
       	github.com/labstack/echo/v4 v4.11.3

     

       11
       11
        
       	github.com/mattn/go-sqlite3 v1.14.22

     

       12
       12
        
       	github.com/pemistahl/lingua-go v1.4.0

     

       13
       13
       +
       	github.com/sanggonlee/gosq v1.2.0

     

       14
       14
       +
       	golang.org/x/text v0.16.0

     

       13
       15
        
       )

     

       14
       16
        
       

     

       15
       17
        
       require (

     
···

       49
       51
        
       	github.com/multiformats/go-multihash v0.2.3 // indirect

     

       50
       52
        
       	github.com/multiformats/go-varint v0.0.7 // indirect

     

       51
       53
        
       	github.com/opentracing/opentracing-go v1.2.0 // indirect

     

       54
       54
       +
       	github.com/pkg/errors v0.9.1 // indirect

     

       52
       55
        
       	github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect

     

       53
       56
        
       	github.com/shopspring/decimal v1.3.1 // indirect

     

       54
       57
        
       	github.com/spaolacci/murmur3 v1.1.0 // indirect

     
···

       66
       69
        
       	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect

     

       67
       70
        
       	golang.org/x/net v0.24.0 // indirect

     

       68
       71
        
       	golang.org/x/sys v0.22.0 // indirect

     

       69
       69
       -
       	golang.org/x/text v0.16.0 // indirect

     

       70
       72
        
       	golang.org/x/time v0.5.0 // indirect

     

       71
       73
        
       	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

     

       72
       74
        
       	google.golang.org/protobuf v1.34.2 // indirect

+11

go.sum

···

       17
       17
        
       github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

     

       18
       18
        
       github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=

     

       19
       19
        
       github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=

     

       20
       20
       +
       github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=

     

       21
       21
       +
       github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=

     

       20
       22
        
       github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=

     

       21
       23
        
       github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=

     

       22
       24
        
       github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

     
···

       117
       119
        
       github.com/pemistahl/lingua-go v1.4.0 h1:ifYhthrlW7iO4icdubwlduYnmwU37V1sbNrwhKBR4rM=

     

       118
       120
        
       github.com/pemistahl/lingua-go v1.4.0/go.mod h1:ECuM1Hp/3hvyh7k8aWSqNCPlTxLemFZsRjocUf3KgME=

     

       119
       121
        
       github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

     

       122
       122
       +
       github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

     

       123
       123
       +
       github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

     

       120
       124
        
       github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

     

       121
       125
        
       github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

     

       122
       126
        
       github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4TCkEuWIckKvdHFeJH/huIFJ9/cXOB0=

     
···

       125
       129
        
       github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=

     

       126
       130
        
       github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=

     

       127
       131
        
       github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

     

       132
       132
       +
       github.com/sanggonlee/gosq v1.2.0 h1:A0648wgV1ISRODDjy3/P5MV4OKGDOI16ol6xMyi0wj4=

     

       133
       133
       +
       github.com/sanggonlee/gosq v1.2.0/go.mod h1:sUXP8djVeH+9XxjsJt93mJJymuRLKSz1KJfS1ByXBe0=

     

       128
       134
        
       github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=

     

       129
       135
        
       github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

     

       130
       136
        
       github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=

     
···

       135
       141
        
       github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=

     

       136
       142
        
       github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=

     

       137
       143
        
       github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

     

       144
       144
       +
       github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=

     

       145
       145
       +
       github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

     

       138
       146
        
       github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

     

       139
       147
        
       github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

     

       140
       148
        
       github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

     

       141
       149
        
       github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

     

       150
       150
       +
       github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

     

       151
       151
       +
       github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=

     

       152
       152
       +
       github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=

     

       142
       153
        
       github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=

     

       143
       154
        
       github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=

     

       144
       155
        
       github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

+5 -1

pkg/feeds/feeds.go

···

       1
       1
        
       package feeds

     

       2
       2
        
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"golang.org/x/text/language"

     

       5
       5
       +
       )

     

       6
       6
       +
       

     

       3
       7
        
       type FeedgenParams struct {

     

       4
       8
        
       	Feed   string

     

       5
       9
        
       	Limit  int64

     

       6
       10
        
       	Offset string

     

       7
       7
       -
       	Langs  []string

     

       11
       11
       +
       	Langs  []language.Tag

     

       8
       12
        
       }

+111

pkg/mostliked/generator.go

···

       1
       1
       +
       package mostliked

     

       2
       2
       +
       

     

       3
       3
       +
       import (

     

       4
       4
       +
       	"context"

     

       5
       5
       +
       	"database/sql"

     

       6
       6
       +
       	"log"

     

       7
       7
       +
       	"strconv"

     

       8
       8
       +
       

     

       9
       9
       +
       	appbsky "github.com/bluesky-social/indigo/api/bsky"

     

       10
       10
       +
       	"github.com/edavis/bsky-feeds/pkg/feeds"

     

       11
       11
       +
       	_ "github.com/mattn/go-sqlite3"

     

       12
       12
       +
       	"github.com/sanggonlee/gosq"

     

       13
       13
       +
       )

     

       14
       14
       +
       

     

       15
       15
       +
       type PostRow struct {

     

       16
       16
       +
       	Uri   string

     

       17
       17
       +
       	Likes int

     

       18
       18
       +
       }

     

       19
       19
       +
       

     

       20
       20
       +
       func getPosts(ctx context.Context, dbCnx *sql.DB, params feeds.FeedgenParams) ([]PostRow, error) {

     

       21
       21
       +
       	q, err := gosq.Execute(`

     

       22
       22
       +
       	SELECT posts.uri, likes

     

       23
       23
       +
       	FROM posts LEFT JOIN langs ON posts.uri = langs.uri

     

       24
       24
       +
       	WHERE 1=1

     

       25
       25
       +
       	{{if .Langs}}

     

       26
       26
       +
       		AND (

     

       27
       27
       +
       			{{- range $index, $lang := .Langs }}

     

       28
       28
       +
       			{{- if $index }} OR {{ end }} lang = ?

     

       29
       29
       +
       			{{- end }}

     

       30
       30
       +
       		)

     

       31
       31
       +
       	{{end}}

     

       32
       32
       +
       	{{if .Offset}}

     

       33
       33
       +
       	AND likes < ?

     

       34
       34
       +
       	{{end}}

     

       35
       35
       +
       	ORDER BY likes DESC, create_ts DESC

     

       36
       36
       +
       	LIMIT ?

     

       37
       37
       +
       	`, params)

     

       38
       38
       +
       	if err != nil {

     

       39
       39
       +
       		log.Println("error in sql template:", err)

     

       40
       40
       +
       	}

     

       41
       41
       +
       

     

       42
       42
       +
       	log.Println(q)

     

       43
       43
       +
       

     

       44
       44
       +
       	var queryParams []any

     

       45
       45
       +
       	for _, lang := range params.Langs {

     

       46
       46
       +
       		queryParams = append(queryParams, lang.String())

     

       47
       47
       +
       	}

     

       48
       48
       +
       

     

       49
       49
       +
       	if params.Offset != "" {

     

       50
       50
       +
       		queryParams = append(queryParams, params.Offset)

     

       51
       51
       +
       	}

     

       52
       52
       +
       

     

       53
       53
       +
       	queryParams = append(queryParams, params.Limit)

     

       54
       54
       +
       

     

       55
       55
       +
       	log.Println(queryParams)

     

       56
       56
       +
       

     

       57
       57
       +
       	rows, err := dbCnx.QueryContext(ctx, q, queryParams...)

     

       58
       58
       +
       	if err != nil {

     

       59
       59
       +
       		return nil, err

     

       60
       60
       +
       	}

     

       61
       61
       +
       	defer rows.Close()

     

       62
       62
       +
       

     

       63
       63
       +
       	var posts []PostRow

     

       64
       64
       +
       	for rows.Next() {

     

       65
       65
       +
       		var post PostRow

     

       66
       66
       +
       		if err := rows.Scan(&post.Uri, &post.Likes); err != nil {

     

       67
       67
       +
       			return nil, err

     

       68
       68
       +
       		}

     

       69
       69
       +
       		posts = append(posts, post)

     

       70
       70
       +
       	}

     

       71
       71
       +
       	if err := rows.Close(); err != nil {

     

       72
       72
       +
       		return nil, err

     

       73
       73
       +
       	}

     

       74
       74
       +
       	if err := rows.Err(); err != nil {

     

       75
       75
       +
       		return nil, err

     

       76
       76
       +
       	}

     

       77
       77
       +
       

     

       78
       78
       +
       	return posts, nil

     

       79
       79
       +
       }

     

       80
       80
       +
       

     

       81
       81
       +
       func Feed(params feeds.FeedgenParams) appbsky.FeedGetFeedSkeleton_Output {

     

       82
       82
       +
       	ctx := context.Background()

     

       83
       83
       +
       	dbCnx, err := sql.Open("sqlite3", "data/mostliked.db?_journal=WAL&_fk=on&mode=ro")

     

       84
       84
       +
       	if err != nil {

     

       85
       85
       +
       		log.Fatal("error opening db")

     

       86
       86
       +
       	}

     

       87
       87
       +
       	defer dbCnx.Close()

     

       88
       88
       +
       

     

       89
       89
       +
       	rows, err := getPosts(ctx, dbCnx, params)

     

       90
       90
       +
       	if err != nil {

     

       91
       91
       +
       		log.Println("error fetching rows:", err)

     

       92
       92
       +
       	}

     

       93
       93
       +
       

     

       94
       94
       +
       	var cursor string

     

       95
       95
       +
       	posts := make([]*appbsky.FeedDefs_SkeletonFeedPost, 0, params.Limit)

     

       96
       96
       +
       

     

       97
       97
       +
       	for _, row := range rows {

     

       98
       98
       +
       		posts = append(posts, &appbsky.FeedDefs_SkeletonFeedPost{Post: row.Uri})

     

       99
       99
       +
       		cursor = strconv.Itoa(row.Likes)

     

       100
       100
       +
       	}

     

       101
       101
       +
       

     

       102
       102
       +
       	skeleton := appbsky.FeedGetFeedSkeleton_Output{

     

       103
       103
       +
       		Feed: posts,

     

       104
       104
       +
       	}

     

       105
       105
       +
       

     

       106
       106
       +
       	if len(rows) == int(params.Limit) {

     

       107
       107
       +
       		skeleton.Cursor = &cursor

     

       108
       108
       +
       	}

     

       109
       109
       +
       

     

       110
       110
       +
       	return skeleton

     

       111
       111
       +
       }

pkg/mostliked/handler.go

···

       48
       48
        
       func findDetectableText(post appbsky.FeedPost) string {

     

       49
       49
        
       	// if we have text, detect against that

     

       50
       50
        
       	// no text but we do have images, detect against first alt text

     

       51
       51
       +
       	// log.Printf("%+v\n", post)

     

       51
       52
        
       

     

       52
       53
        
       	if post.Text != "" {

     

       53
       54
        
       		return post.Text

-46

pkg/mostliked/view.go

···

       1
       1
       -
       package mostliked

     

       2
       2
       -
       

     

       3
       3
       -
       import (

     

       4
       4
       -
       	"context"

     

       5
       5
       -
       	"database/sql"

     

       6
       6
       -
       	"log"

     

       7
       7
       -
       	"strconv"

     

       8
       8
       -
       

     

       9
       9
       -
       	appbsky "github.com/bluesky-social/indigo/api/bsky"

     

       10
       10
       -
       	db "github.com/edavis/bsky-feeds/db/mostliked"

     

       11
       11
       -
       	"github.com/edavis/bsky-feeds/pkg/feeds"

     

       12
       12
       -
       	_ "github.com/mattn/go-sqlite3"

     

       13
       13
       -
       )

     

       14
       14
       -
       

     

       15
       15
       -
       func Feed(params feeds.FeedgenParams) appbsky.FeedGetFeedSkeleton_Output {

     

       16
       16
       -
       	ctx := context.Background()

     

       17
       17
       -
       	dbCnx, err := sql.Open("sqlite3", "data/mostliked.db?_journal=WAL&_fk=on&mode=ro")

     

       18
       18
       -
       	if err != nil {

     

       19
       19
       -
       		log.Fatal("error opening db")

     

       20
       20
       -
       	}

     

       21
       21
       -
       	defer dbCnx.Close()

     

       22
       22
       -
       

     

       23
       23
       -
       	offset, err := strconv.Atoi(params.Offset)

     

       24
       24
       -
       	if err != nil {

     

       25
       25
       -
       		log.Println("error converting offset to integer")

     

       26
       26
       -
       	}

     

       27
       27
       -
       

     

       28
       28
       -
       	queries := db.New(dbCnx)

     

       29
       29
       -
       	rows, err := queries.ViewFeed(ctx, db.ViewFeedParams{

     

       30
       30
       -
       		Limit:  params.Limit,

     

       31
       31
       -
       		Offset: int64(offset),

     

       32
       32
       -
       	})

     

       33
       33
       -
       	if err != nil {

     

       34
       34
       -
       		log.Println("error fetching rows")

     

       35
       35
       -
       	}

     

       36
       36
       -
       	var cursor string

     

       37
       37
       -
       	posts := make([]*appbsky.FeedDefs_SkeletonFeedPost, 0, params.Limit)

     

       38
       38
       -
       	for _, row := range rows {

     

       39
       39
       -
       		posts = append(posts, &appbsky.FeedDefs_SkeletonFeedPost{Post: row.Uri})

     

       40
       40
       -
       		cursor = row.Uri

     

       41
       41
       -
       	}

     

       42
       42
       -
       	return appbsky.FeedGetFeedSkeleton_Output{

     

       43
       43
       -
       		Cursor: &cursor,

     

       44
       44
       -
       		Feed: posts,

     

       45
       45
       -
       	}

     

       46
       46
       -
       }