From 78f3b98f2feb1381a5d5cf014152749add04f80b Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 5 Nov 2025 17:54:13 +0900 Subject: [PATCH] appview,lexicons: atprotate the mentions & references Change-Id: xokkvtswyxnvwmltoxuszyzwuvyynrxm Storing references parsed from the markdown body in atproto record and DB. There can be lots of reference types considering the from/to types so storing both as AT-URIs Using `sql.Tx` more to combine multiple DB query to single recoverable operation. Note: Pulls don't have mentinos/references yet Signed-off-by: Seongmin Lee --- api/tangled/cbor_gen.go | 493 +++++++++++++++++++++++++++++++++++- api/tangled/issuecomment.go | 12 +- api/tangled/pullcomment.go | 10 +- api/tangled/repoissue.go | 12 +- appview/db/db.go | 9 + appview/db/issues.go | 91 +++++-- appview/db/pulls.go | 34 ++- appview/db/reference.go | 78 ++++++ appview/ingester.go | 27 +- appview/issues/issues.go | 68 +++-- appview/models/issue.go | 80 ++++-- appview/models/pull.go | 26 ++ appview/pulls/pulls.go | 4 +- lexicons/issue/comment.json | 14 + lexicons/issue/issue.json | 14 + lexicons/pulls/comment.json | 14 + 16 files changed, 891 insertions(+), 95 deletions(-) diff --git a/api/tangled/cbor_gen.go b/api/tangled/cbor_gen.go index 39193aae..12074ec3 100644 --- a/api/tangled/cbor_gen.go +++ b/api/tangled/cbor_gen.go @@ -6744,12 +6744,20 @@ func (t *RepoIssue) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) - fieldCount := 5 + fieldCount := 7 if t.Body == nil { fieldCount-- } + if t.Mentions == nil { + fieldCount-- + } + + if t.References == nil { + fieldCount-- + } + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -6851,6 +6859,42 @@ func (t *RepoIssue) MarshalCBOR(w io.Writer) error { return err } + // t.Mentions ([]string) (slice) + if t.Mentions != nil { + + if len("mentions") > 1000000 { + return xerrors.Errorf("Value in field \"mentions\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mentions"))); err != nil { + return err + } + if _, err := cw.WriteString(string("mentions")); err != nil { + return err + } + + if len(t.Mentions) > 8192 { + return xerrors.Errorf("Slice value in field t.Mentions was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Mentions))); err != nil { + return err + } + for _, v := range t.Mentions { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } + // t.CreatedAt (string) (string) if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") @@ -6873,6 +6917,42 @@ func (t *RepoIssue) MarshalCBOR(w io.Writer) error { if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } + + // t.References ([]string) (slice) + if t.References != nil { + + if len("references") > 1000000 { + return xerrors.Errorf("Value in field \"references\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("references"))); err != nil { + return err + } + if _, err := cw.WriteString(string("references")); err != nil { + return err + } + + if len(t.References) > 8192 { + return xerrors.Errorf("Slice value in field t.References was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.References))); err != nil { + return err + } + for _, v := range t.References { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } return nil } @@ -6901,7 +6981,7 @@ func (t *RepoIssue) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 9) + nameBuf := make([]byte, 10) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -6971,6 +7051,46 @@ func (t *RepoIssue) UnmarshalCBOR(r io.Reader) (err error) { t.Title = string(sval) } + // t.Mentions ([]string) (slice) + case "mentions": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Mentions: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Mentions = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Mentions[i] = string(sval) + } + + } + } // t.CreatedAt (string) (string) case "createdAt": @@ -6982,6 +7102,46 @@ func (t *RepoIssue) UnmarshalCBOR(r io.Reader) (err error) { t.CreatedAt = string(sval) } + // t.References ([]string) (slice) + case "references": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.References: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.References = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.References[i] = string(sval) + } + + } + } default: // Field doesn't exist on this type, so ignore it @@ -7000,7 +7160,15 @@ func (t *RepoIssueComment) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) - fieldCount := 5 + fieldCount := 7 + + if t.Mentions == nil { + fieldCount-- + } + + if t.References == nil { + fieldCount-- + } if t.ReplyTo == nil { fieldCount-- @@ -7107,6 +7275,42 @@ func (t *RepoIssueComment) MarshalCBOR(w io.Writer) error { } } + // t.Mentions ([]string) (slice) + if t.Mentions != nil { + + if len("mentions") > 1000000 { + return xerrors.Errorf("Value in field \"mentions\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mentions"))); err != nil { + return err + } + if _, err := cw.WriteString(string("mentions")); err != nil { + return err + } + + if len(t.Mentions) > 8192 { + return xerrors.Errorf("Slice value in field t.Mentions was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Mentions))); err != nil { + return err + } + for _, v := range t.Mentions { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } + // t.CreatedAt (string) (string) if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") @@ -7129,6 +7333,42 @@ func (t *RepoIssueComment) MarshalCBOR(w io.Writer) error { if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } + + // t.References ([]string) (slice) + if t.References != nil { + + if len("references") > 1000000 { + return xerrors.Errorf("Value in field \"references\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("references"))); err != nil { + return err + } + if _, err := cw.WriteString(string("references")); err != nil { + return err + } + + if len(t.References) > 8192 { + return xerrors.Errorf("Slice value in field t.References was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.References))); err != nil { + return err + } + for _, v := range t.References { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } return nil } @@ -7157,7 +7397,7 @@ func (t *RepoIssueComment) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 9) + nameBuf := make([]byte, 10) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -7227,6 +7467,46 @@ func (t *RepoIssueComment) UnmarshalCBOR(r io.Reader) (err error) { t.ReplyTo = (*string)(&sval) } } + // t.Mentions ([]string) (slice) + case "mentions": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Mentions: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Mentions = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Mentions[i] = string(sval) + } + + } + } // t.CreatedAt (string) (string) case "createdAt": @@ -7238,6 +7518,46 @@ func (t *RepoIssueComment) UnmarshalCBOR(r io.Reader) (err error) { t.CreatedAt = string(sval) } + // t.References ([]string) (slice) + case "references": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.References: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.References = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.References[i] = string(sval) + } + + } + } default: // Field doesn't exist on this type, so ignore it @@ -7755,8 +8075,17 @@ func (t *RepoPullComment) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 6 - if _, err := cw.Write([]byte{164}); err != nil { + if t.Mentions == nil { + fieldCount-- + } + + if t.References == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -7825,6 +8154,42 @@ func (t *RepoPullComment) MarshalCBOR(w io.Writer) error { return err } + // t.Mentions ([]string) (slice) + if t.Mentions != nil { + + if len("mentions") > 1000000 { + return xerrors.Errorf("Value in field \"mentions\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mentions"))); err != nil { + return err + } + if _, err := cw.WriteString(string("mentions")); err != nil { + return err + } + + if len(t.Mentions) > 8192 { + return xerrors.Errorf("Slice value in field t.Mentions was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Mentions))); err != nil { + return err + } + for _, v := range t.Mentions { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } + // t.CreatedAt (string) (string) if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") @@ -7847,6 +8212,42 @@ func (t *RepoPullComment) MarshalCBOR(w io.Writer) error { if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } + + // t.References ([]string) (slice) + if t.References != nil { + + if len("references") > 1000000 { + return xerrors.Errorf("Value in field \"references\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("references"))); err != nil { + return err + } + if _, err := cw.WriteString(string("references")); err != nil { + return err + } + + if len(t.References) > 8192 { + return xerrors.Errorf("Slice value in field t.References was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.References))); err != nil { + return err + } + for _, v := range t.References { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } return nil } @@ -7875,7 +8276,7 @@ func (t *RepoPullComment) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 9) + nameBuf := make([]byte, 10) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -7924,6 +8325,46 @@ func (t *RepoPullComment) UnmarshalCBOR(r io.Reader) (err error) { t.LexiconTypeID = string(sval) } + // t.Mentions ([]string) (slice) + case "mentions": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Mentions: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Mentions = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Mentions[i] = string(sval) + } + + } + } // t.CreatedAt (string) (string) case "createdAt": @@ -7935,6 +8376,46 @@ func (t *RepoPullComment) UnmarshalCBOR(r io.Reader) (err error) { t.CreatedAt = string(sval) } + // t.References ([]string) (slice) + case "references": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.References: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.References = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.References[i] = string(sval) + } + + } + } default: // Field doesn't exist on this type, so ignore it diff --git a/api/tangled/issuecomment.go b/api/tangled/issuecomment.go index 3a2e7b8e..17e8a1f0 100644 --- a/api/tangled/issuecomment.go +++ b/api/tangled/issuecomment.go @@ -17,9 +17,11 @@ func init() { } // // RECORDTYPE: RepoIssueComment type RepoIssueComment struct { - LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue.comment" cborgen:"$type,const=sh.tangled.repo.issue.comment"` - Body string `json:"body" cborgen:"body"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Issue string `json:"issue" cborgen:"issue"` - ReplyTo *string `json:"replyTo,omitempty" cborgen:"replyTo,omitempty"` + LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue.comment" cborgen:"$type,const=sh.tangled.repo.issue.comment"` + Body string `json:"body" cborgen:"body"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Issue string `json:"issue" cborgen:"issue"` + Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` + References []string `json:"references,omitempty" cborgen:"references,omitempty"` + ReplyTo *string `json:"replyTo,omitempty" cborgen:"replyTo,omitempty"` } diff --git a/api/tangled/pullcomment.go b/api/tangled/pullcomment.go index f1da0d80..87748486 100644 --- a/api/tangled/pullcomment.go +++ b/api/tangled/pullcomment.go @@ -17,8 +17,10 @@ func init() { } // // RECORDTYPE: RepoPullComment type RepoPullComment struct { - LexiconTypeID string `json:"$type,const=sh.tangled.repo.pull.comment" cborgen:"$type,const=sh.tangled.repo.pull.comment"` - Body string `json:"body" cborgen:"body"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Pull string `json:"pull" cborgen:"pull"` + LexiconTypeID string `json:"$type,const=sh.tangled.repo.pull.comment" cborgen:"$type,const=sh.tangled.repo.pull.comment"` + Body string `json:"body" cborgen:"body"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` + Pull string `json:"pull" cborgen:"pull"` + References []string `json:"references,omitempty" cborgen:"references,omitempty"` } diff --git a/api/tangled/repoissue.go b/api/tangled/repoissue.go index e30dad86..3d02d83d 100644 --- a/api/tangled/repoissue.go +++ b/api/tangled/repoissue.go @@ -17,9 +17,11 @@ func init() { } // // RECORDTYPE: RepoIssue type RepoIssue struct { - LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue" cborgen:"$type,const=sh.tangled.repo.issue"` - Body *string `json:"body,omitempty" cborgen:"body,omitempty"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Repo string `json:"repo" cborgen:"repo"` - Title string `json:"title" cborgen:"title"` + LexiconTypeID string `json:"$type,const=sh.tangled.repo.issue" cborgen:"$type,const=sh.tangled.repo.issue"` + Body *string `json:"body,omitempty" cborgen:"body,omitempty"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` + References []string `json:"references,omitempty" cborgen:"references,omitempty"` + Repo string `json:"repo" cborgen:"repo"` + Title string `json:"title" cborgen:"title"` } diff --git a/appview/db/db.go b/appview/db/db.go index 260cb8ec..9ccacdb2 100644 --- a/appview/db/db.go +++ b/appview/db/db.go @@ -561,6 +561,13 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { email_notifications integer not null default 0 ); + create table if not exists references ( + id integer primary key autoincrement, + from_at text not null, + to_at text not null, + unique (from, to) + ); + create table if not exists migrations ( id integer primary key autoincrement, name text unique @@ -571,6 +578,8 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { create index if not exists idx_notifications_recipient_read on notifications(recipient_did, read); create index if not exists idx_stars_created on stars(created); create index if not exists idx_stars_repo_at_created on stars(repo_at, created); + create index if not exists idx_references_from_at on references(from_at); + create index if not exists idx_references_to_at on references(to_at); `) if err != nil { return nil, err diff --git a/appview/db/issues.go b/appview/db/issues.go index c8cd7ffa..1ef964ac 100644 --- a/appview/db/issues.go +++ b/appview/db/issues.go @@ -10,6 +10,7 @@ import ( "time" "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" "tangled.org/core/appview/models" "tangled.org/core/appview/pagination" ) @@ -69,7 +70,15 @@ func createNewIssue(tx *sql.Tx, issue *models.Issue) error { returning rowid, issue_id `, issue.RepoAt, issue.Did, issue.Rkey, newIssueId, issue.Title, issue.Body) - return row.Scan(&issue.Id, &issue.IssueId) + err = row.Scan(&issue.Id, &issue.IssueId) + if err != nil { + return fmt.Errorf("scan row: %w", err) + } + + if err := putReferences(tx, issue.AtUri(), issue.References); err != nil { + return fmt.Errorf("put references: %w", err) + } + return nil } func updateIssue(tx *sql.Tx, issue *models.Issue) error { @@ -79,7 +88,14 @@ func updateIssue(tx *sql.Tx, issue *models.Issue) error { set title = ?, body = ?, edited = ? where did = ? and rkey = ? `, issue.Title, issue.Body, time.Now().Format(time.RFC3339), issue.Did, issue.Rkey) - return err + if err != nil { + return err + } + + if err := putReferences(tx, issue.AtUri(), issue.References); err != nil { + return fmt.Errorf("put references: %w", err) + } + return nil } func GetIssuesPaginated(e Execer, page pagination.Page, filters ...filter) ([]models.Issue, error) { @@ -234,6 +250,17 @@ func GetIssuesPaginated(e Execer, page pagination.Page, filters ...filter) ([]mo } } + // collect references for each issue + allReferencs, err := GetReferencesAll(e, FilterIn("from_at", issueAts)) + if err != nil { + return nil, fmt.Errorf("failed to query references: %w", err) + } + for issueAt, references := range allReferencs { + if issue, ok := issueMap[issueAt.String()]; ok { + issue.References = references + } + } + var issues []models.Issue for _, i := range issueMap { issues = append(issues, *i) @@ -323,8 +350,8 @@ func GetIssueIDs(e Execer, opts models.IssueSearchOptions) ([]int64, error) { return ids, nil } -func AddIssueComment(e Execer, c models.IssueComment) (int64, error) { - result, err := e.Exec( +func AddIssueComment(tx *sql.Tx, c models.IssueComment) (int64, error) { + result, err := tx.Exec( `insert into issue_comments ( did, rkey, @@ -358,6 +385,10 @@ func AddIssueComment(e Execer, c models.IssueComment) (int64, error) { return 0, err } + if err := putReferences(tx, c.AtUri(), c.References); err != nil { + return 0, fmt.Errorf("put references: %w", err) + } + id, err := result.LastInsertId() if err != nil { return 0, err @@ -386,7 +417,7 @@ func DeleteIssueComments(e Execer, filters ...filter) error { } func GetIssueComments(e Execer, filters ...filter) ([]models.IssueComment, error) { - var comments []models.IssueComment + commentMap := make(map[string]*models.IssueComment) var conditions []string var args []any @@ -465,32 +496,56 @@ func GetIssueComments(e Execer, filters ...filter) ([]models.IssueComment, error comment.ReplyTo = &replyTo.V } - comments = append(comments, comment) + atUri := comment.AtUri().String() + commentMap[atUri] = &comment } if err = rows.Err(); err != nil { return nil, err } + // collect references for each comments + commentAts := slices.Collect(maps.Keys(commentMap)) + allReferencs, err := GetReferencesAll(e, FilterIn("from_at", commentAts)) + if err != nil { + return nil, fmt.Errorf("failed to query references: %w", err) + } + for commentAt, references := range allReferencs { + if comment, ok := commentMap[commentAt.String()]; ok { + comment.References = references + } + } + + var comments []models.IssueComment + for _, c := range commentMap { + comments = append(comments, *c) + } + + sort.Slice(comments, func(i, j int) bool { + return comments[i].Created.After(comments[j].Created) + }) + return comments, nil } -func DeleteIssues(e Execer, filters ...filter) error { - var conditions []string - var args []any - for _, filter := range filters { - conditions = append(conditions, filter.Condition()) - args = append(args, filter.Arg()...) +func DeleteIssues(tx *sql.Tx, did, rkey string) error { + _, err := tx.Exec( + `delete from issues + where did = ? and rkey = ?`, + did, + rkey, + ) + if err != nil { + return fmt.Errorf("delete issue: %w", err) } - whereClause := "" - if conditions != nil { - whereClause = " where " + strings.Join(conditions, " and ") + uri := syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", did, tangled.RepoIssueNSID, rkey)) + err = deleteReferences(tx, uri) + if err != nil { + return fmt.Errorf("delete references: %w", err) } - query := fmt.Sprintf(`delete from issues %s`, whereClause) - _, err := e.Exec(query, args...) - return err + return nil } func CloseIssues(e Execer, filters ...filter) error { diff --git a/appview/db/pulls.go b/appview/db/pulls.go index ff5d2def..8b4de107 100644 --- a/appview/db/pulls.go +++ b/appview/db/pulls.go @@ -492,7 +492,7 @@ func GetPullComments(e Execer, filters ...filter) ([]models.PullComment, error) } defer rows.Close() - var comments []models.PullComment + commentMap := make(map[string]*models.PullComment) for rows.Next() { var comment models.PullComment var createdAt string @@ -514,13 +514,35 @@ func GetPullComments(e Execer, filters ...filter) ([]models.PullComment, error) comment.Created = t } - comments = append(comments, comment) + atUri := comment.AtUri().String() + commentMap[atUri] = &comment } if err := rows.Err(); err != nil { return nil, err } + // collect references for each comments + commentAts := slices.Collect(maps.Keys(commentMap)) + allReferencs, err := GetReferencesAll(e, FilterIn("from_at", commentAts)) + if err != nil { + return nil, fmt.Errorf("failed to query references: %w", err) + } + for commentAt, references := range allReferencs { + if comment, ok := commentMap[commentAt.String()]; ok { + comment.References = references + } + } + + var comments []models.PullComment + for _, c := range commentMap { + comments = append(comments, *c) + } + + sort.Slice(comments, func(i, j int) bool { + return comments[i].Created.After(comments[j].Created) + }) + return comments, nil } @@ -600,9 +622,9 @@ func GetPullsByOwnerDid(e Execer, did, timeframe string) ([]models.Pull, error) return pulls, nil } -func NewPullComment(e Execer, comment *models.PullComment) (int64, error) { +func NewPullComment(tx *sql.Tx, comment *models.PullComment) (int64, error) { query := `insert into pull_comments (owner_did, repo_at, submission_id, comment_at, pull_id, body) values (?, ?, ?, ?, ?, ?)` - res, err := e.Exec( + res, err := tx.Exec( query, comment.OwnerDid, comment.RepoAt, @@ -620,6 +642,10 @@ func NewPullComment(e Execer, comment *models.PullComment) (int64, error) { return 0, err } + if err := putReferences(tx, comment.AtUri(), comment.References); err != nil { + return 0, fmt.Errorf("put references: %w", err) + } + return i, nil } diff --git a/appview/db/reference.go b/appview/db/reference.go index 19a3a0f0..a105a265 100644 --- a/appview/db/reference.go +++ b/appview/db/reference.go @@ -101,6 +101,10 @@ func findIssueReferences(e Execer, refLinks []models.ReferenceLink) ([]syntax.AT } uris = append(uris, uri) } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate rows: %w", err) + } + return uris, nil } @@ -170,3 +174,77 @@ func findPullReferences(e Execer, refLinks []models.ReferenceLink) ([]syntax.ATU } return uris, nil } + +func putReferences(tx *sql.Tx, fromAt syntax.ATURI, references []syntax.ATURI) error { + err := deleteReferences(tx, fromAt) + if err != nil { + return fmt.Errorf("delete old references: %w", err) + } + + values := make([]string, 0, len(references)) + args := make([]any, 0, len(references)*2) + for _, ref := range references { + values = append(values, "(?, ?)") + args = append(args, fromAt, ref) + } + _, err = tx.Exec( + fmt.Sprintf( + `insert into references (from, at) + values %s`, + strings.Join(values, ","), + ), + args..., + ) + if err != nil { + return fmt.Errorf("insert new references: %w", err) + } + return nil +} + +func deleteReferences(tx *sql.Tx, fromAt syntax.ATURI) error { + _, err := tx.Exec(`delete from references where from_at = ?`, fromAt) + return err +} + +func GetReferencesAll(e Execer, filters ...filter) (map[syntax.ATURI][]syntax.ATURI, error) { + var ( + conditions []string + args []any + ) + for _, filter := range filters { + conditions = append(conditions, filter.Condition()) + args = append(args, filter.Arg()...) + } + + whereClause := "" + if conditions != nil { + whereClause = " where " + strings.Join(conditions, " and ") + } + + rows, err := e.Query( + fmt.Sprintf( + `select from_at, to_at from references %s`, + whereClause, + ), + ) + if err != nil { + return nil, fmt.Errorf("query references: %w", err) + } + defer rows.Close() + + result := make(map[syntax.ATURI][]syntax.ATURI) + + for rows.Next() { + var from, to syntax.ATURI + if err := rows.Scan(&from, &to); err != nil { + return nil, fmt.Errorf("scan row: %w", err) + } + + result[from] = append(result[from], to) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate rows: %w", err) + } + + return result, nil +} diff --git a/appview/ingester.go b/appview/ingester.go index 1c5d6d0a..a57ec9c0 100644 --- a/appview/ingester.go +++ b/appview/ingester.go @@ -841,14 +841,25 @@ func (i *Ingester) ingestIssue(ctx context.Context, e *jmodels.Event) error { return nil case jmodels.CommitOperationDelete: + tx, err := ddb.BeginTx(ctx, nil) + if err != nil { + l.Error("failed to begin transaction", "err", err) + return err + } + defer tx.Rollback() + if err := db.DeleteIssues( - ddb, - db.FilterEq("did", did), - db.FilterEq("rkey", rkey), + tx, + did, + rkey, ); err != nil { l.Error("failed to delete", "err", err) return fmt.Errorf("failed to delete issue record: %w", err) } + if err := tx.Commit(); err != nil { + l.Error("failed to commit txn", "err", err) + return err + } return nil } @@ -888,12 +899,18 @@ func (i *Ingester) ingestIssueComment(e *jmodels.Event) error { return fmt.Errorf("failed to validate comment: %w", err) } - _, err = db.AddIssueComment(ddb, *comment) + tx, err := ddb.Begin() + if err != nil { + return fmt.Errorf("failed to start transaction: %w", err) + } + defer tx.Rollback() + + _, err = db.AddIssueComment(tx, *comment) if err != nil { return fmt.Errorf("failed to create issue comment: %w", err) } - return nil + return tx.Commit() case jmodels.CommitOperationDelete: if err := db.DeleteIssueComments( diff --git a/appview/issues/issues.go b/appview/issues/issues.go index a720bd4e..a8177167 100644 --- a/appview/issues/issues.go +++ b/appview/issues/issues.go @@ -241,6 +241,14 @@ func (rp *Issues) DeleteIssue(w http.ResponseWriter, r *http.Request) { } l = l.With("did", issue.Did, "rkey", issue.Rkey) + tx, err := rp.db.Begin() + if err != nil { + l.Error("failed to start transaction", "err", err) + rp.pages.Notice(w, "issue-comment", "Failed to create comment, try again later.") + return + } + defer tx.Rollback() + // delete from PDS client, err := rp.oauth.AuthorizedClient(r) if err != nil { @@ -261,11 +269,12 @@ func (rp *Issues) DeleteIssue(w http.ResponseWriter, r *http.Request) { } // delete from db - if err := db.DeleteIssues(rp.db, db.FilterEq("id", issue.Id)); err != nil { + if err := db.DeleteIssues(tx, issue.Did, issue.Rkey); err != nil { l.Error("failed to delete issue", "err", err) rp.pages.Notice(w, noticeId, "Failed to delete issue.") return } + tx.Commit() rp.notifier.DeleteIssue(r.Context(), issue) @@ -402,15 +411,17 @@ func (rp *Issues) NewIssueComment(w http.ResponseWriter, r *http.Request) { replyTo = &replyToUri } - mentions, _ := rp.refResolver.Resolve(r.Context(), body) + mentions, references := rp.refResolver.Resolve(r.Context(), body) comment := models.IssueComment{ - Did: user.Did, - Rkey: tid.TID(), - IssueAt: issue.AtUri().String(), - ReplyTo: replyTo, - Body: body, - Created: time.Now(), + Did: user.Did, + Rkey: tid.TID(), + IssueAt: issue.AtUri().String(), + ReplyTo: replyTo, + Body: body, + Created: time.Now(), + Mentions: mentions, + References: references, } if err = rp.validator.ValidateIssueComment(&comment); err != nil { l.Error("failed to validate comment", "err", err) @@ -447,7 +458,15 @@ func (rp *Issues) NewIssueComment(w http.ResponseWriter, r *http.Request) { } }() - commentId, err := db.AddIssueComment(rp.db, comment) + tx, err := rp.db.Begin() + if err != nil { + l.Error("failed to start transaction", "err", err) + rp.pages.Notice(w, "issue-comment", "Failed to create comment, try again later.") + return + } + defer tx.Rollback() + + commentId, err := db.AddIssueComment(tx, comment) if err != nil { l.Error("failed to create comment", "err", err) rp.pages.Notice(w, "issue-comment", "Failed to create comment.") @@ -569,12 +588,21 @@ func (rp *Issues) EditIssueComment(w http.ResponseWriter, r *http.Request) { newComment.Edited = &now record := newComment.AsRecord() - _, err = db.AddIssueComment(rp.db, newComment) + tx, err := rp.db.Begin() + if err != nil { + l.Error("failed to start transaction", "err", err) + rp.pages.Notice(w, "repo-notice", "Failed to update description, try again later.") + return + } + defer tx.Rollback() + + _, err = db.AddIssueComment(tx, newComment) if err != nil { l.Error("failed to perferom update-description query", "err", err) rp.pages.Notice(w, "repo-notice", "Failed to update description, try again later.") return } + tx.Commit() // rkey is optional, it was introduced later if newComment.Rkey != "" { @@ -881,17 +909,19 @@ func (rp *Issues) NewIssue(w http.ResponseWriter, r *http.Request) { }) case http.MethodPost: body := r.FormValue("body") - mentions, _ := rp.refResolver.Resolve(r.Context(), body) + mentions, references := rp.refResolver.Resolve(r.Context(), body) issue := &models.Issue{ - RepoAt: f.RepoAt(), - Rkey: tid.TID(), - Title: r.FormValue("title"), - Body: body, - Open: true, - Did: user.Did, - Created: time.Now(), - Repo: &f.Repo, + RepoAt: f.RepoAt(), + Rkey: tid.TID(), + Title: r.FormValue("title"), + Body: body, + Open: true, + Did: user.Did, + Created: time.Now(), + Mentions: mentions, + References: references, + Repo: &f.Repo, } if err := rp.validator.ValidateIssue(issue); err != nil { diff --git a/appview/models/issue.go b/appview/models/issue.go index db09fbcc..04ec8a0e 100644 --- a/appview/models/issue.go +++ b/appview/models/issue.go @@ -10,17 +10,19 @@ import ( ) type Issue struct { - Id int64 - Did string - Rkey string - RepoAt syntax.ATURI - IssueId int - Created time.Time - Edited *time.Time - Deleted *time.Time - Title string - Body string - Open bool + Id int64 + Did string + Rkey string + RepoAt syntax.ATURI + IssueId int + Created time.Time + Edited *time.Time + Deleted *time.Time + Title string + Body string + Open bool + Mentions []syntax.DID + References []syntax.ATURI // optionally, populate this when querying for reverse mappings // like comment counts, parent repo etc. @@ -34,11 +36,21 @@ func (i *Issue) AtUri() syntax.ATURI { } func (i *Issue) AsRecord() tangled.RepoIssue { + mentions := make([]string, len(i.Mentions)) + for i, did := range i.Mentions { + mentions[i] = string(did) + } + references := make([]string, len(i.References)) + for i, uri := range i.References { + references[i] = string(uri) + } return tangled.RepoIssue{ - Repo: i.RepoAt.String(), - Title: i.Title, - Body: &i.Body, - CreatedAt: i.Created.Format(time.RFC3339), + Repo: i.RepoAt.String(), + Title: i.Title, + Body: &i.Body, + Mentions: mentions, + References: references, + CreatedAt: i.Created.Format(time.RFC3339), } } @@ -161,15 +173,17 @@ func IssueFromRecord(did, rkey string, record tangled.RepoIssue) Issue { } type IssueComment struct { - Id int64 - Did string - Rkey string - IssueAt string - ReplyTo *string - Body string - Created time.Time - Edited *time.Time - Deleted *time.Time + Id int64 + Did string + Rkey string + IssueAt string + ReplyTo *string + Body string + Created time.Time + Edited *time.Time + Deleted *time.Time + Mentions []syntax.DID + References []syntax.ATURI } func (i *IssueComment) AtUri() syntax.ATURI { @@ -177,11 +191,21 @@ func (i *IssueComment) AtUri() syntax.ATURI { } func (i *IssueComment) AsRecord() tangled.RepoIssueComment { + mentions := make([]string, len(i.Mentions)) + for i, did := range i.Mentions { + mentions[i] = string(did) + } + references := make([]string, len(i.References)) + for i, uri := range i.References { + references[i] = string(uri) + } return tangled.RepoIssueComment{ - Body: i.Body, - Issue: i.IssueAt, - CreatedAt: i.Created.Format(time.RFC3339), - ReplyTo: i.ReplyTo, + Body: i.Body, + Issue: i.IssueAt, + CreatedAt: i.Created.Format(time.RFC3339), + ReplyTo: i.ReplyTo, + Mentions: mentions, + References: references, } } diff --git a/appview/models/pull.go b/appview/models/pull.go index 2ca51db7..cc2df601 100644 --- a/appview/models/pull.go +++ b/appview/models/pull.go @@ -147,10 +147,36 @@ type PullComment struct { // content Body string + // meta + Mentions []syntax.DID + References []syntax.ATURI + // meta Created time.Time } +func (p *PullComment) AtUri() syntax.ATURI { + return syntax.ATURI(p.CommentAt) +} + +// func (p *PullComment) AsRecord() tangled.RepoPullComment { +// mentions := make([]string, len(p.Mentions)) +// for i, did := range p.Mentions { +// mentions[i] = string(did) +// } +// references := make([]string, len(p.References)) +// for i, uri := range p.References { +// references[i] = string(uri) +// } +// return tangled.RepoPullComment{ +// Pull: p.PullAt, +// Body: p.Body, +// Mentions: mentions, +// References: references, +// CreatedAt: p.Created.Format(time.RFC3339), +// } +// } + func (p *Pull) LastRoundNumber() int { return len(p.Submissions) - 1 } diff --git a/appview/pulls/pulls.go b/appview/pulls/pulls.go index f757392a..eee9f817 100644 --- a/appview/pulls/pulls.go +++ b/appview/pulls/pulls.go @@ -733,7 +733,7 @@ func (s *Pulls) PullComment(w http.ResponseWriter, r *http.Request) { return } - mentions, _ := s.refResolver.Resolve(r.Context(), body) + mentions, references := s.refResolver.Resolve(r.Context(), body) // Start a transaction tx, err := s.db.BeginTx(r.Context(), nil) @@ -777,6 +777,8 @@ func (s *Pulls) PullComment(w http.ResponseWriter, r *http.Request) { Body: body, CommentAt: atResp.Uri, SubmissionId: pull.Submissions[roundNumber].ID, + Mentions: mentions, + References: references, } // Create the pull comment in the database with the commentAt field diff --git a/lexicons/issue/comment.json b/lexicons/issue/comment.json index 09cc8e0b..1bccebbf 100644 --- a/lexicons/issue/comment.json +++ b/lexicons/issue/comment.json @@ -29,6 +29,20 @@ "replyTo": { "type": "string", "format": "at-uri" + }, + "mentions": { + "type": "array", + "items": { + "type": "string", + "format": "did" + } + }, + "references": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + } } } } diff --git a/lexicons/issue/issue.json b/lexicons/issue/issue.json index c6c7976c..e576e782 100644 --- a/lexicons/issue/issue.json +++ b/lexicons/issue/issue.json @@ -24,6 +24,20 @@ "createdAt": { "type": "string", "format": "datetime" + }, + "mentions": { + "type": "array", + "items": { + "type": "string", + "format": "did" + } + }, + "references": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + } } } } diff --git a/lexicons/pulls/comment.json b/lexicons/pulls/comment.json index 79463942..c8304433 100644 --- a/lexicons/pulls/comment.json +++ b/lexicons/pulls/comment.json @@ -25,6 +25,20 @@ "createdAt": { "type": "string", "format": "datetime" + }, + "mentions": { + "type": "array", + "items": { + "type": "string", + "format": "did" + } + }, + "references": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + } } } } -- 2.43.0