···
"github.com/bluesky-social/indigo/atproto/syntax"
···
"github.com/go-git/go-git/v5/plumbing"
"tangled.sh/tangled.sh/core/api/tangled"
"tangled.sh/tangled.sh/core/appview/db"
"tangled.sh/tangled.sh/core/rbac"
-
type Ingester func(ctx context.Context, e *models.Event) error
-
func Ingest(d db.DbWrapper, enforcer *rbac.Enforcer) Ingester {
return func(ctx context.Context, e *models.Event) error {
lastTimeUs := eventTime + 1
-
if err := d.SaveLastTimeUs(lastTimeUs); err != nil {
err = fmt.Errorf("(deferred) failed to save last time us: %w", err)
···
switch e.Commit.Collection {
case tangled.GraphFollowNSID:
case tangled.FeedStarNSID:
case tangled.PublicKeyNSID:
case tangled.RepoArtifactNSID:
-
ingestArtifact(&d, e, enforcer)
case tangled.ActorProfileNSID:
case tangled.SpindleMemberNSID:
-
ingestSpindleMember(&d, e, enforcer)
case tangled.SpindleNSID:
-
ingestSpindle(&d, e, true) // TODO: change this to dynamic
-
func ingestStar(d *db.DbWrapper, e *models.Event) error {
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
var subjectUri syntax.ATURI
···
record := tangled.FeedStar{}
err := json.Unmarshal(raw, &record)
-
log.Println("invalid record")
subjectUri, err = syntax.ParseATURI(record.Subject)
-
log.Println("invalid record")
-
err = db.AddStar(d, did, subjectUri, e.Commit.RKey)
case models.CommitOperationDelete:
-
err = db.DeleteStarByRkey(d, did, e.Commit.RKey)
···
-
func ingestFollow(d *db.DbWrapper, e *models.Event) error {
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
···
record := tangled.GraphFollow{}
err = json.Unmarshal(raw, &record)
-
log.Println("invalid record")
subjectDid := record.Subject
-
err = db.AddFollow(d, did, subjectDid, e.Commit.RKey)
case models.CommitOperationDelete:
-
err = db.DeleteFollowByRkey(d, did, e.Commit.RKey)
···
-
func ingestPublicKey(d *db.DbWrapper, e *models.Event) error {
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
-
log.Println("processing add of pubkey")
raw := json.RawMessage(e.Commit.Record)
record := tangled.PublicKey{}
err = json.Unmarshal(raw, &record)
-
log.Printf("invalid record: %s", err)
-
err = db.AddPublicKey(d, did, name, key, e.Commit.RKey)
case models.CommitOperationDelete:
-
log.Println("processing delete of pubkey")
-
err = db.DeletePublicKeyByRkey(d, did, e.Commit.RKey)
···
-
func ingestArtifact(d *db.DbWrapper, e *models.Event, enforcer *rbac.Enforcer) error {
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
raw := json.RawMessage(e.Commit.Record)
record := tangled.RepoArtifact{}
err = json.Unmarshal(raw, &record)
-
log.Printf("invalid record: %s", err)
···
-
repo, err := db.GetRepoByAtUri(d, repoAt.String())
-
ok, err := enforcer.E.Enforce(did, repo.Knot, repo.DidSlashRepo(), "repo:push")
···
MimeType: record.Artifact.MimeType,
-
err = db.AddArtifact(d, artifact)
case models.CommitOperationDelete:
-
err = db.DeleteArtifact(d, db.FilterEq("did", did), db.FilterEq("rkey", e.Commit.RKey))
···
-
func ingestProfile(d *db.DbWrapper, e *models.Event) error {
if e.Commit.RKey != "self" {
return fmt.Errorf("ingestProfile only ingests `self` record")
···
record := tangled.ActorProfile{}
err = json.Unmarshal(raw, &record)
-
log.Printf("invalid record: %s", err)
···
-
ddb, ok := d.Execer.(*db.DB)
return fmt.Errorf("failed to index profile record, invalid db cast")
···
err = db.UpsertProfile(tx, &profile)
case models.CommitOperationDelete:
-
err = db.DeleteArtifact(d, db.FilterEq("did", did), db.FilterEq("rkey", e.Commit.RKey))
···
-
func ingestSpindleMember(_ *db.DbWrapper, e *models.Event, enforcer *rbac.Enforcer) error {
switch e.Commit.Operation {
case models.CommitOperationCreate:
raw := json.RawMessage(e.Commit.Record)
record := tangled.SpindleMember{}
err = json.Unmarshal(raw, &record)
-
log.Printf("invalid record: %s", err)
// only spindle owner can invite to spindles
-
ok, err := enforcer.IsSpindleInviteAllowed(did, record.Instance)
return fmt.Errorf("failed to enforce permissions: %w", err)
-
err = enforcer.AddSpindleMember(record.Instance, record.Subject)
-
return fmt.Errorf("failed to add member: %w", err)
-
func ingestSpindle(d *db.DbWrapper, e *models.Event, dev bool) error {
switch e.Commit.Operation {
case models.CommitOperationCreate:
raw := json.RawMessage(e.Commit.Record)
record := tangled.Spindle{}
err = json.Unmarshal(raw, &record)
-
log.Printf("invalid record: %s", err)
-
// this is a special record whose rkey is the instance of the spindle itself
instance := e.Commit.RKey
-
owner, err := fetchOwner(context.TODO(), instance, dev)
-
log.Printf("failed to verify owner of %s: %s", instance, err)
-
// verify that the spindle owner points back to this did
-
log.Printf("incorrect owner for domain: %s, %s != %s", instance, owner, did)
-
// mark this spindle as registered
-
ddb, ok := d.Execer.(*db.DB)
return fmt.Errorf("failed to index profile record, invalid db cast")
-
_, err = db.VerifySpindle(
db.FilterEq("owner", did),
db.FilterEq("instance", instance),
-
func fetchOwner(ctx context.Context, domain string, dev bool) (string, error) {
-
url := fmt.Sprintf("%s://%s/owner", scheme, domain)
-
req, err := http.NewRequest("GET", url, nil)
-
client := &http.Client{
-
Timeout: 1 * time.Second,
-
resp, err := client.Do(req.WithContext(ctx))
-
if err != nil || resp.StatusCode != 200 {
-
return "", errors.New("failed to fetch /owner")
-
body, err := io.ReadAll(io.LimitReader(resp.Body, 1024)) // read atmost 1kb of data
-
return "", fmt.Errorf("failed to read /owner response: %w", err)
-
did := strings.TrimSpace(string(body))
-
return "", errors.New("empty DID in /owner response")
···
"github.com/bluesky-social/indigo/atproto/syntax"
···
"github.com/go-git/go-git/v5/plumbing"
"tangled.sh/tangled.sh/core/api/tangled"
+
"tangled.sh/tangled.sh/core/appview/config"
"tangled.sh/tangled.sh/core/appview/db"
+
"tangled.sh/tangled.sh/core/appview/idresolver"
+
"tangled.sh/tangled.sh/core/appview/spindleverify"
"tangled.sh/tangled.sh/core/rbac"
+
Enforcer *rbac.Enforcer
+
IdResolver *idresolver.Resolver
+
type processFunc func(ctx context.Context, e *models.Event) error
+
func (i *Ingester) Ingest() processFunc {
return func(ctx context.Context, e *models.Event) error {
lastTimeUs := eventTime + 1
+
if err := i.Db.SaveLastTimeUs(lastTimeUs); err != nil {
err = fmt.Errorf("(deferred) failed to save last time us: %w", err)
···
switch e.Commit.Collection {
case tangled.GraphFollowNSID:
+
err = i.ingestFollow(e)
case tangled.FeedStarNSID:
case tangled.PublicKeyNSID:
+
err = i.ingestPublicKey(e)
case tangled.RepoArtifactNSID:
+
err = i.ingestArtifact(e)
case tangled.ActorProfileNSID:
+
err = i.ingestProfile(e)
case tangled.SpindleMemberNSID:
+
err = i.ingestSpindleMember(e)
case tangled.SpindleNSID:
+
err = i.ingestSpindle(e)
+
l := i.Logger.With("nsid", e.Commit.Collection)
+
l.Error("error ingesting record", "err", err)
+
func (i *Ingester) ingestStar(e *models.Event) error {
+
l := i.Logger.With("handler", "ingestStar")
+
l = l.With("nsid", e.Commit.Collection)
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
var subjectUri syntax.ATURI
···
record := tangled.FeedStar{}
err := json.Unmarshal(raw, &record)
+
l.Error("invalid record", "err", err)
subjectUri, err = syntax.ParseATURI(record.Subject)
+
l.Error("invalid record", "err", err)
+
err = db.AddStar(i.Db, did, subjectUri, e.Commit.RKey)
case models.CommitOperationDelete:
+
err = db.DeleteStarByRkey(i.Db, did, e.Commit.RKey)
···
+
func (i *Ingester) ingestFollow(e *models.Event) error {
+
l := i.Logger.With("handler", "ingestFollow")
+
l = l.With("nsid", e.Commit.Collection)
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
···
record := tangled.GraphFollow{}
err = json.Unmarshal(raw, &record)
+
l.Error("invalid record", "err", err)
subjectDid := record.Subject
+
err = db.AddFollow(i.Db, did, subjectDid, e.Commit.RKey)
case models.CommitOperationDelete:
+
err = db.DeleteFollowByRkey(i.Db, did, e.Commit.RKey)
···
+
func (i *Ingester) ingestPublicKey(e *models.Event) error {
+
l := i.Logger.With("handler", "ingestPublicKey")
+
l = l.With("nsid", e.Commit.Collection)
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
+
l.Debug("processing add of pubkey")
raw := json.RawMessage(e.Commit.Record)
record := tangled.PublicKey{}
err = json.Unmarshal(raw, &record)
+
l.Error("invalid record", "err", err)
+
err = db.AddPublicKey(i.Db, did, name, key, e.Commit.RKey)
case models.CommitOperationDelete:
+
l.Debug("processing delete of pubkey")
+
err = db.DeletePublicKeyByRkey(i.Db, did, e.Commit.RKey)
···
+
func (i *Ingester) ingestArtifact(e *models.Event) error {
+
l := i.Logger.With("handler", "ingestArtifact")
+
l = l.With("nsid", e.Commit.Collection)
switch e.Commit.Operation {
case models.CommitOperationCreate, models.CommitOperationUpdate:
raw := json.RawMessage(e.Commit.Record)
record := tangled.RepoArtifact{}
err = json.Unmarshal(raw, &record)
+
l.Error("invalid record", "err", err)
···
+
repo, err := db.GetRepoByAtUri(i.Db, repoAt.String())
+
ok, err := i.Enforcer.E.Enforce(did, repo.Knot, repo.DidSlashRepo(), "repo:push")
···
MimeType: record.Artifact.MimeType,
+
err = db.AddArtifact(i.Db, artifact)
case models.CommitOperationDelete:
+
err = db.DeleteArtifact(i.Db, db.FilterEq("did", did), db.FilterEq("rkey", e.Commit.RKey))
···
+
func (i *Ingester) ingestProfile(e *models.Event) error {
+
l := i.Logger.With("handler", "ingestProfile")
+
l = l.With("nsid", e.Commit.Collection)
if e.Commit.RKey != "self" {
return fmt.Errorf("ingestProfile only ingests `self` record")
···
record := tangled.ActorProfile{}
err = json.Unmarshal(raw, &record)
+
l.Error("invalid record", "err", err)
···
+
ddb, ok := i.Db.Execer.(*db.DB)
return fmt.Errorf("failed to index profile record, invalid db cast")
···
err = db.UpsertProfile(tx, &profile)
case models.CommitOperationDelete:
+
err = db.DeleteArtifact(i.Db, db.FilterEq("did", did), db.FilterEq("rkey", e.Commit.RKey))
···
+
func (i *Ingester) ingestSpindleMember(e *models.Event) error {
+
l := i.Logger.With("handler", "ingestSpindleMember")
+
l = l.With("nsid", e.Commit.Collection)
switch e.Commit.Operation {
case models.CommitOperationCreate:
raw := json.RawMessage(e.Commit.Record)
record := tangled.SpindleMember{}
err = json.Unmarshal(raw, &record)
+
l.Error("invalid record", "err", err)
// only spindle owner can invite to spindles
+
ok, err := i.Enforcer.IsSpindleInviteAllowed(did, record.Instance)
return fmt.Errorf("failed to enforce permissions: %w", err)
+
memberId, err := i.IdResolver.ResolveIdent(context.Background(), record.Subject)
+
if memberId.Handle.IsInvalidHandle() {
+
ddb, ok := i.Db.Execer.(*db.DB)
+
return fmt.Errorf("failed to index profile record, invalid db cast")
+
err = db.AddSpindleMember(ddb, db.SpindleMember{
+
Instance: record.Instance,
+
return fmt.Errorf("failed to add to db: %w", err)
+
err = i.Enforcer.AddSpindleMember(record.Instance, memberId.DID.String())
+
return fmt.Errorf("failed to update ACLs: %w", err)
+
case models.CommitOperationDelete:
+
ddb, ok := i.Db.Execer.(*db.DB)
+
return fmt.Errorf("failed to index profile record, invalid db cast")
+
// get record from db first
+
members, err := db.GetSpindleMembers(
+
db.FilterEq("did", did),
+
db.FilterEq("rkey", rkey),
+
if err != nil || len(members) != 1 {
+
return fmt.Errorf("failed to get member: %w, len(members) = %d", err, len(members))
+
return fmt.Errorf("failed to start txn: %w", err)
+
// remove record by rkey && update enforcer
+
if err = db.RemoveSpindleMember(
+
db.FilterEq("did", did),
+
db.FilterEq("rkey", rkey),
+
return fmt.Errorf("failed to remove from db: %w", err)
+
err = i.Enforcer.RemoveSpindleMember(member.Instance, member.Subject.String())
+
return fmt.Errorf("failed to update ACLs: %w", err)
+
if err = tx.Commit(); err != nil {
+
return fmt.Errorf("failed to commit txn: %w", err)
+
if err = i.Enforcer.E.SavePolicy(); err != nil {
+
return fmt.Errorf("failed to save ACLs: %w", err)
+
func (i *Ingester) ingestSpindle(e *models.Event) error {
+
l := i.Logger.With("handler", "ingestSpindle")
+
l = l.With("nsid", e.Commit.Collection)
switch e.Commit.Operation {
case models.CommitOperationCreate:
raw := json.RawMessage(e.Commit.Record)
record := tangled.Spindle{}
err = json.Unmarshal(raw, &record)
+
l.Error("invalid record", "err", err)
instance := e.Commit.RKey
+
ddb, ok := i.Db.Execer.(*db.DB)
+
return fmt.Errorf("failed to index profile record, invalid db cast")
+
err := db.AddSpindle(ddb, db.Spindle{
+
Owner: syntax.DID(did),
+
l.Error("failed to add spindle to db", "err", err, "instance", instance)
+
err = spindleverify.RunVerification(context.Background(), instance, did, i.Config.Core.Dev)
+
l.Error("failed to add spindle to db", "err", err, "instance", instance)
+
_, err = spindleverify.MarkVerified(ddb, i.Enforcer, instance, did)
+
return fmt.Errorf("failed to mark verified: %w", err)
+
case models.CommitOperationDelete:
+
instance := e.Commit.RKey
+
ddb, ok := i.Db.Execer.(*db.DB)
return fmt.Errorf("failed to index profile record, invalid db cast")
+
i.Enforcer.E.LoadPolicy()
+
err = db.DeleteSpindle(
db.FilterEq("owner", did),
db.FilterEq("instance", instance),
+
err = i.Enforcer.RemoveSpindle(instance)
+
err = i.Enforcer.E.SavePolicy()
nel.pet
oppi.li