From 3db1b26ed5218abea784073b720471d1fdc999bc Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Sat, 6 Dec 2025 22:57:02 +0900 Subject: [PATCH] nix: bump gomod2nix Change-Id: swssttysvkstsxxpxxopmwqmwylotmlo Signed-off-by: Seongmin Lee --- nix/gomod2nix.toml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml index 082331e8..7c518020 100644 --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -165,9 +165,6 @@ schema = 3 [mod."github.com/davecgh/go-spew"] version = "v1.1.2-0.20180830191138-d8f796af33cc" hash = "sha256-fV9oI51xjHdOmEx6+dlq7Ku2Ag+m/bmbzPo6A4Y74qc=" - [mod."github.com/decred/dcrd/dcrec/secp256k1/v4"] - version = "v4.4.0" - hash = "sha256-qrhEIwhDll3cxoVpMbm1NQ9/HTI42S7ms8Buzlo5HCg=" [mod."github.com/dgraph-io/ristretto"] version = "v0.2.0" hash = "sha256-bnpxX+oO/Qf7IJevA0gsbloVoqRx+5bh7RQ9d9eLNYw=" @@ -373,24 +370,6 @@ schema = 3 [mod."github.com/klauspost/cpuid/v2"] version = "v2.3.0" hash = "sha256-50JhbQyT67BK38HIdJihPtjV7orYp96HknI2VP7A9Yc=" - [mod."github.com/lestrrat-go/blackmagic"] - version = "v1.0.4" - hash = "sha256-HmWOpwoPDNMwLdOi7onNn3Sb+ZsAa3Ai3gVBbXmQ0e8=" - [mod."github.com/lestrrat-go/httpcc"] - version = "v1.0.1" - hash = "sha256-SMRSwJpqDIs/xL0l2e8vP0W65qtCHX2wigcOeqPJmos=" - [mod."github.com/lestrrat-go/httprc"] - version = "v1.0.6" - hash = "sha256-mfZzePEhrmyyu/avEBd2MsDXyto8dq5+fyu5lA8GUWM=" - [mod."github.com/lestrrat-go/iter"] - version = "v1.0.2" - hash = "sha256-30tErRf7Qu/NOAt1YURXY/XJSA6sCr6hYQfO8QqHrtw=" - [mod."github.com/lestrrat-go/jwx/v2"] - version = "v2.1.6" - hash = "sha256-0LszXRZIba+X8AOrs3T4uanAUafBdlVB8/MpUNEFpbc=" - [mod."github.com/lestrrat-go/option"] - version = "v1.0.1" - hash = "sha256-jVcIYYVsxElIS/l2akEw32vdEPR8+anR6oeT1FoYULI=" [mod."github.com/lucasb-eyer/go-colorful"] version = "v1.2.0" hash = "sha256-Gg9dDJFCTaHrKHRR1SrJgZ8fWieJkybljybkI9x0gyE=" @@ -511,9 +490,6 @@ schema = 3 [mod."github.com/ryanuber/go-glob"] version = "v1.0.0" hash = "sha256-YkMl1utwUhi3E0sHK23ISpAsPyj4+KeXyXKoFYGXGVY=" - [mod."github.com/segmentio/asm"] - version = "v1.2.0" - hash = "sha256-zbNuKxNrUDUc6IlmRQNuJQzVe5Ol/mqp7srDg9IMMqs=" [mod."github.com/sergi/go-diff"] version = "v1.1.0" hash = "sha256-8NJMabldpf40uwQN20T6QXx5KORDibCBJL02KD661xY=" -- 2.43.0 From 214ccbdd64c32a72bca8e240d460062bd9072098 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 13:55:56 +0900 Subject: [PATCH] appview,knotserver: support immutable nix flakeref link header Change-Id: uxntvwzvsmqpkmonkspxmzqpxktqrusw This will allow users to directly request archive from the knot without using xrpc. Xrpc doesn't fit here because it strips out the http headers which might include valuable metadata like download filename or immutable link. - implement archive on knot as `/{owner}/{repo}/archive/{ref}` endpoint - appview proxies the request to knot on `/archive` like it is doing for git http endpoints. if knot version isn't compatible, it will fallback to legacy xrpc endpoint. - rename the `git_http.go` file to generalized `proxy_knot.go` filaname xrpc method `sh.tangled.repo.archive` will be deprecated in future added `go-version` depenedency to make version constraints Close: Signed-off-by: Seongmin Lee --- appview/repo/archive.go | 49 -------------- appview/repo/router.go | 4 -- appview/state/{git_http.go => proxy_knot.go} | 71 ++++++++++++++++++++ appview/state/router.go | 4 +- go.mod | 1 + go.sum | 2 + knotserver/archive.go | 69 +++++++++++++++++++ knotserver/git/git.go | 4 ++ knotserver/router.go | 2 + nix/gomod2nix.toml | 3 + 10 files changed, 155 insertions(+), 54 deletions(-) delete mode 100644 appview/repo/archive.go rename appview/state/{git_http.go => proxy_knot.go} (51%) create mode 100644 knotserver/archive.go diff --git a/appview/repo/archive.go b/appview/repo/archive.go deleted file mode 100644 index eac24201..00000000 --- a/appview/repo/archive.go +++ /dev/null @@ -1,49 +0,0 @@ -package repo - -import ( - "fmt" - "net/http" - "net/url" - "strings" - - "tangled.org/core/api/tangled" - xrpcclient "tangled.org/core/appview/xrpcclient" - - indigoxrpc "github.com/bluesky-social/indigo/xrpc" - "github.com/go-chi/chi/v5" - "github.com/go-git/go-git/v5/plumbing" -) - -func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { - l := rp.logger.With("handler", "DownloadArchive") - ref := chi.URLParam(r, "ref") - ref, _ = url.PathUnescape(ref) - f, err := rp.repoResolver.Resolve(r) - if err != nil { - l.Error("failed to get repo and knot", "err", err) - return - } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - didSlashRepo := f.DidSlashRepo() - archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, didSlashRepo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.archive", "err", xrpcerr) - rp.pages.Error503(w) - return - } - // Set headers for file download, just pass along whatever the knot specifies - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) - // Write the archive data directly - w.Write(archiveBytes) -} diff --git a/appview/repo/router.go b/appview/repo/router.go index 1667ae58..6e097e5d 100644 --- a/appview/repo/router.go +++ b/appview/repo/router.go @@ -40,10 +40,6 @@ func (rp *Repo) Router(mw *middleware.Middleware) http.Handler { r.Get("/blob/{ref}/*", rp.Blob) r.Get("/raw/{ref}/*", rp.RepoBlobRaw) - // intentionally doesn't use /* as this isn't - // a file path - r.Get("/archive/{ref}", rp.DownloadArchive) - r.Route("/fork", func(r chi.Router) { r.Use(middleware.AuthMiddleware(rp.oauth)) r.Get("/", rp.ForkRepo) diff --git a/appview/state/git_http.go b/appview/state/proxy_knot.go similarity index 51% rename from appview/state/git_http.go rename to appview/state/proxy_knot.go index 36ce3e8d..d13476f0 100644 --- a/appview/state/git_http.go +++ b/appview/state/proxy_knot.go @@ -5,10 +5,16 @@ import ( "io" "maps" "net/http" + "strings" "github.com/bluesky-social/indigo/atproto/identity" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/hashicorp/go-version" + "tangled.org/core/api/tangled" "tangled.org/core/appview/models" + xrpcclient "tangled.org/core/appview/xrpcclient" ) func (s *State) InfoRefs(w http.ResponseWriter, r *http.Request) { @@ -59,6 +65,71 @@ func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) { s.proxyRequest(w, r, targetURL) } +var knotVersionDownloadArchiveConstraint = version.MustConstraints(version.NewConstraint(">= 1.12.0-alpha")) + +func (s *State) DownloadArchive(w http.ResponseWriter, r *http.Request) { + l := s.logger.With("handler", "DownloadArchive") + ref := chi.URLParam(r, "ref") + + user, ok := r.Context().Value("resolvedId").(identity.Identity) + if !ok { + l.Error("failed to resolve user") + http.Error(w, "failed to resolve user", http.StatusInternalServerError) + return + } + repo := r.Context().Value("repo").(*models.Repo) + + scheme := "https" + if s.config.Core.Dev { + scheme = "http" + } + + host := fmt.Sprintf("%s://%s", scheme, repo.Knot) + xrpcc := &indigoxrpc.Client{ + Host: host, + } + l = l.With("knot", repo.Knot) + + isCompatible := func() bool { + out, err := tangled.KnotVersion(r.Context(), xrpcc) + if err != nil { + l.Warn("failed to get knot version", "err", err) + return false + } + + v, err := version.NewVersion(out.Version) + if err != nil { + l.Warn("failed to parse knot version", "version", out.Version, "err", err) + return false + } + + if !knotVersionDownloadArchiveConstraint.Check(v) { + l.Warn("knot version incompatible.", "version", v) + return false + } + return true + }() + l.Debug("knot compatibility check", "isCompatible", isCompatible) + if isCompatible { + targetURL := fmt.Sprintf("%s://%s/%s/%s/archive/%s", scheme, repo.Knot, user.DID, repo.Name, ref) + s.proxyRequest(w, r, targetURL) + } else { + l.Debug("requesting xrpc/sh.tangled.repo.archive") + archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, repo.DidSlashRepo()) + if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { + l.Error("failed to call XRPC repo.archive", "err", xrpcerr) + s.pages.Error503(w) + return + } + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + filename := fmt.Sprintf("%s-%s.tar.gz", repo.Name, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) + w.Write(archiveBytes) + } +} + func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) { client := &http.Client{} diff --git a/appview/state/router.go b/appview/state/router.go index 968ed403..2f71a895 100644 --- a/appview/state/router.go +++ b/appview/state/router.go @@ -103,7 +103,9 @@ func (s *State) UserRouter(mw *middleware.Middleware) http.Handler { r.Get("/info/refs", s.InfoRefs) r.Post("/git-upload-pack", s.UploadPack) r.Post("/git-receive-pack", s.ReceivePack) - + // intentionally doesn't use /* as this isn't + // a file path + r.Get("/archive/{ref}", s.DownloadArchive) }) }) diff --git a/go.mod b/go.mod index a944a511..bbfa50ef 100644 --- a/go.mod +++ b/go.mod @@ -132,6 +132,7 @@ require ( github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.1-vault-7 // indirect diff --git a/go.sum b/go.sum index 33e1f691..5fa30c99 100644 --- a/go.sum +++ b/go.sum @@ -264,6 +264,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= diff --git a/knotserver/archive.go b/knotserver/archive.go new file mode 100644 index 00000000..9a5f48b1 --- /dev/null +++ b/knotserver/archive.go @@ -0,0 +1,69 @@ +package knotserver + +import ( + "compress/gzip" + "fmt" + "net/http" + "strings" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/knotserver/git" +) + +func (h *Knot) Archive(w http.ResponseWriter, r *http.Request) { + var ( + did = chi.URLParam(r, "did") + name = chi.URLParam(r, "name") + ref = chi.URLParam(r, "ref") + ) + repo, err := securejoin.SecureJoin(did, name) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, repo) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + gr, err := git.Open(repoPath, ref) + + immutableLink := fmt.Sprintf( + "https://%s/%s/%s/archive/%s", + h.c.Server.Hostname, + did, + name, + gr.Hash(), + ) + + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + filename := fmt.Sprintf("%s-%s.tar.gz", name, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) + + gw := gzip.NewWriter(w) + defer gw.Close() + + err = gr.WriteTar(gw, "") + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("writing tar file", "error", err) + return + } + + err = gw.Flush() + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("flushing", "error", err.Error()) + return + } +} diff --git a/knotserver/git/git.go b/knotserver/git/git.go index 6fbb3329..5d888852 100644 --- a/knotserver/git/git.go +++ b/knotserver/git/git.go @@ -76,6 +76,10 @@ func PlainOpen(path string) (*GitRepo, error) { return &g, nil } +func (g *GitRepo) Hash() plumbing.Hash { + return g.h +} + // re-open a repository and update references func (g *GitRepo) Refresh() error { refreshed, err := PlainOpen(g.path) diff --git a/knotserver/router.go b/knotserver/router.go index 482f932d..2aef216d 100644 --- a/knotserver/router.go +++ b/knotserver/router.go @@ -84,6 +84,8 @@ func (h *Knot) Router() http.Handler { r.Get("/info/refs", h.InfoRefs) r.Post("/git-upload-pack", h.UploadPack) r.Post("/git-receive-pack", h.ReceivePack) + // convenience routes + r.Get("/archive/{ref}", h.Archive) }) }) diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml index 7c518020..1e406dce 100644 --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -304,6 +304,9 @@ schema = 3 [mod."github.com/hashicorp/go-sockaddr"] version = "v1.0.7" hash = "sha256-p6eDOrGzN1jMmT/F/f/VJMq0cKNFhUcEuVVwTE6vSrs=" + [mod."github.com/hashicorp/go-version"] + version = "v1.8.0" + hash = "sha256-KXtqERmYrWdpqPCViWcHbe6jnuH7k16bvBIcuJuevj8=" [mod."github.com/hashicorp/golang-lru"] version = "v1.0.2" hash = "sha256-yy+5botc6T5wXgOe2mfNXJP3wr+MkVlUZ2JBkmmrA48=" -- 2.43.0 From 4ed66443944a20a4346f61d35d2597d3c881e959 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 15:54:15 +0900 Subject: [PATCH] lexicons,knotserver: remove `/xrpc/sh.tangled.repo.archive` endpoint Change-Id: xuplvooxylpkzuvqzoryvlpxuzxpvmpn replaced by `/{did}/{reponame}/archive/{ref}`. The lexicon definitions will be removed later. Signed-off-by: Seongmin Lee --- knotserver/xrpc/repo_archive.go | 81 --------------------------------- knotserver/xrpc/xrpc.go | 1 - lexicons/repo/archive.json | 1 + 3 files changed, 1 insertion(+), 82 deletions(-) delete mode 100644 knotserver/xrpc/repo_archive.go diff --git a/knotserver/xrpc/repo_archive.go b/knotserver/xrpc/repo_archive.go deleted file mode 100644 index 42fe498e..00000000 --- a/knotserver/xrpc/repo_archive.go +++ /dev/null @@ -1,81 +0,0 @@ -package xrpc - -import ( - "compress/gzip" - "fmt" - "net/http" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - - "tangled.org/core/knotserver/git" - xrpcerr "tangled.org/core/xrpc/errors" -) - -func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) { - repo := r.URL.Query().Get("repo") - repoPath, err := x.parseRepoParam(repo) - if err != nil { - writeError(w, err.(xrpcerr.XrpcError), http.StatusBadRequest) - return - } - - ref := r.URL.Query().Get("ref") - // ref can be empty (git.Open handles this) - - format := r.URL.Query().Get("format") - if format == "" { - format = "tar.gz" // default - } - - prefix := r.URL.Query().Get("prefix") - - if format != "tar.gz" { - writeError(w, xrpcerr.NewXrpcError( - xrpcerr.WithTag("InvalidRequest"), - xrpcerr.WithMessage("only tar.gz format is supported"), - ), http.StatusBadRequest) - return - } - - gr, err := git.Open(repoPath, ref) - if err != nil { - writeError(w, xrpcerr.RefNotFoundError, http.StatusNotFound) - return - } - - repoParts := strings.Split(repo, "/") - repoName := repoParts[len(repoParts)-1] - - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - - var archivePrefix string - if prefix != "" { - archivePrefix = prefix - } else { - archivePrefix = fmt.Sprintf("%s-%s", repoName, safeRefFilename) - } - - filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - - gw := gzip.NewWriter(w) - defer gw.Close() - - err = gr.WriteTar(gw, archivePrefix) - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("writing tar file", "error", err.Error()) - return - } - - err = gw.Flush() - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("flushing", "error", err.Error()) - return - } -} diff --git a/knotserver/xrpc/xrpc.go b/knotserver/xrpc/xrpc.go index c110356f..50615e80 100644 --- a/knotserver/xrpc/xrpc.go +++ b/knotserver/xrpc/xrpc.go @@ -64,7 +64,6 @@ func (x *Xrpc) Router() http.Handler { r.Get("/"+tangled.RepoCompareNSID, x.RepoCompare) r.Get("/"+tangled.RepoGetDefaultBranchNSID, x.RepoGetDefaultBranch) r.Get("/"+tangled.RepoBranchNSID, x.RepoBranch) - r.Get("/"+tangled.RepoArchiveNSID, x.RepoArchive) r.Get("/"+tangled.RepoLanguagesNSID, x.RepoLanguages) // knot query endpoints (no auth required) diff --git a/lexicons/repo/archive.json b/lexicons/repo/archive.json index 05e313d1..5e9e038c 100644 --- a/lexicons/repo/archive.json +++ b/lexicons/repo/archive.json @@ -4,6 +4,7 @@ "defs": { "main": { "type": "query", + "description": "deprecated. use `/{did}/{reponame}/archive/{ref} endpoint instead", "parameters": { "type": "params", "required": ["repo", "ref"], -- 2.43.0