From ace30b7556cee736cbee90d6839cdbd51fbf0c89 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 13:55:56 +0900 Subject: [PATCH] appview,knotserver: support knot http endpoint for archive Change-Id: uxntvwzvsmqpkmonkspxmzqpxktqrusw This will allow users to directly request archive from the knot without using xrpc. Xrpc doesn't fit here because it strips out the http headers which might include valuable metadata like download filename or immutable link. - remove xrpc method `sh.tangled.repo.archive` - reimplement archive on knot as `/{owner}/{repo}/archive/{ref}` endpoint - appview will just proxy the request to knot on `/archive` like it is doing for git http endpoints - rename the `git_http.go` file to generalized `proxy_knot.go` filaname Signed-off-by: Seongmin Lee --- appview/repo/archive.go | 49 -------------------- appview/repo/router.go | 4 -- appview/state/{git_http.go => proxy_knot.go} | 19 ++++++++ appview/state/router.go | 4 +- knotserver/router.go | 2 + 5 files changed, 24 insertions(+), 54 deletions(-) delete mode 100644 appview/repo/archive.go rename appview/state/{git_http.go => proxy_knot.go} (82%) diff --git a/appview/repo/archive.go b/appview/repo/archive.go deleted file mode 100644 index 3d3ee729..00000000 --- a/appview/repo/archive.go +++ /dev/null @@ -1,49 +0,0 @@ -package repo - -import ( - "fmt" - "net/http" - "net/url" - "strings" - - "tangled.org/core/api/tangled" - xrpcclient "tangled.org/core/appview/xrpcclient" - - indigoxrpc "github.com/bluesky-social/indigo/xrpc" - "github.com/go-chi/chi/v5" - "github.com/go-git/go-git/v5/plumbing" -) - -func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { - l := rp.logger.With("handler", "DownloadArchive") - ref := chi.URLParam(r, "ref") - ref, _ = url.PathUnescape(ref) - f, err := rp.repoResolver.Resolve(r) - if err != nil { - l.Error("failed to get repo and knot", "err", err) - return - } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - repo := fmt.Sprintf("%s/%s", f.OwnerDid(), f.Name) - archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, repo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.archive", "err", xrpcerr) - rp.pages.Error503(w) - return - } - // Set headers for file download, just pass along whatever the knot specifies - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) - // Write the archive data directly - w.Write(archiveBytes) -} diff --git a/appview/repo/router.go b/appview/repo/router.go index b5def882..4dd15534 100644 --- a/appview/repo/router.go +++ b/appview/repo/router.go @@ -40,10 +40,6 @@ func (rp *Repo) Router(mw *middleware.Middleware) http.Handler { r.Get("/blob/{ref}/*", rp.Blob) r.Get("/raw/{ref}/*", rp.RepoBlobRaw) - // intentionally doesn't use /* as this isn't - // a file path - r.Get("/archive/{ref}", rp.DownloadArchive) - r.Route("/fork", func(r chi.Router) { r.Use(middleware.AuthMiddleware(rp.oauth)) r.Get("/", rp.ForkRepo) diff --git a/appview/state/git_http.go b/appview/state/proxy_knot.go similarity index 82% rename from appview/state/git_http.go rename to appview/state/proxy_knot.go index 36ce3e8d..d1788ab5 100644 --- a/appview/state/git_http.go +++ b/appview/state/proxy_knot.go @@ -59,6 +59,25 @@ func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) { s.proxyRequest(w, r, targetURL) } +func (s *State) DownloadArchive(w http.ResponseWriter, r *http.Request) { + ref := chi.URLParam(r, "ref") + + user, ok := r.Context().Value("resolvedId").(identity.Identity) + if !ok { + http.Error(w, "failed to resolve user", http.StatusInternalServerError) + return + } + repo := r.Context().Value("repo").(*models.Repo) + + scheme := "https" + if s.config.Core.Dev { + scheme = "http" + } + + targetURL := fmt.Sprintf("%s://%s/%s/%s/archive/%s", scheme, repo.Knot, user.DID, repo.Name, ref) + s.proxyRequest(w, r, targetURL) +} + func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) { client := &http.Client{} diff --git a/appview/state/router.go b/appview/state/router.go index 84f47217..d943cd6d 100644 --- a/appview/state/router.go +++ b/appview/state/router.go @@ -100,7 +100,9 @@ func (s *State) UserRouter(mw *middleware.Middleware) http.Handler { r.Get("/info/refs", s.InfoRefs) r.Post("/git-upload-pack", s.UploadPack) r.Post("/git-receive-pack", s.ReceivePack) - + // intentionally doesn't use /* as this isn't + // a file path + r.Get("/archive/{ref}", s.DownloadArchive) }) }) diff --git a/knotserver/router.go b/knotserver/router.go index 482f932d..2aef216d 100644 --- a/knotserver/router.go +++ b/knotserver/router.go @@ -84,6 +84,8 @@ func (h *Knot) Router() http.Handler { r.Get("/info/refs", h.InfoRefs) r.Post("/git-upload-pack", h.UploadPack) r.Post("/git-receive-pack", h.ReceivePack) + // convenience routes + r.Get("/archive/{ref}", h.Archive) }) }) -- 2.43.0 From 89c69cc1ff6f77f43369fac5a73f927217abb770 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Sat, 18 Oct 2025 14:45:24 +0900 Subject: [PATCH] knotserver: support immutable nix flakeref link header Change-Id: ptrrwwvnkmxqwyvqlklqlmwpzwnkmuqx Close: #231 Signed-off-by: Seongmin Lee --- knotserver/archive.go | 69 +++++++++++++++++++++++++++++++++++++++++++ knotserver/git/git.go | 4 +++ 2 files changed, 73 insertions(+) create mode 100644 knotserver/archive.go diff --git a/knotserver/archive.go b/knotserver/archive.go new file mode 100644 index 00000000..9a5f48b1 --- /dev/null +++ b/knotserver/archive.go @@ -0,0 +1,69 @@ +package knotserver + +import ( + "compress/gzip" + "fmt" + "net/http" + "strings" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/knotserver/git" +) + +func (h *Knot) Archive(w http.ResponseWriter, r *http.Request) { + var ( + did = chi.URLParam(r, "did") + name = chi.URLParam(r, "name") + ref = chi.URLParam(r, "ref") + ) + repo, err := securejoin.SecureJoin(did, name) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, repo) + if err != nil { + gitError(w, "repository not found", http.StatusNotFound) + h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err) + return + } + + gr, err := git.Open(repoPath, ref) + + immutableLink := fmt.Sprintf( + "https://%s/%s/%s/archive/%s", + h.c.Server.Hostname, + did, + name, + gr.Hash(), + ) + + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + filename := fmt.Sprintf("%s-%s.tar.gz", name, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) + + gw := gzip.NewWriter(w) + defer gw.Close() + + err = gr.WriteTar(gw, "") + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("writing tar file", "error", err) + return + } + + err = gw.Flush() + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + h.l.Error("flushing", "error", err.Error()) + return + } +} diff --git a/knotserver/git/git.go b/knotserver/git/git.go index 2fe31c70..977d462a 100644 --- a/knotserver/git/git.go +++ b/knotserver/git/git.go @@ -71,6 +71,10 @@ func PlainOpen(path string) (*GitRepo, error) { return &g, nil } +func (g *GitRepo) Hash() plumbing.Hash { + return g.h +} + // re-open a repository and update references func (g *GitRepo) Refresh() error { refreshed, err := PlainOpen(g.path) -- 2.43.0 From 1a5864d9cef637bcfabe4d20031dee63227c1f2b Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 12 Nov 2025 15:54:15 +0900 Subject: [PATCH] lexicons,knotserver: remove `sh.tangled.repo.archive` Change-Id: xuplvooxylpkzuvqzoryvlpxuzxpvmpn Xrpc is not a good way to implement this as Xrpc clients usually strips out the response http headers. The archive should be implemented in raw http endpoint instead like git https endpoints Signed-off-by: Seongmin Lee --- api/tangled/repoarchive.go | 41 ----------------- knotserver/xrpc/repo_archive.go | 81 --------------------------------- knotserver/xrpc/xrpc.go | 1 - lexicons/repo/archive.json | 55 ---------------------- 4 files changed, 178 deletions(-) delete mode 100644 api/tangled/repoarchive.go delete mode 100644 knotserver/xrpc/repo_archive.go delete mode 100644 lexicons/repo/archive.json diff --git a/api/tangled/repoarchive.go b/api/tangled/repoarchive.go deleted file mode 100644 index cb9f2cb2..00000000 --- a/api/tangled/repoarchive.go +++ /dev/null @@ -1,41 +0,0 @@ -// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. - -package tangled - -// schema: sh.tangled.repo.archive - -import ( - "bytes" - "context" - - "github.com/bluesky-social/indigo/lex/util" -) - -const ( - RepoArchiveNSID = "sh.tangled.repo.archive" -) - -// RepoArchive calls the XRPC method "sh.tangled.repo.archive". -// -// format: Archive format -// prefix: Prefix for files in the archive -// ref: Git reference (branch, tag, or commit SHA) -// repo: Repository identifier in format 'did:plc:.../repoName' -func RepoArchive(ctx context.Context, c util.LexClient, format string, prefix string, ref string, repo string) ([]byte, error) { - buf := new(bytes.Buffer) - - params := map[string]interface{}{} - if format != "" { - params["format"] = format - } - if prefix != "" { - params["prefix"] = prefix - } - params["ref"] = ref - params["repo"] = repo - if err := c.LexDo(ctx, util.Query, "", "sh.tangled.repo.archive", params, nil, buf); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} diff --git a/knotserver/xrpc/repo_archive.go b/knotserver/xrpc/repo_archive.go deleted file mode 100644 index 42fe498e..00000000 --- a/knotserver/xrpc/repo_archive.go +++ /dev/null @@ -1,81 +0,0 @@ -package xrpc - -import ( - "compress/gzip" - "fmt" - "net/http" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - - "tangled.org/core/knotserver/git" - xrpcerr "tangled.org/core/xrpc/errors" -) - -func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) { - repo := r.URL.Query().Get("repo") - repoPath, err := x.parseRepoParam(repo) - if err != nil { - writeError(w, err.(xrpcerr.XrpcError), http.StatusBadRequest) - return - } - - ref := r.URL.Query().Get("ref") - // ref can be empty (git.Open handles this) - - format := r.URL.Query().Get("format") - if format == "" { - format = "tar.gz" // default - } - - prefix := r.URL.Query().Get("prefix") - - if format != "tar.gz" { - writeError(w, xrpcerr.NewXrpcError( - xrpcerr.WithTag("InvalidRequest"), - xrpcerr.WithMessage("only tar.gz format is supported"), - ), http.StatusBadRequest) - return - } - - gr, err := git.Open(repoPath, ref) - if err != nil { - writeError(w, xrpcerr.RefNotFoundError, http.StatusNotFound) - return - } - - repoParts := strings.Split(repo, "/") - repoName := repoParts[len(repoParts)-1] - - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - - var archivePrefix string - if prefix != "" { - archivePrefix = prefix - } else { - archivePrefix = fmt.Sprintf("%s-%s", repoName, safeRefFilename) - } - - filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - - gw := gzip.NewWriter(w) - defer gw.Close() - - err = gr.WriteTar(gw, archivePrefix) - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("writing tar file", "error", err.Error()) - return - } - - err = gw.Flush() - if err != nil { - // once we start writing to the body we can't report error anymore - // so we are only left with logging the error - x.Logger.Error("flushing", "error", err.Error()) - return - } -} diff --git a/knotserver/xrpc/xrpc.go b/knotserver/xrpc/xrpc.go index c110356f..50615e80 100644 --- a/knotserver/xrpc/xrpc.go +++ b/knotserver/xrpc/xrpc.go @@ -64,7 +64,6 @@ func (x *Xrpc) Router() http.Handler { r.Get("/"+tangled.RepoCompareNSID, x.RepoCompare) r.Get("/"+tangled.RepoGetDefaultBranchNSID, x.RepoGetDefaultBranch) r.Get("/"+tangled.RepoBranchNSID, x.RepoBranch) - r.Get("/"+tangled.RepoArchiveNSID, x.RepoArchive) r.Get("/"+tangled.RepoLanguagesNSID, x.RepoLanguages) // knot query endpoints (no auth required) diff --git a/lexicons/repo/archive.json b/lexicons/repo/archive.json deleted file mode 100644 index 05e313d1..00000000 --- a/lexicons/repo/archive.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "lexicon": 1, - "id": "sh.tangled.repo.archive", - "defs": { - "main": { - "type": "query", - "parameters": { - "type": "params", - "required": ["repo", "ref"], - "properties": { - "repo": { - "type": "string", - "description": "Repository identifier in format 'did:plc:.../repoName'" - }, - "ref": { - "type": "string", - "description": "Git reference (branch, tag, or commit SHA)" - }, - "format": { - "type": "string", - "description": "Archive format", - "enum": ["tar", "zip", "tar.gz", "tar.bz2", "tar.xz"], - "default": "tar.gz" - }, - "prefix": { - "type": "string", - "description": "Prefix for files in the archive" - } - } - }, - "output": { - "encoding": "*/*", - "description": "Binary archive data" - }, - "errors": [ - { - "name": "RepoNotFound", - "description": "Repository not found or access denied" - }, - { - "name": "RefNotFound", - "description": "Git reference not found" - }, - { - "name": "InvalidRequest", - "description": "Invalid request parameters" - }, - { - "name": "ArchiveError", - "description": "Failed to create archive" - } - ] - } - } -} -- 2.43.0