spindle: move the clone step out of nixery into a shared package for all spindle engines #827

merged
opened by evan.jarrett.net targeting master from evan.jarrett.net/core: spindle-clone
Changed files
+583 -58
spindle
engines
workflow
+19 -58
spindle/engines/nixery/setup_steps.go
···
import (
"fmt"
-
"path"
"strings"
"tangled.org/core/api/tangled"
-
"tangled.org/core/workflow"
+
"tangled.org/core/spindle/workflow"
)
func nixConfStep() Step {
···
}
}
-
// cloneOptsAsSteps processes clone options and adds corresponding steps
-
// to the beginning of the workflow's step list if cloning is not skipped.
-
//
-
// the steps to do here are:
+
// cloneStep uses the shared clone step builder to generate git clone commands.
+
// The shared builder handles:
// - git init
// - git remote add origin <url>
// - git fetch --depth=<d> --recurse-submodules=<yes|no> <sha>
// - git checkout FETCH_HEAD
+
// And supports all trigger types (push, PR, manual) and clone options.
func cloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) Step {
-
if twf.Clone.Skip {
-
return Step{}
-
}
-
-
var commands []string
-
-
// initialize git repo in workspace
-
commands = append(commands, "git init")
-
-
// add repo as git remote
-
scheme := "https://"
-
if dev {
-
scheme = "http://"
-
tr.Repo.Knot = strings.ReplaceAll(tr.Repo.Knot, "localhost", "host.docker.internal")
-
}
-
url := scheme + path.Join(tr.Repo.Knot, tr.Repo.Did, tr.Repo.Repo)
-
commands = append(commands, fmt.Sprintf("git remote add origin %s", url))
-
-
// run git fetch
-
{
-
var fetchArgs []string
-
-
// default clone depth is 1
-
depth := 1
-
if twf.Clone.Depth > 1 {
-
depth = int(twf.Clone.Depth)
+
info, err := workflow.GetCloneInfo(workflow.CloneOptions{
+
Workflow: twf,
+
TriggerMetadata: tr,
+
DevMode: dev,
+
WorkspaceDir: workspaceDir,
+
})
+
if err != nil {
+
return Step{
+
command: fmt.Sprintf("echo 'Failed to get clone info: %s' && exit 1", err.Error()),
+
name: "Clone repository into workspace (error)",
}
-
fetchArgs = append(fetchArgs, fmt.Sprintf("--depth=%d", depth))
-
-
// optionally recurse submodules
-
if twf.Clone.Submodules {
-
fetchArgs = append(fetchArgs, "--recurse-submodules=yes")
-
}
-
-
// set remote to fetch from
-
fetchArgs = append(fetchArgs, "origin")
-
-
// set revision to checkout
-
switch workflow.TriggerKind(tr.Kind) {
-
case workflow.TriggerKindManual:
-
// TODO: unimplemented
-
case workflow.TriggerKindPush:
-
fetchArgs = append(fetchArgs, tr.Push.NewSha)
-
case workflow.TriggerKindPullRequest:
-
fetchArgs = append(fetchArgs, tr.PullRequest.SourceSha)
-
}
-
-
commands = append(commands, fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")))
}
-
// run git checkout
-
commands = append(commands, "git checkout FETCH_HEAD")
+
if info.Skip {
+
return Step{}
+
}
-
cloneStep := Step{
-
command: strings.Join(commands, "\n"),
+
return Step{
+
command: strings.Join(info.Commands, "\n"),
name: "Clone repository into workspace",
}
-
return cloneStep
}
// dependencyStep processes dependencies defined in the workflow.
+144
spindle/workflow/clone.go
···
+
package workflow
+
+
import (
+
"fmt"
+
"strings"
+
+
"tangled.org/core/api/tangled"
+
"tangled.org/core/workflow"
+
)
+
+
type CloneOptions struct {
+
Workflow tangled.Pipeline_Workflow
+
TriggerMetadata tangled.Pipeline_TriggerMetadata
+
DevMode bool
+
WorkspaceDir string
+
}
+
+
type CloneInfo struct {
+
Commands []string
+
RepoURL string
+
CommitSHA string
+
Skip bool
+
}
+
+
// GetCloneInfo generates git clone commands and metadata from pipeline trigger metadata
+
func GetCloneInfo(opts CloneOptions) (*CloneInfo, error) {
+
if opts.Workflow.Clone != nil && opts.Workflow.Clone.Skip {
+
return &CloneInfo{Skip: true}, nil
+
}
+
+
commitSHA, err := extractCommitSHA(opts.TriggerMetadata)
+
if err != nil {
+
return nil, fmt.Errorf("failed to extract commit SHA: %w", err)
+
}
+
+
repoURL := buildRepoURL(opts.TriggerMetadata, opts.DevMode)
+
+
workspaceDir := opts.WorkspaceDir
+
if workspaceDir == "" {
+
workspaceDir = "/tangled/workspace"
+
}
+
+
initCmd := fmt.Sprintf("git init %s", workspaceDir)
+
remoteCmd := fmt.Sprintf("git remote add origin %s", repoURL)
+
+
var cloneOpts tangled.Pipeline_CloneOpts
+
if opts.Workflow.Clone != nil {
+
cloneOpts = *opts.Workflow.Clone
+
}
+
fetchArgs := buildFetchArgs(cloneOpts, commitSHA)
+
fetchCmd := fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " "))
+
+
checkoutCmd := "git checkout FETCH_HEAD"
+
+
commands := []string{
+
initCmd,
+
fmt.Sprintf("cd %s", workspaceDir),
+
remoteCmd,
+
fetchCmd,
+
checkoutCmd,
+
}
+
+
return &CloneInfo{
+
Commands: commands,
+
RepoURL: repoURL,
+
CommitSHA: commitSHA,
+
Skip: false,
+
}, nil
+
}
+
+
// extractCommitSHA extracts the commit SHA from trigger metadata based on trigger type
+
func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) {
+
switch workflow.TriggerKind(tr.Kind) {
+
case workflow.TriggerKindPush:
+
if tr.Push == nil {
+
return "", fmt.Errorf("push trigger metadata is nil")
+
}
+
return tr.Push.NewSha, nil
+
+
case workflow.TriggerKindPullRequest:
+
if tr.PullRequest == nil {
+
return "", fmt.Errorf("pull request trigger metadata is nil")
+
}
+
return tr.PullRequest.SourceSha, nil
+
+
case workflow.TriggerKindManual:
+
// Manual triggers don't have an explicit SHA in the metadata
+
// For now, return empty string - could be enhanced to fetch from default branch
+
// TODO: Implement manual trigger SHA resolution (fetch default branch HEAD)
+
return "", nil
+
+
default:
+
return "", fmt.Errorf("unknown trigger kind: %s", tr.Kind)
+
}
+
}
+
+
// buildRepoURL constructs the repository URL from trigger metadata
+
func buildRepoURL(tr tangled.Pipeline_TriggerMetadata, devMode bool) string {
+
if tr.Repo == nil {
+
return ""
+
}
+
+
// Determine protocol
+
scheme := "https://"
+
if devMode {
+
scheme = "http://"
+
}
+
+
// Get host from knot
+
host := tr.Repo.Knot
+
+
// In dev mode, replace localhost with host.docker.internal for Docker networking
+
if devMode && strings.Contains(host, "localhost") {
+
host = strings.ReplaceAll(host, "localhost", "host.docker.internal")
+
}
+
+
// Build URL: {scheme}{knot}/{did}/{repo}
+
return fmt.Sprintf("%s%s/%s/%s", scheme, host, tr.Repo.Did, tr.Repo.Repo)
+
}
+
+
// buildFetchArgs constructs the arguments for git fetch based on clone options
+
func buildFetchArgs(clone tangled.Pipeline_CloneOpts, sha string) []string {
+
args := []string{}
+
+
// Set fetch depth (default to 1 for shallow clone)
+
depth := clone.Depth
+
if depth == 0 {
+
depth = 1
+
}
+
args = append(args, fmt.Sprintf("--depth=%d", depth))
+
+
// Add submodules if requested
+
if clone.Submodules {
+
args = append(args, "--recurse-submodules=yes")
+
}
+
+
// Add remote and SHA
+
args = append(args, "origin")
+
if sha != "" {
+
args = append(args, sha)
+
}
+
+
return args
+
}
+420
spindle/workflow/clone_test.go
···
+
package workflow
+
+
import (
+
"strings"
+
"testing"
+
+
"tangled.org/core/api/tangled"
+
"tangled.org/core/workflow"
+
)
+
+
func TestGetCloneInfo_PushTrigger(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Submodules: false,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
OldSha: "def456",
+
Ref: "refs/heads/main",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
DevMode: false,
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
if info.Skip {
+
t.Error("Expected Skip to be false")
+
}
+
+
if info.CommitSHA != "abc123" {
+
t.Errorf("Expected CommitSHA 'abc123', got '%s'", info.CommitSHA)
+
}
+
+
expectedURL := "https://example.com/did:plc:user123/my-repo"
+
if info.RepoURL != expectedURL {
+
t.Errorf("Expected RepoURL '%s', got '%s'", expectedURL, info.RepoURL)
+
}
+
+
if len(info.Commands) != 5 {
+
t.Errorf("Expected 5 commands, got %d", len(info.Commands))
+
}
+
+
// Verify commands contain expected git operations
+
allCmds := strings.Join(info.Commands, " ")
+
if !strings.Contains(allCmds, "git init") {
+
t.Error("Commands should contain 'git init'")
+
}
+
if !strings.Contains(allCmds, "git remote add origin") {
+
t.Error("Commands should contain 'git remote add origin'")
+
}
+
if !strings.Contains(allCmds, "git fetch") {
+
t.Error("Commands should contain 'git fetch'")
+
}
+
if !strings.Contains(allCmds, "abc123") {
+
t.Error("Commands should contain commit SHA")
+
}
+
if !strings.Contains(allCmds, "git checkout FETCH_HEAD") {
+
t.Error("Commands should contain 'git checkout FETCH_HEAD'")
+
}
+
}
+
+
func TestGetCloneInfo_PullRequestTrigger(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPullRequest),
+
PullRequest: &tangled.Pipeline_PullRequestTriggerData{
+
SourceSha: "pr-sha-789",
+
SourceBranch: "feature-branch",
+
TargetBranch: "main",
+
Action: "opened",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
DevMode: false,
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
if info.CommitSHA != "pr-sha-789" {
+
t.Errorf("Expected CommitSHA 'pr-sha-789', got '%s'", info.CommitSHA)
+
}
+
+
allCmds := strings.Join(info.Commands, " ")
+
if !strings.Contains(allCmds, "pr-sha-789") {
+
t.Error("Commands should contain PR commit SHA")
+
}
+
}
+
+
func TestGetCloneInfo_ManualTrigger(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindManual),
+
Manual: &tangled.Pipeline_ManualTriggerData{
+
Inputs: nil,
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
DevMode: false,
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
// Manual triggers don't have a SHA yet (TODO)
+
if info.CommitSHA != "" {
+
t.Errorf("Expected empty CommitSHA for manual trigger, got '%s'", info.CommitSHA)
+
}
+
}
+
+
func TestGetCloneInfo_SkipFlag(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Skip: true,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
if !info.Skip {
+
t.Error("Expected Skip to be true")
+
}
+
+
if len(info.Commands) != 0 {
+
t.Errorf("Expected no commands when Skip is true, got %d commands", len(info.Commands))
+
}
+
}
+
+
func TestGetCloneInfo_DevMode(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "localhost:3000",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
DevMode: true,
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
// In dev mode, should use http:// and replace localhost with host.docker.internal
+
expectedURL := "http://host.docker.internal:3000/did:plc:user123/my-repo"
+
if info.RepoURL != expectedURL {
+
t.Errorf("Expected dev mode URL '%s', got '%s'", expectedURL, info.RepoURL)
+
}
+
}
+
+
func TestGetCloneInfo_DepthAndSubmodules(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 10,
+
Submodules: true,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
DevMode: false,
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
allCmds := strings.Join(info.Commands, " ")
+
if !strings.Contains(allCmds, "--depth=10") {
+
t.Error("Commands should contain '--depth=10'")
+
}
+
+
if !strings.Contains(allCmds, "--recurse-submodules=yes") {
+
t.Error("Commands should contain '--recurse-submodules=yes'")
+
}
+
}
+
+
func TestGetCloneInfo_DefaultDepth(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 0, // Default should be 1
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
allCmds := strings.Join(info.Commands, " ")
+
if !strings.Contains(allCmds, "--depth=1") {
+
t.Error("Commands should default to '--depth=1'")
+
}
+
}
+
+
func TestGetCloneInfo_NilPushData(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: nil, // Nil push data should return error
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
_, err := GetCloneInfo(cfg)
+
if err == nil {
+
t.Error("Expected error when push data is nil")
+
}
+
+
if !strings.Contains(err.Error(), "push trigger metadata is nil") {
+
t.Errorf("Expected error about nil push metadata, got: %v", err)
+
}
+
}
+
+
func TestGetCloneInfo_NilPRData(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPullRequest),
+
PullRequest: nil, // Nil PR data should return error
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
WorkspaceDir: "/tangled/workspace",
+
}
+
+
_, err := GetCloneInfo(cfg)
+
if err == nil {
+
t.Error("Expected error when pull request data is nil")
+
}
+
+
if !strings.Contains(err.Error(), "pull request trigger metadata is nil") {
+
t.Errorf("Expected error about nil PR metadata, got: %v", err)
+
}
+
}
+
+
func TestGetCloneInfo_CustomWorkspace(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
DevMode: false,
+
WorkspaceDir: "/custom/path",
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
allCmds := strings.Join(info.Commands, " ")
+
if !strings.Contains(allCmds, "/custom/path") {
+
t.Error("Commands should use custom workspace directory")
+
}
+
}
+
+
func TestGetCloneInfo_DefaultWorkspace(t *testing.T) {
+
cfg := CloneOptions{
+
Workflow: tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
},
+
TriggerMetadata: tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
},
+
DevMode: false,
+
WorkspaceDir: "", // Empty should default to /tangled/workspace
+
}
+
+
info, err := GetCloneInfo(cfg)
+
if err != nil {
+
t.Fatalf("GetCloneInfo failed: %v", err)
+
}
+
+
allCmds := strings.Join(info.Commands, " ")
+
if !strings.Contains(allCmds, "/tangled/workspace") {
+
t.Error("Commands should default to /tangled/workspace")
+
}
+
}