From f1ead503c09713c41ce8da7a5504f017bf14929e Mon Sep 17 00:00:00 2001 From: Evan Jarrett Date: Mon, 10 Nov 2025 20:04:24 -0600 Subject: [PATCH] spindle: move the clone step out of nixery into a shared function for all spindle engines Signed-off-by: Evan Jarrett --- spindle/engines/nixery/setup_steps.go | 77 ++--- spindle/workflow/clone.go | 144 +++++++++ spindle/workflow/clone_test.go | 420 ++++++++++++++++++++++++++ 3 files changed, 583 insertions(+), 58 deletions(-) create mode 100644 spindle/workflow/clone.go create mode 100644 spindle/workflow/clone_test.go diff --git a/spindle/engines/nixery/setup_steps.go b/spindle/engines/nixery/setup_steps.go index 03269488..9399b6e5 100644 --- a/spindle/engines/nixery/setup_steps.go +++ b/spindle/engines/nixery/setup_steps.go @@ -2,11 +2,10 @@ package nixery import ( "fmt" - "path" "strings" "tangled.org/core/api/tangled" - "tangled.org/core/workflow" + "tangled.org/core/spindle/workflow" ) func nixConfStep() Step { @@ -19,73 +18,35 @@ echo 'build-users-group = ' >> /etc/nix/nix.conf` } } -// cloneOptsAsSteps processes clone options and adds corresponding steps -// to the beginning of the workflow's step list if cloning is not skipped. -// -// the steps to do here are: +// cloneStep uses the shared clone step builder to generate git clone commands. +// The shared builder handles: // - git init // - git remote add origin // - git fetch --depth= --recurse-submodules= // - git checkout FETCH_HEAD +// And supports all trigger types (push, PR, manual) and clone options. func cloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) Step { - if twf.Clone.Skip { - return Step{} - } - - var commands []string - - // initialize git repo in workspace - commands = append(commands, "git init") - - // add repo as git remote - scheme := "https://" - if dev { - scheme = "http://" - tr.Repo.Knot = strings.ReplaceAll(tr.Repo.Knot, "localhost", "host.docker.internal") - } - url := scheme + path.Join(tr.Repo.Knot, tr.Repo.Did, tr.Repo.Repo) - commands = append(commands, fmt.Sprintf("git remote add origin %s", url)) - - // run git fetch - { - var fetchArgs []string - - // default clone depth is 1 - depth := 1 - if twf.Clone.Depth > 1 { - depth = int(twf.Clone.Depth) + info, err := workflow.GetCloneInfo(workflow.CloneOptions{ + Workflow: twf, + TriggerMetadata: tr, + DevMode: dev, + WorkspaceDir: workspaceDir, + }) + if err != nil { + return Step{ + command: fmt.Sprintf("echo 'Failed to get clone info: %s' && exit 1", err.Error()), + name: "Clone repository into workspace (error)", } - fetchArgs = append(fetchArgs, fmt.Sprintf("--depth=%d", depth)) - - // optionally recurse submodules - if twf.Clone.Submodules { - fetchArgs = append(fetchArgs, "--recurse-submodules=yes") - } - - // set remote to fetch from - fetchArgs = append(fetchArgs, "origin") - - // set revision to checkout - switch workflow.TriggerKind(tr.Kind) { - case workflow.TriggerKindManual: - // TODO: unimplemented - case workflow.TriggerKindPush: - fetchArgs = append(fetchArgs, tr.Push.NewSha) - case workflow.TriggerKindPullRequest: - fetchArgs = append(fetchArgs, tr.PullRequest.SourceSha) - } - - commands = append(commands, fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " "))) } - // run git checkout - commands = append(commands, "git checkout FETCH_HEAD") + if info.Skip { + return Step{} + } - cloneStep := Step{ - command: strings.Join(commands, "\n"), + return Step{ + command: strings.Join(info.Commands, "\n"), name: "Clone repository into workspace", } - return cloneStep } // dependencyStep processes dependencies defined in the workflow. diff --git a/spindle/workflow/clone.go b/spindle/workflow/clone.go new file mode 100644 index 00000000..d9bb9ee5 --- /dev/null +++ b/spindle/workflow/clone.go @@ -0,0 +1,144 @@ +package workflow + +import ( + "fmt" + "strings" + + "tangled.org/core/api/tangled" + "tangled.org/core/workflow" +) + +type CloneOptions struct { + Workflow tangled.Pipeline_Workflow + TriggerMetadata tangled.Pipeline_TriggerMetadata + DevMode bool + WorkspaceDir string +} + +type CloneInfo struct { + Commands []string + RepoURL string + CommitSHA string + Skip bool +} + +// GetCloneInfo generates git clone commands and metadata from pipeline trigger metadata +func GetCloneInfo(opts CloneOptions) (*CloneInfo, error) { + if opts.Workflow.Clone != nil && opts.Workflow.Clone.Skip { + return &CloneInfo{Skip: true}, nil + } + + commitSHA, err := extractCommitSHA(opts.TriggerMetadata) + if err != nil { + return nil, fmt.Errorf("failed to extract commit SHA: %w", err) + } + + repoURL := buildRepoURL(opts.TriggerMetadata, opts.DevMode) + + workspaceDir := opts.WorkspaceDir + if workspaceDir == "" { + workspaceDir = "/tangled/workspace" + } + + initCmd := fmt.Sprintf("git init %s", workspaceDir) + remoteCmd := fmt.Sprintf("git remote add origin %s", repoURL) + + var cloneOpts tangled.Pipeline_CloneOpts + if opts.Workflow.Clone != nil { + cloneOpts = *opts.Workflow.Clone + } + fetchArgs := buildFetchArgs(cloneOpts, commitSHA) + fetchCmd := fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")) + + checkoutCmd := "git checkout FETCH_HEAD" + + commands := []string{ + initCmd, + fmt.Sprintf("cd %s", workspaceDir), + remoteCmd, + fetchCmd, + checkoutCmd, + } + + return &CloneInfo{ + Commands: commands, + RepoURL: repoURL, + CommitSHA: commitSHA, + Skip: false, + }, nil +} + +// extractCommitSHA extracts the commit SHA from trigger metadata based on trigger type +func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { + switch workflow.TriggerKind(tr.Kind) { + case workflow.TriggerKindPush: + if tr.Push == nil { + return "", fmt.Errorf("push trigger metadata is nil") + } + return tr.Push.NewSha, nil + + case workflow.TriggerKindPullRequest: + if tr.PullRequest == nil { + return "", fmt.Errorf("pull request trigger metadata is nil") + } + return tr.PullRequest.SourceSha, nil + + case workflow.TriggerKindManual: + // Manual triggers don't have an explicit SHA in the metadata + // For now, return empty string - could be enhanced to fetch from default branch + // TODO: Implement manual trigger SHA resolution (fetch default branch HEAD) + return "", nil + + default: + return "", fmt.Errorf("unknown trigger kind: %s", tr.Kind) + } +} + +// buildRepoURL constructs the repository URL from trigger metadata +func buildRepoURL(tr tangled.Pipeline_TriggerMetadata, devMode bool) string { + if tr.Repo == nil { + return "" + } + + // Determine protocol + scheme := "https://" + if devMode { + scheme = "http://" + } + + // Get host from knot + host := tr.Repo.Knot + + // In dev mode, replace localhost with host.docker.internal for Docker networking + if devMode && strings.Contains(host, "localhost") { + host = strings.ReplaceAll(host, "localhost", "host.docker.internal") + } + + // Build URL: {scheme}{knot}/{did}/{repo} + return fmt.Sprintf("%s%s/%s/%s", scheme, host, tr.Repo.Did, tr.Repo.Repo) +} + +// buildFetchArgs constructs the arguments for git fetch based on clone options +func buildFetchArgs(clone tangled.Pipeline_CloneOpts, sha string) []string { + args := []string{} + + // Set fetch depth (default to 1 for shallow clone) + depth := clone.Depth + if depth == 0 { + depth = 1 + } + args = append(args, fmt.Sprintf("--depth=%d", depth)) + + // Add submodules if requested + if clone.Submodules { + args = append(args, "--recurse-submodules=yes") + } + + // Add remote and SHA + args = append(args, "origin") + if sha != "" { + args = append(args, sha) + } + + return args +} diff --git a/spindle/workflow/clone_test.go b/spindle/workflow/clone_test.go new file mode 100644 index 00000000..69629965 --- /dev/null +++ b/spindle/workflow/clone_test.go @@ -0,0 +1,420 @@ +package workflow + +import ( + "strings" + "testing" + + "tangled.org/core/api/tangled" + "tangled.org/core/workflow" +) + +func TestGetCloneInfo_PushTrigger(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Submodules: false, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + OldSha: "def456", + Ref: "refs/heads/main", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + DevMode: false, + WorkspaceDir: "/tangled/workspace", + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + if info.Skip { + t.Error("Expected Skip to be false") + } + + if info.CommitSHA != "abc123" { + t.Errorf("Expected CommitSHA 'abc123', got '%s'", info.CommitSHA) + } + + expectedURL := "https://example.com/did:plc:user123/my-repo" + if info.RepoURL != expectedURL { + t.Errorf("Expected RepoURL '%s', got '%s'", expectedURL, info.RepoURL) + } + + if len(info.Commands) != 5 { + t.Errorf("Expected 5 commands, got %d", len(info.Commands)) + } + + // Verify commands contain expected git operations + allCmds := strings.Join(info.Commands, " ") + if !strings.Contains(allCmds, "git init") { + t.Error("Commands should contain 'git init'") + } + if !strings.Contains(allCmds, "git remote add origin") { + t.Error("Commands should contain 'git remote add origin'") + } + if !strings.Contains(allCmds, "git fetch") { + t.Error("Commands should contain 'git fetch'") + } + if !strings.Contains(allCmds, "abc123") { + t.Error("Commands should contain commit SHA") + } + if !strings.Contains(allCmds, "git checkout FETCH_HEAD") { + t.Error("Commands should contain 'git checkout FETCH_HEAD'") + } +} + +func TestGetCloneInfo_PullRequestTrigger(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPullRequest), + PullRequest: &tangled.Pipeline_PullRequestTriggerData{ + SourceSha: "pr-sha-789", + SourceBranch: "feature-branch", + TargetBranch: "main", + Action: "opened", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + DevMode: false, + WorkspaceDir: "/tangled/workspace", + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + if info.CommitSHA != "pr-sha-789" { + t.Errorf("Expected CommitSHA 'pr-sha-789', got '%s'", info.CommitSHA) + } + + allCmds := strings.Join(info.Commands, " ") + if !strings.Contains(allCmds, "pr-sha-789") { + t.Error("Commands should contain PR commit SHA") + } +} + +func TestGetCloneInfo_ManualTrigger(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindManual), + Manual: &tangled.Pipeline_ManualTriggerData{ + Inputs: nil, + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + DevMode: false, + WorkspaceDir: "/tangled/workspace", + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + // Manual triggers don't have a SHA yet (TODO) + if info.CommitSHA != "" { + t.Errorf("Expected empty CommitSHA for manual trigger, got '%s'", info.CommitSHA) + } +} + +func TestGetCloneInfo_SkipFlag(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Skip: true, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + if !info.Skip { + t.Error("Expected Skip to be true") + } + + if len(info.Commands) != 0 { + t.Errorf("Expected no commands when Skip is true, got %d commands", len(info.Commands)) + } +} + +func TestGetCloneInfo_DevMode(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "localhost:3000", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + DevMode: true, + WorkspaceDir: "/tangled/workspace", + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + // In dev mode, should use http:// and replace localhost with host.docker.internal + expectedURL := "http://host.docker.internal:3000/did:plc:user123/my-repo" + if info.RepoURL != expectedURL { + t.Errorf("Expected dev mode URL '%s', got '%s'", expectedURL, info.RepoURL) + } +} + +func TestGetCloneInfo_DepthAndSubmodules(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 10, + Submodules: true, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + DevMode: false, + WorkspaceDir: "/tangled/workspace", + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + allCmds := strings.Join(info.Commands, " ") + if !strings.Contains(allCmds, "--depth=10") { + t.Error("Commands should contain '--depth=10'") + } + + if !strings.Contains(allCmds, "--recurse-submodules=yes") { + t.Error("Commands should contain '--recurse-submodules=yes'") + } +} + +func TestGetCloneInfo_DefaultDepth(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 0, // Default should be 1 + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + WorkspaceDir: "/tangled/workspace", + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + allCmds := strings.Join(info.Commands, " ") + if !strings.Contains(allCmds, "--depth=1") { + t.Error("Commands should default to '--depth=1'") + } +} + +func TestGetCloneInfo_NilPushData(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: nil, // Nil push data should return error + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + WorkspaceDir: "/tangled/workspace", + } + + _, err := GetCloneInfo(cfg) + if err == nil { + t.Error("Expected error when push data is nil") + } + + if !strings.Contains(err.Error(), "push trigger metadata is nil") { + t.Errorf("Expected error about nil push metadata, got: %v", err) + } +} + +func TestGetCloneInfo_NilPRData(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPullRequest), + PullRequest: nil, // Nil PR data should return error + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + WorkspaceDir: "/tangled/workspace", + } + + _, err := GetCloneInfo(cfg) + if err == nil { + t.Error("Expected error when pull request data is nil") + } + + if !strings.Contains(err.Error(), "pull request trigger metadata is nil") { + t.Errorf("Expected error about nil PR metadata, got: %v", err) + } +} + +func TestGetCloneInfo_CustomWorkspace(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + DevMode: false, + WorkspaceDir: "/custom/path", + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + allCmds := strings.Join(info.Commands, " ") + if !strings.Contains(allCmds, "/custom/path") { + t.Error("Commands should use custom workspace directory") + } +} + +func TestGetCloneInfo_DefaultWorkspace(t *testing.T) { + cfg := CloneOptions{ + Workflow: tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + }, + TriggerMetadata: tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + }, + DevMode: false, + WorkspaceDir: "", // Empty should default to /tangled/workspace + } + + info, err := GetCloneInfo(cfg) + if err != nil { + t.Fatalf("GetCloneInfo failed: %v", err) + } + + allCmds := strings.Join(info.Commands, " ") + if !strings.Contains(allCmds, "/tangled/workspace") { + t.Error("Commands should default to /tangled/workspace") + } +} -- 2.43.0