From 3cdf71e7987fbd41217f354424056e2d8e546bf8 Mon Sep 17 00:00:00 2001 From: Evan Jarrett Date: Mon, 10 Nov 2025 20:04:24 -0600 Subject: [PATCH] spindle: implement shared clone step outside of nixery engine. Signed-off-by: Evan Jarrett --- spindle/engines/nixery/engine.go | 2 +- spindle/engines/nixery/setup_steps.go | 73 ----- spindle/models/clone.go | 151 +++++++++++ spindle/models/clone_test.go | 371 ++++++++++++++++++++++++++ 4 files changed, 523 insertions(+), 74 deletions(-) create mode 100644 spindle/models/clone.go create mode 100644 spindle/models/clone_test.go diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go index 64f0b5f7..86a49d47 100644 --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -109,7 +109,7 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin setup := &setupSteps{} setup.addStep(nixConfStep()) - setup.addStep(cloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev)) + setup.addStep(models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev)) // this step could be empty if s := dependencyStep(dwf.Dependencies); s != nil { setup.addStep(*s) diff --git a/spindle/engines/nixery/setup_steps.go b/spindle/engines/nixery/setup_steps.go index 03269488..b22dd687 100644 --- a/spindle/engines/nixery/setup_steps.go +++ b/spindle/engines/nixery/setup_steps.go @@ -2,11 +2,7 @@ package nixery import ( "fmt" - "path" "strings" - - "tangled.org/core/api/tangled" - "tangled.org/core/workflow" ) func nixConfStep() Step { @@ -19,75 +15,6 @@ echo 'build-users-group = ' >> /etc/nix/nix.conf` } } -// cloneOptsAsSteps processes clone options and adds corresponding steps -// to the beginning of the workflow's step list if cloning is not skipped. -// -// the steps to do here are: -// - git init -// - git remote add origin -// - git fetch --depth= --recurse-submodules= -// - git checkout FETCH_HEAD -func cloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) Step { - if twf.Clone.Skip { - return Step{} - } - - var commands []string - - // initialize git repo in workspace - commands = append(commands, "git init") - - // add repo as git remote - scheme := "https://" - if dev { - scheme = "http://" - tr.Repo.Knot = strings.ReplaceAll(tr.Repo.Knot, "localhost", "host.docker.internal") - } - url := scheme + path.Join(tr.Repo.Knot, tr.Repo.Did, tr.Repo.Repo) - commands = append(commands, fmt.Sprintf("git remote add origin %s", url)) - - // run git fetch - { - var fetchArgs []string - - // default clone depth is 1 - depth := 1 - if twf.Clone.Depth > 1 { - depth = int(twf.Clone.Depth) - } - fetchArgs = append(fetchArgs, fmt.Sprintf("--depth=%d", depth)) - - // optionally recurse submodules - if twf.Clone.Submodules { - fetchArgs = append(fetchArgs, "--recurse-submodules=yes") - } - - // set remote to fetch from - fetchArgs = append(fetchArgs, "origin") - - // set revision to checkout - switch workflow.TriggerKind(tr.Kind) { - case workflow.TriggerKindManual: - // TODO: unimplemented - case workflow.TriggerKindPush: - fetchArgs = append(fetchArgs, tr.Push.NewSha) - case workflow.TriggerKindPullRequest: - fetchArgs = append(fetchArgs, tr.PullRequest.SourceSha) - } - - commands = append(commands, fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " "))) - } - - // run git checkout - commands = append(commands, "git checkout FETCH_HEAD") - - cloneStep := Step{ - command: strings.Join(commands, "\n"), - name: "Clone repository into workspace", - } - return cloneStep -} - // dependencyStep processes dependencies defined in the workflow. // For dependencies using a custom registry (i.e. not nixpkgs), it collects // all packages and adds a single 'nix profile install' step to the diff --git a/spindle/models/clone.go b/spindle/models/clone.go new file mode 100644 index 00000000..c4b99826 --- /dev/null +++ b/spindle/models/clone.go @@ -0,0 +1,151 @@ +package models + +import ( + "fmt" + "strings" + + "tangled.org/core/api/tangled" + "tangled.org/core/workflow" +) + +type CloneStep struct { + name string + kind StepKind + commands []string +} + +func (s CloneStep) Name() string { + return s.name +} + +func (s CloneStep) Commands() []string { + return s.commands +} + +func (s CloneStep) Command() string { + return strings.Join(s.commands, "\n") +} + +func (s CloneStep) Kind() StepKind { + return s.kind +} + +// BuildCloneStep generates git clone commands. +// The caller must ensure the current working directory is set to the desired +// workspace directory before executing these commands. +// +// The generated commands are: +// - git init +// - git remote add origin +// - git fetch --depth= --recurse-submodules= +// - git checkout FETCH_HEAD +// +// Supports all trigger types (push, PR, manual) and clone options. +func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) CloneStep { + if twf.Clone != nil && twf.Clone.Skip { + return CloneStep{} + } + + commitSHA, err := extractCommitSHA(tr) + if err != nil { + return CloneStep{ + kind: StepKindSystem, + name: "Clone repository into workspace (error)", + commands: []string{fmt.Sprintf("echo 'Failed to get clone info: %s' && exit 1", err.Error())}, + } + } + + repoURL := buildRepoURL(tr, dev) + + var cloneOpts tangled.Pipeline_CloneOpts + if twf.Clone != nil { + cloneOpts = *twf.Clone + } + fetchArgs := buildFetchArgs(cloneOpts, commitSHA) + + return CloneStep{ + kind: StepKindSystem, + name: "Clone repository into workspace", + commands: []string{ + "git init", + fmt.Sprintf("git remote add origin %s", repoURL), + fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")), + "git checkout FETCH_HEAD", + }, + } +} + +// extractCommitSHA extracts the commit SHA from trigger metadata based on trigger type +func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { + switch workflow.TriggerKind(tr.Kind) { + case workflow.TriggerKindPush: + if tr.Push == nil { + return "", fmt.Errorf("push trigger metadata is nil") + } + return tr.Push.NewSha, nil + + case workflow.TriggerKindPullRequest: + if tr.PullRequest == nil { + return "", fmt.Errorf("pull request trigger metadata is nil") + } + return tr.PullRequest.SourceSha, nil + + case workflow.TriggerKindManual: + // Manual triggers don't have an explicit SHA in the metadata + // For now, return empty string - could be enhanced to fetch from default branch + // TODO: Implement manual trigger SHA resolution (fetch default branch HEAD) + return "", nil + + default: + return "", fmt.Errorf("unknown trigger kind: %s", tr.Kind) + } +} + +// buildRepoURL constructs the repository URL from trigger metadata +func buildRepoURL(tr tangled.Pipeline_TriggerMetadata, devMode bool) string { + if tr.Repo == nil { + return "" + } + + // Determine protocol + scheme := "https://" + if devMode { + scheme = "http://" + } + + // Get host from knot + host := tr.Repo.Knot + + // In dev mode, replace localhost with host.docker.internal for Docker networking + if devMode && strings.Contains(host, "localhost") { + host = strings.ReplaceAll(host, "localhost", "host.docker.internal") + } + + // Build URL: {scheme}{knot}/{did}/{repo} + return fmt.Sprintf("%s%s/%s/%s", scheme, host, tr.Repo.Did, tr.Repo.Repo) +} + +// buildFetchArgs constructs the arguments for git fetch based on clone options +func buildFetchArgs(clone tangled.Pipeline_CloneOpts, sha string) []string { + args := []string{} + + // Set fetch depth (default to 1 for shallow clone) + depth := clone.Depth + if depth == 0 { + depth = 1 + } + args = append(args, fmt.Sprintf("--depth=%d", depth)) + + // Add submodules if requested + if clone.Submodules { + args = append(args, "--recurse-submodules=yes") + } + + // Add remote and SHA + args = append(args, "origin") + if sha != "" { + args = append(args, sha) + } + + return args +} diff --git a/spindle/models/clone_test.go b/spindle/models/clone_test.go new file mode 100644 index 00000000..80700319 --- /dev/null +++ b/spindle/models/clone_test.go @@ -0,0 +1,371 @@ +package models + +import ( + "strings" + "testing" + + "tangled.org/core/api/tangled" + "tangled.org/core/workflow" +) + +func TestBuildCloneStep_PushTrigger(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Submodules: false, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + OldSha: "def456", + Ref: "refs/heads/main", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + if step.Kind() != StepKindSystem { + t.Errorf("Expected StepKindSystem, got %v", step.Kind()) + } + + if step.Name() != "Clone repository into workspace" { + t.Errorf("Expected 'Clone repository into workspace', got '%s'", step.Name()) + } + + commands := step.Commands() + if len(commands) != 4 { + t.Errorf("Expected 4 commands, got %d", len(commands)) + } + + // Verify commands contain expected git operations + allCmds := strings.Join(commands, " ") + if !strings.Contains(allCmds, "git init") { + t.Error("Commands should contain 'git init'") + } + if !strings.Contains(allCmds, "git remote add origin") { + t.Error("Commands should contain 'git remote add origin'") + } + if !strings.Contains(allCmds, "git fetch") { + t.Error("Commands should contain 'git fetch'") + } + if !strings.Contains(allCmds, "abc123") { + t.Error("Commands should contain commit SHA") + } + if !strings.Contains(allCmds, "git checkout FETCH_HEAD") { + t.Error("Commands should contain 'git checkout FETCH_HEAD'") + } + if !strings.Contains(allCmds, "https://example.com/did:plc:user123/my-repo") { + t.Error("Commands should contain expected repo URL") + } +} + +func TestBuildCloneStep_PullRequestTrigger(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPullRequest), + PullRequest: &tangled.Pipeline_PullRequestTriggerData{ + SourceSha: "pr-sha-789", + SourceBranch: "feature-branch", + TargetBranch: "main", + Action: "opened", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + allCmds := strings.Join(step.Commands(), " ") + if !strings.Contains(allCmds, "pr-sha-789") { + t.Error("Commands should contain PR commit SHA") + } +} + +func TestBuildCloneStep_ManualTrigger(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindManual), + Manual: &tangled.Pipeline_ManualTriggerData{ + Inputs: nil, + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + // Manual triggers don't have a SHA yet (TODO), so git fetch won't include a SHA + allCmds := strings.Join(step.Commands(), " ") + // Should still have basic git commands + if !strings.Contains(allCmds, "git init") { + t.Error("Commands should contain 'git init'") + } + if !strings.Contains(allCmds, "git fetch") { + t.Error("Commands should contain 'git fetch'") + } +} + +func TestBuildCloneStep_SkipFlag(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Skip: true, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + // Empty step when skip is true + if step.Name() != "" { + t.Error("Expected empty step name when Skip is true") + } + if len(step.Commands()) != 0 { + t.Errorf("Expected no commands when Skip is true, got %d commands", len(step.Commands())) + } +} + +func TestBuildCloneStep_DevMode(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "localhost:3000", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, true) + + // In dev mode, should use http:// and replace localhost with host.docker.internal + allCmds := strings.Join(step.Commands(), " ") + expectedURL := "http://host.docker.internal:3000/did:plc:user123/my-repo" + if !strings.Contains(allCmds, expectedURL) { + t.Errorf("Expected dev mode URL '%s' in commands", expectedURL) + } +} + +func TestBuildCloneStep_DepthAndSubmodules(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 10, + Submodules: true, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + allCmds := strings.Join(step.Commands(), " ") + if !strings.Contains(allCmds, "--depth=10") { + t.Error("Commands should contain '--depth=10'") + } + + if !strings.Contains(allCmds, "--recurse-submodules=yes") { + t.Error("Commands should contain '--recurse-submodules=yes'") + } +} + +func TestBuildCloneStep_DefaultDepth(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 0, // Default should be 1 + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + allCmds := strings.Join(step.Commands(), " ") + if !strings.Contains(allCmds, "--depth=1") { + t.Error("Commands should default to '--depth=1'") + } +} + +func TestBuildCloneStep_NilPushData(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: nil, // Nil push data should create error step + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + // Should return an error step + if !strings.Contains(step.Name(), "error") { + t.Error("Expected error in step name when push data is nil") + } + + allCmds := strings.Join(step.Commands(), " ") + if !strings.Contains(allCmds, "Failed to get clone info") { + t.Error("Commands should contain error message") + } + if !strings.Contains(allCmds, "exit 1") { + t.Error("Commands should exit with error") + } +} + +func TestBuildCloneStep_NilPRData(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPullRequest), + PullRequest: nil, // Nil PR data should create error step + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + // Should return an error step + if !strings.Contains(step.Name(), "error") { + t.Error("Expected error in step name when pull request data is nil") + } + + allCmds := strings.Join(step.Commands(), " ") + if !strings.Contains(allCmds, "Failed to get clone info") { + t.Error("Commands should contain error message") + } +} + +func TestBuildCloneStep_UnknownTriggerKind(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: &tangled.Pipeline_CloneOpts{ + Depth: 1, + Skip: false, + }, + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: "unknown_trigger", + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + // Should return an error step + if !strings.Contains(step.Name(), "error") { + t.Error("Expected error in step name for unknown trigger kind") + } + + allCmds := strings.Join(step.Commands(), " ") + if !strings.Contains(allCmds, "unknown trigger kind") { + t.Error("Commands should contain error message about unknown trigger kind") + } +} + +func TestBuildCloneStep_NilCloneOpts(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Clone: nil, // Nil clone options should use defaults + } + tr := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + + step := BuildCloneStep(twf, tr, false) + + // Should still work with default options + if step.Kind() != StepKindSystem { + t.Errorf("Expected StepKindSystem, got %v", step.Kind()) + } + + allCmds := strings.Join(step.Commands(), " ") + if !strings.Contains(allCmds, "--depth=1") { + t.Error("Commands should default to '--depth=1' when Clone is nil") + } + if !strings.Contains(allCmds, "git init") { + t.Error("Commands should contain 'git init'") + } +} -- 2.43.0