spindle: move the clone step out of nixery into a shared package for all spindle engines #827

merged
opened by evan.jarrett.net targeting master from evan.jarrett.net/core: spindle-clone
Changed files
+523 -74
spindle
+1 -1
spindle/engines/nixery/engine.go
···
setup := &setupSteps{}
setup.addStep(nixConfStep())
-
setup.addStep(cloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev))
+
setup.addStep(models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev))
// this step could be empty
if s := dependencyStep(dwf.Dependencies); s != nil {
setup.addStep(*s)
-73
spindle/engines/nixery/setup_steps.go
···
import (
"fmt"
-
"path"
"strings"
-
-
"tangled.org/core/api/tangled"
-
"tangled.org/core/workflow"
)
func nixConfStep() Step {
···
}
}
-
// cloneOptsAsSteps processes clone options and adds corresponding steps
-
// to the beginning of the workflow's step list if cloning is not skipped.
-
//
-
// the steps to do here are:
-
// - git init
-
// - git remote add origin <url>
-
// - git fetch --depth=<d> --recurse-submodules=<yes|no> <sha>
-
// - git checkout FETCH_HEAD
-
func cloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) Step {
-
if twf.Clone.Skip {
-
return Step{}
-
}
-
-
var commands []string
-
-
// initialize git repo in workspace
-
commands = append(commands, "git init")
-
-
// add repo as git remote
-
scheme := "https://"
-
if dev {
-
scheme = "http://"
-
tr.Repo.Knot = strings.ReplaceAll(tr.Repo.Knot, "localhost", "host.docker.internal")
-
}
-
url := scheme + path.Join(tr.Repo.Knot, tr.Repo.Did, tr.Repo.Repo)
-
commands = append(commands, fmt.Sprintf("git remote add origin %s", url))
-
-
// run git fetch
-
{
-
var fetchArgs []string
-
-
// default clone depth is 1
-
depth := 1
-
if twf.Clone.Depth > 1 {
-
depth = int(twf.Clone.Depth)
-
}
-
fetchArgs = append(fetchArgs, fmt.Sprintf("--depth=%d", depth))
-
-
// optionally recurse submodules
-
if twf.Clone.Submodules {
-
fetchArgs = append(fetchArgs, "--recurse-submodules=yes")
-
}
-
-
// set remote to fetch from
-
fetchArgs = append(fetchArgs, "origin")
-
-
// set revision to checkout
-
switch workflow.TriggerKind(tr.Kind) {
-
case workflow.TriggerKindManual:
-
// TODO: unimplemented
-
case workflow.TriggerKindPush:
-
fetchArgs = append(fetchArgs, tr.Push.NewSha)
-
case workflow.TriggerKindPullRequest:
-
fetchArgs = append(fetchArgs, tr.PullRequest.SourceSha)
-
}
-
-
commands = append(commands, fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")))
-
}
-
-
// run git checkout
-
commands = append(commands, "git checkout FETCH_HEAD")
-
-
cloneStep := Step{
-
command: strings.Join(commands, "\n"),
-
name: "Clone repository into workspace",
-
}
-
return cloneStep
-
}
-
// dependencyStep processes dependencies defined in the workflow.
// For dependencies using a custom registry (i.e. not nixpkgs), it collects
// all packages and adds a single 'nix profile install' step to the
+151
spindle/models/clone.go
···
+
package models
+
+
import (
+
"fmt"
+
"strings"
+
+
"tangled.org/core/api/tangled"
+
"tangled.org/core/workflow"
+
)
+
+
type CloneStep struct {
+
name string
+
kind StepKind
+
commands []string
+
}
+
+
func (s CloneStep) Name() string {
+
return s.name
+
}
+
+
func (s CloneStep) Commands() []string {
+
return s.commands
+
}
+
+
func (s CloneStep) Command() string {
+
return strings.Join(s.commands, "\n")
+
}
+
+
func (s CloneStep) Kind() StepKind {
+
return s.kind
+
}
+
+
// BuildCloneStep generates git clone commands.
+
// The caller must ensure the current working directory is set to the desired
+
// workspace directory before executing these commands.
+
//
+
// The generated commands are:
+
// - git init
+
// - git remote add origin <url>
+
// - git fetch --depth=<d> --recurse-submodules=<yes|no> <sha>
+
// - git checkout FETCH_HEAD
+
//
+
// Supports all trigger types (push, PR, manual) and clone options.
+
func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) CloneStep {
+
if twf.Clone != nil && twf.Clone.Skip {
+
return CloneStep{}
+
}
+
+
commitSHA, err := extractCommitSHA(tr)
+
if err != nil {
+
return CloneStep{
+
kind: StepKindSystem,
+
name: "Clone repository into workspace (error)",
+
commands: []string{fmt.Sprintf("echo 'Failed to get clone info: %s' && exit 1", err.Error())},
+
}
+
}
+
+
repoURL := buildRepoURL(tr, dev)
+
+
var cloneOpts tangled.Pipeline_CloneOpts
+
if twf.Clone != nil {
+
cloneOpts = *twf.Clone
+
}
+
fetchArgs := buildFetchArgs(cloneOpts, commitSHA)
+
+
return CloneStep{
+
kind: StepKindSystem,
+
name: "Clone repository into workspace",
+
commands: []string{
+
"git init",
+
fmt.Sprintf("git remote add origin %s", repoURL),
+
fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")),
+
"git checkout FETCH_HEAD",
+
},
+
}
+
}
+
+
// extractCommitSHA extracts the commit SHA from trigger metadata based on trigger type
+
func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) {
+
switch workflow.TriggerKind(tr.Kind) {
+
case workflow.TriggerKindPush:
+
if tr.Push == nil {
+
return "", fmt.Errorf("push trigger metadata is nil")
+
}
+
return tr.Push.NewSha, nil
+
+
case workflow.TriggerKindPullRequest:
+
if tr.PullRequest == nil {
+
return "", fmt.Errorf("pull request trigger metadata is nil")
+
}
+
return tr.PullRequest.SourceSha, nil
+
+
case workflow.TriggerKindManual:
+
// Manual triggers don't have an explicit SHA in the metadata
+
// For now, return empty string - could be enhanced to fetch from default branch
+
// TODO: Implement manual trigger SHA resolution (fetch default branch HEAD)
+
return "", nil
+
+
default:
+
return "", fmt.Errorf("unknown trigger kind: %s", tr.Kind)
+
}
+
}
+
+
// buildRepoURL constructs the repository URL from trigger metadata
+
func buildRepoURL(tr tangled.Pipeline_TriggerMetadata, devMode bool) string {
+
if tr.Repo == nil {
+
return ""
+
}
+
+
// Determine protocol
+
scheme := "https://"
+
if devMode {
+
scheme = "http://"
+
}
+
+
// Get host from knot
+
host := tr.Repo.Knot
+
+
// In dev mode, replace localhost with host.docker.internal for Docker networking
+
if devMode && strings.Contains(host, "localhost") {
+
host = strings.ReplaceAll(host, "localhost", "host.docker.internal")
+
}
+
+
// Build URL: {scheme}{knot}/{did}/{repo}
+
return fmt.Sprintf("%s%s/%s/%s", scheme, host, tr.Repo.Did, tr.Repo.Repo)
+
}
+
+
// buildFetchArgs constructs the arguments for git fetch based on clone options
+
func buildFetchArgs(clone tangled.Pipeline_CloneOpts, sha string) []string {
+
args := []string{}
+
+
// Set fetch depth (default to 1 for shallow clone)
+
depth := clone.Depth
+
if depth == 0 {
+
depth = 1
+
}
+
args = append(args, fmt.Sprintf("--depth=%d", depth))
+
+
// Add submodules if requested
+
if clone.Submodules {
+
args = append(args, "--recurse-submodules=yes")
+
}
+
+
// Add remote and SHA
+
args = append(args, "origin")
+
if sha != "" {
+
args = append(args, sha)
+
}
+
+
return args
+
}
+371
spindle/models/clone_test.go
···
+
package models
+
+
import (
+
"strings"
+
"testing"
+
+
"tangled.org/core/api/tangled"
+
"tangled.org/core/workflow"
+
)
+
+
func TestBuildCloneStep_PushTrigger(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Submodules: false,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
OldSha: "def456",
+
Ref: "refs/heads/main",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
if step.Kind() != StepKindSystem {
+
t.Errorf("Expected StepKindSystem, got %v", step.Kind())
+
}
+
+
if step.Name() != "Clone repository into workspace" {
+
t.Errorf("Expected 'Clone repository into workspace', got '%s'", step.Name())
+
}
+
+
commands := step.Commands()
+
if len(commands) != 4 {
+
t.Errorf("Expected 4 commands, got %d", len(commands))
+
}
+
+
// Verify commands contain expected git operations
+
allCmds := strings.Join(commands, " ")
+
if !strings.Contains(allCmds, "git init") {
+
t.Error("Commands should contain 'git init'")
+
}
+
if !strings.Contains(allCmds, "git remote add origin") {
+
t.Error("Commands should contain 'git remote add origin'")
+
}
+
if !strings.Contains(allCmds, "git fetch") {
+
t.Error("Commands should contain 'git fetch'")
+
}
+
if !strings.Contains(allCmds, "abc123") {
+
t.Error("Commands should contain commit SHA")
+
}
+
if !strings.Contains(allCmds, "git checkout FETCH_HEAD") {
+
t.Error("Commands should contain 'git checkout FETCH_HEAD'")
+
}
+
if !strings.Contains(allCmds, "https://example.com/did:plc:user123/my-repo") {
+
t.Error("Commands should contain expected repo URL")
+
}
+
}
+
+
func TestBuildCloneStep_PullRequestTrigger(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPullRequest),
+
PullRequest: &tangled.Pipeline_PullRequestTriggerData{
+
SourceSha: "pr-sha-789",
+
SourceBranch: "feature-branch",
+
TargetBranch: "main",
+
Action: "opened",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
allCmds := strings.Join(step.Commands(), " ")
+
if !strings.Contains(allCmds, "pr-sha-789") {
+
t.Error("Commands should contain PR commit SHA")
+
}
+
}
+
+
func TestBuildCloneStep_ManualTrigger(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindManual),
+
Manual: &tangled.Pipeline_ManualTriggerData{
+
Inputs: nil,
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
// Manual triggers don't have a SHA yet (TODO), so git fetch won't include a SHA
+
allCmds := strings.Join(step.Commands(), " ")
+
// Should still have basic git commands
+
if !strings.Contains(allCmds, "git init") {
+
t.Error("Commands should contain 'git init'")
+
}
+
if !strings.Contains(allCmds, "git fetch") {
+
t.Error("Commands should contain 'git fetch'")
+
}
+
}
+
+
func TestBuildCloneStep_SkipFlag(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Skip: true,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
// Empty step when skip is true
+
if step.Name() != "" {
+
t.Error("Expected empty step name when Skip is true")
+
}
+
if len(step.Commands()) != 0 {
+
t.Errorf("Expected no commands when Skip is true, got %d commands", len(step.Commands()))
+
}
+
}
+
+
func TestBuildCloneStep_DevMode(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "localhost:3000",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, true)
+
+
// In dev mode, should use http:// and replace localhost with host.docker.internal
+
allCmds := strings.Join(step.Commands(), " ")
+
expectedURL := "http://host.docker.internal:3000/did:plc:user123/my-repo"
+
if !strings.Contains(allCmds, expectedURL) {
+
t.Errorf("Expected dev mode URL '%s' in commands", expectedURL)
+
}
+
}
+
+
func TestBuildCloneStep_DepthAndSubmodules(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 10,
+
Submodules: true,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
allCmds := strings.Join(step.Commands(), " ")
+
if !strings.Contains(allCmds, "--depth=10") {
+
t.Error("Commands should contain '--depth=10'")
+
}
+
+
if !strings.Contains(allCmds, "--recurse-submodules=yes") {
+
t.Error("Commands should contain '--recurse-submodules=yes'")
+
}
+
}
+
+
func TestBuildCloneStep_DefaultDepth(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 0, // Default should be 1
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
allCmds := strings.Join(step.Commands(), " ")
+
if !strings.Contains(allCmds, "--depth=1") {
+
t.Error("Commands should default to '--depth=1'")
+
}
+
}
+
+
func TestBuildCloneStep_NilPushData(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: nil, // Nil push data should create error step
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
// Should return an error step
+
if !strings.Contains(step.Name(), "error") {
+
t.Error("Expected error in step name when push data is nil")
+
}
+
+
allCmds := strings.Join(step.Commands(), " ")
+
if !strings.Contains(allCmds, "Failed to get clone info") {
+
t.Error("Commands should contain error message")
+
}
+
if !strings.Contains(allCmds, "exit 1") {
+
t.Error("Commands should exit with error")
+
}
+
}
+
+
func TestBuildCloneStep_NilPRData(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPullRequest),
+
PullRequest: nil, // Nil PR data should create error step
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
// Should return an error step
+
if !strings.Contains(step.Name(), "error") {
+
t.Error("Expected error in step name when pull request data is nil")
+
}
+
+
allCmds := strings.Join(step.Commands(), " ")
+
if !strings.Contains(allCmds, "Failed to get clone info") {
+
t.Error("Commands should contain error message")
+
}
+
}
+
+
func TestBuildCloneStep_UnknownTriggerKind(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: &tangled.Pipeline_CloneOpts{
+
Depth: 1,
+
Skip: false,
+
},
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: "unknown_trigger",
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
// Should return an error step
+
if !strings.Contains(step.Name(), "error") {
+
t.Error("Expected error in step name for unknown trigger kind")
+
}
+
+
allCmds := strings.Join(step.Commands(), " ")
+
if !strings.Contains(allCmds, "unknown trigger kind") {
+
t.Error("Commands should contain error message about unknown trigger kind")
+
}
+
}
+
+
func TestBuildCloneStep_NilCloneOpts(t *testing.T) {
+
twf := tangled.Pipeline_Workflow{
+
Clone: nil, // Nil clone options should use defaults
+
}
+
tr := tangled.Pipeline_TriggerMetadata{
+
Kind: string(workflow.TriggerKindPush),
+
Push: &tangled.Pipeline_PushTriggerData{
+
NewSha: "abc123",
+
},
+
Repo: &tangled.Pipeline_TriggerRepo{
+
Knot: "example.com",
+
Did: "did:plc:user123",
+
Repo: "my-repo",
+
},
+
}
+
+
step := BuildCloneStep(twf, tr, false)
+
+
// Should still work with default options
+
if step.Kind() != StepKindSystem {
+
t.Errorf("Expected StepKindSystem, got %v", step.Kind())
+
}
+
+
allCmds := strings.Join(step.Commands(), " ")
+
if !strings.Contains(allCmds, "--depth=1") {
+
t.Error("Commands should default to '--depth=1' when Clone is nil")
+
}
+
if !strings.Contains(allCmds, "git init") {
+
t.Error("Commands should contain 'git init'")
+
}
+
}