spindle: rework clone step #277

merged

opened by

oppi.li 5 months ago targeting master from push-vquoltwpkuny

use git fetch instead of git clone. git clone is lacking in that it does not permit cloning at a specific rev. when paired with --branch flag, there is no guarantee if the required rev is available in the shallow clone.

with git fetch however, we are able to fetch a specific rev. the steps are a little more involved however:

git init
git remote add origin <url>
git fetch --depth=<d> --recurse-submodules=<yes|no> <sha>
git checkout FETCH_HEAD

Signed-off-by: oppiliappan me@oppi.li

options

unified split

Changed files

+58 -61

spindle

models

pipeline.go

setup_steps.go

workflow

def.go

-1

spindle/models/pipeline.go

···

       69
        
       

     

       70
        
       		setup.addStep(nixConfStep())

     

       71
        
       		setup.addStep(cloneStep(*twf, *pl.TriggerMetadata, cfg.Server.Dev))

     

       72
       -
       		setup.addStep(checkoutStep(*twf, *pl.TriggerMetadata))

     

       73
        
       		// this step could be empty

     

       74
        
       		if s := dependencyStep(*twf); s != nil {

     

       75
        
       			setup.addStep(*s)

···

       69
        
       

     

       70
        
       		setup.addStep(nixConfStep())

     

       71
        
       		setup.addStep(cloneStep(*twf, *pl.TriggerMetadata, cfg.Server.Dev))

     

       0
        
       
     

       72
        
       		// this step could be empty

     

       73
        
       		if s := dependencyStep(*twf); s != nil {

     

       74
        
       			setup.addStep(*s)

+46 -57

spindle/models/setup_steps.go

···

       5
        
       	"path"

     

       6
        
       	"strings"

     

       7
        
       

     

       8
       -
       	"github.com/go-git/go-git/v5/plumbing"

     

       9
        
       	"tangled.sh/tangled.sh/core/api/tangled"

     

       10
        
       	"tangled.sh/tangled.sh/core/workflow"

     

       11
        
       )

     
···

       19
        
       	}

     

       20
        
       }

     

       21
        
       

     

       22
       -
       // checkoutStep checks out the specified ref in the cloned repository.

     

       23
       -
       func checkoutStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata) Step {

     

       24
       -
       	if twf.Clone.Skip {

     

       25
       -
       		return Step{}

     

       26
       -
       	}

     

       27
       -
       

     

       28
       -
       	var ref string

     

       29
       -
       	switch tr.Kind {

     

       30
       -
       	case "push":

     

       31
       -
       		ref = tr.Push.NewSha

     

       32
       -
       	case "pull_request":

     

       33
       -
       		ref = tr.PullRequest.TargetBranch

     

       34
       -
       

     

       35
       -
       	// TODO: this needs to be specified in lexicon

     

       36
       -
       	case "manual":

     

       37
       -
       		ref = tr.Repo.DefaultBranch

     

       38
       -
       	}

     

       39
       -
       

     

       40
       -
       	checkoutCmd := fmt.Sprintf("git config advice.detachedHead false; git checkout --progress --force %s", ref)

     

       41
       -
       

     

       42
       -
       	return Step{

     

       43
       -
       		Command: checkoutCmd,

     

       44
       -
       		Name:    "Checkout ref " + ref,

     

       45
       -
       	}

     

       46
       -
       }

     

       47
       -
       

     

       48
        
       // cloneOptsAsSteps processes clone options and adds corresponding steps

     

       49
        
       // to the beginning of the workflow's step list if cloning is not skipped.

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       50
        
       func cloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) Step {

     

       51
        
       	if twf.Clone.Skip {

     

       52
        
       		return Step{}

     

       53
        
       	}

     

       54
        
       

     

       55
       -
       	uri := "https://"

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       56
        
       	if dev {

     

       57
       -
       		uri = "http://"

     

       58
        
       		tr.Repo.Knot = strings.ReplaceAll(tr.Repo.Knot, "localhost", "host.docker.internal")

     

       59
        
       	}

     

       0
        
       
     

       0
        
       
     

       60
        
       

     

       61
       -
       	cloneUrl := uri + path.Join(tr.Repo.Knot, tr.Repo.Did, tr.Repo.Repo)

     

       62
       -
       	cloneCmd := []string{"git", "clone", cloneUrl, "."}

     

       0
        
       
     

       63
        
       

     

       64
       -
       	// default clone depth is 1

     

       65
       -
       	cloneDepth := 1

     

       66
       -
       	if twf.Clone.Depth > 1 {

     

       67
       -
       		cloneDepth = int(twf.Clone.Depth)

     

       68
       -
       	}

     

       69
       -
       	cloneCmd = append(cloneCmd, fmt.Sprintf("--depth=%d", cloneDepth))

     

       70
       -
       

     

       71
       -
       	// select the clone branch

     

       72
       -
       	cloneBranch := ""

     

       73
       -
       	switch tr.Kind {

     

       74
       -
       	case workflow.TriggerKindManual:

     

       75
       -
       		// TODO: unimplemented

     

       76
       -
       	case workflow.TriggerKindPush:

     

       77
       -
       		ref := tr.Push.Ref

     

       78
       -
       		refName := plumbing.ReferenceName(ref)

     

       79
       -
       		cloneBranch = refName.Short()

     

       80
       -
       	case workflow.TriggerKindPullRequest:

     

       81
       -
       		cloneBranch = tr.PullRequest.SourceBranch

     

       82
       -
       	}

     

       83
        
       

     

       84
       -
       	if cloneBranch != "" {

     

       85
       -
       		cloneCmd = append(cloneCmd, fmt.Sprintf("--branch=%s", cloneBranch))

     

       86
       -
       	}

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       87
        
       

     

       88
       -
       	if twf.Clone.Submodules {

     

       89
       -
       		cloneCmd = append(cloneCmd, "--recursive")

     

       90
        
       	}

     

       91
        
       

     

       92
       -
       	fmt.Println(strings.Join(cloneCmd, " "))

     

       0
        
       
     

       93
        
       

     

       94
        
       	cloneStep := Step{

     

       95
       -
       		Command: strings.Join(cloneCmd, " "),

     

       96
        
       		Name:    "Clone repository into workspace",

     

       97
        
       	}

     

       98
        
       	return cloneStep

···

       5
        
       	"path"

     

       6
        
       	"strings"

     

       7
        
       

     

       0
        
       
     

       8
        
       	"tangled.sh/tangled.sh/core/api/tangled"

     

       9
        
       	"tangled.sh/tangled.sh/core/workflow"

     

       10
        
       )

     
···

       18
        
       	}

     

       19
        
       }

     

       20
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       21
        
       // cloneOptsAsSteps processes clone options and adds corresponding steps

     

       22
        
       // to the beginning of the workflow's step list if cloning is not skipped.

     

       23
       +
       //

     

       24
       +
       // the steps to do here are:

     

       25
       +
       // - git init

     

       26
       +
       // - git remote add origin <url>

     

       27
       +
       // - git fetch --depth=<d> --recurse-submodules=<yes|no> <sha>

     

       28
       +
       // - git checkout FETCH_HEAD

     

       29
        
       func cloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata, dev bool) Step {

     

       30
        
       	if twf.Clone.Skip {

     

       31
        
       		return Step{}

     

       32
        
       	}

     

       33
        
       

     

       34
       +
       	var commands []string

     

       35
       +
       

     

       36
       +
       	// initialize git repo in workspace

     

       37
       +
       	commands = append(commands, "git init")

     

       38
       +
       

     

       39
       +
       	// add repo as git remote

     

       40
       +
       	scheme := "https://"

     

       41
        
       	if dev {

     

       42
       +
       		scheme = "http://"

     

       43
        
       		tr.Repo.Knot = strings.ReplaceAll(tr.Repo.Knot, "localhost", "host.docker.internal")

     

       44
        
       	}

     

       45
       +
       	url := scheme + path.Join(tr.Repo.Knot, tr.Repo.Did, tr.Repo.Repo)

     

       46
       +
       	commands = append(commands, fmt.Sprintf("git remote add origin %s", url))

     

       47
        
       

     

       48
       +
       	// run git fetch

     

       49
       +
       	{

     

       50
       +
       		var fetchArgs []string

     

       51
        
       

     

       52
       +
       		// default clone depth is 1

     

       53
       +
       		depth := 1

     

       54
       +
       		if twf.Clone.Depth > 1 {

     

       55
       +
       			depth = int(twf.Clone.Depth)

     

       56
       +
       		}

     

       57
       +
       		fetchArgs = append(fetchArgs, fmt.Sprintf("--depth=%d", depth))

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       58
        
       

     

       59
       +
       		// optionally recurse submodules

     

       60
       +
       		if twf.Clone.Submodules {

     

       61
       +
       			fetchArgs = append(fetchArgs, "--recurse-submodules=yes")

     

       62
       +
       		}

     

       63
       +
       

     

       64
       +
       		// set remote to fetch from

     

       65
       +
       		fetchArgs = append(fetchArgs, "origin")

     

       66
       +
       

     

       67
       +
       		// set revision to checkout

     

       68
       +
       		switch workflow.TriggerKind(tr.Kind) {

     

       69
       +
       		case workflow.TriggerKindManual:

     

       70
       +
       			// TODO: unimplemented

     

       71
       +
       		case workflow.TriggerKindPush:

     

       72
       +
       			fetchArgs = append(fetchArgs, tr.Push.NewSha)

     

       73
       +
       		case workflow.TriggerKindPullRequest:

     

       74
       +
       			fetchArgs = append(fetchArgs, tr.PullRequest.SourceSha)

     

       75
       +
       		}

     

       76
        
       

     

       77
       +
       		commands = append(commands, fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")))

     

       0
        
       
     

       78
        
       	}

     

       79
        
       

     

       80
       +
       	// run git checkout

     

       81
       +
       	commands = append(commands, "git checkout FETCH_HEAD")

     

       82
        
       

     

       83
        
       	cloneStep := Step{

     

       84
       +
       		Command: strings.Join(commands, "\n"),

     

       85
        
       		Name:    "Clone repository into workspace",

     

       86
        
       	}

     

       87
        
       	return cloneStep

+12 -3

workflow/def.go

···

       4
        
       	"errors"

     

       5
        
       	"fmt"

     

       6
        
       	"slices"

     

       0
        
       
     

       7
        
       

     

       8
        
       	"tangled.sh/tangled.sh/core/api/tangled"

     

       9
        
       

     
···

       51
        
       	}

     

       52
        
       

     

       53
        
       	StringList []string

     

       0
        
       
     

       0
        
       
     

       54
        
       )

     

       55
        
       

     

       56
        
       const (

     

       57
       -
       	TriggerKindPush        string = "push"

     

       58
       -
       	TriggerKindPullRequest string = "pull_request"

     

       59
       -
       	TriggerKindManual      string = "manual"

     

       0
        
       
     

       0
        
       
     

       60
        
       )

     

       61
        
       

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       62
        
       func FromFile(name string, contents []byte) (Workflow, error) {

     

       63
        
       	var wf Workflow

     

       64

···

       4
        
       	"errors"

     

       5
        
       	"fmt"

     

       6
        
       	"slices"

     

       7
       +
       	"strings"

     

       8
        
       

     

       9
        
       	"tangled.sh/tangled.sh/core/api/tangled"

     

       10
        
       

     
···

       52
        
       	}

     

       53
        
       

     

       54
        
       	StringList []string

     

       55
       +
       

     

       56
       +
       	TriggerKind string

     

       57
        
       )

     

       58
        
       

     

       59
        
       const (

     

       60
       +
       	WorkflowDir = ".tangled/workflows"

     

       61
       +
       

     

       62
       +
       	TriggerKindPush        TriggerKind = "push"

     

       63
       +
       	TriggerKindPullRequest TriggerKind = "pull_request"

     

       64
       +
       	TriggerKindManual      TriggerKind = "manual"

     

       65
        
       )

     

       66
        
       

     

       67
       +
       func (t TriggerKind) String() string {

     

       68
       +
       	return strings.ReplaceAll(string(t), "_", " ")

     

       69
       +
       }

     

       70
       +
       

     

       71
        
       func FromFile(name string, contents []byte) (Workflow, error) {

     

       72
        
       	var wf Workflow

     

       73