veryroundbird.house / core
forked from tangled.org/core
this repo has no description
fork atom

spindle: make workflows engine-agnostic

Signed-off-by: Winter <winter@winter.cafe>

winter.bsky.social 251f3d44 3ee002a1

options
unified split
Changed files
+780 -1445
api
tangled
cbor_gen.go
tangledpipeline.go
cmd
gen.go
docs
spindle
pipeline.md
lexicons
pipeline
pipeline.json
nix
modules
spindle.nix
spindle
config
config.go
engine
engine.go
errors.go
engines
nixery
ansi_stripper.go
engine.go
envs.go
envs_test.go
errors.go
setup_steps.go
models
engine.go
logger.go
models.go
pipeline.go
server.go
stream.go
xrpc
xrpc.go
workflow
compile.go
compile_test.go
def.go
def_test.go
+53 -600
api/tangled/cbor_gen.go 
···

       
2728

       
2728

       
 

       



     

       
2729

       
2729

       
 

       
	return nil


     

       
2730

       
2730

       
 

       
}


     

       
2731

       

       
-

       
func (t *Pipeline_Dependency) MarshalCBOR(w io.Writer) error {


     

       
2732

       

       
-

       
	if t == nil {


     

       
2733

       

       
-

       
		_, err := w.Write(cbg.CborNull)


     

       
2734

       

       
-

       
		return err


     

       
2735

       

       
-

       
	}


     

       
2736

       

       
-

       



     

       
2737

       

       
-

       
	cw := cbg.NewCborWriter(w)


     

       
2738

       

       
-

       



     

       
2739

       

       
-

       
	if _, err := cw.Write([]byte{162}); err != nil {


     

       
2740

       

       
-

       
		return err


     

       
2741

       

       
-

       
	}


     

       
2742

       

       
-

       



     

       
2743

       

       
-

       
	// t.Packages ([]string) (slice)


     

       
2744

       

       
-

       
	if len("packages") > 1000000 {


     

       
2745

       

       
-

       
		return xerrors.Errorf("Value in field \"packages\" was too long")


     

       
2746

       

       
-

       
	}


     

       
2747

       

       
-

       



     

       
2748

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("packages"))); err != nil {


     

       
2749

       

       
-

       
		return err


     

       
2750

       

       
-

       
	}


     

       
2751

       

       
-

       
	if _, err := cw.WriteString(string("packages")); err != nil {


     

       
2752

       

       
-

       
		return err


     

       
2753

       

       
-

       
	}


     

       
2754

       

       
-

       



     

       
2755

       

       
-

       
	if len(t.Packages) > 8192 {


     

       
2756

       

       
-

       
		return xerrors.Errorf("Slice value in field t.Packages was too long")


     

       
2757

       

       
-

       
	}


     

       
2758

       

       
-

       



     

       
2759

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Packages))); err != nil {


     

       
2760

       

       
-

       
		return err


     

       
2761

       

       
-

       
	}


     

       
2762

       

       
-

       
	for _, v := range t.Packages {


     

       
2763

       

       
-

       
		if len(v) > 1000000 {


     

       
2764

       

       
-

       
			return xerrors.Errorf("Value in field v was too long")


     

       
2765

       

       
-

       
		}


     

       
2766

       

       
-

       



     

       
2767

       

       
-

       
		if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil {


     

       
2768

       

       
-

       
			return err


     

       
2769

       

       
-

       
		}


     

       
2770

       

       
-

       
		if _, err := cw.WriteString(string(v)); err != nil {


     

       
2771

       

       
-

       
			return err


     

       
2772

       

       
-

       
		}


     

       
2773

       

       
-

       



     

       
2774

       

       
-

       
	}


     

       
2775

       

       
-

       



     

       
2776

       

       
-

       
	// t.Registry (string) (string)


     

       
2777

       

       
-

       
	if len("registry") > 1000000 {


     

       
2778

       

       
-

       
		return xerrors.Errorf("Value in field \"registry\" was too long")


     

       
2779

       

       
-

       
	}


     

       
2780

       

       
-

       



     

       
2781

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("registry"))); err != nil {


     

       
2782

       

       
-

       
		return err


     

       
2783

       

       
-

       
	}


     

       
2784

       

       
-

       
	if _, err := cw.WriteString(string("registry")); err != nil {


     

       
2785

       

       
-

       
		return err


     

       
2786

       

       
-

       
	}


     

       
2787

       

       
-

       



     

       
2788

       

       
-

       
	if len(t.Registry) > 1000000 {


     

       
2789

       

       
-

       
		return xerrors.Errorf("Value in field t.Registry was too long")


     

       
2790

       

       
-

       
	}


     

       
2791

       

       
-

       



     

       
2792

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Registry))); err != nil {


     

       
2793

       

       
-

       
		return err


     

       
2794

       

       
-

       
	}


     

       
2795

       

       
-

       
	if _, err := cw.WriteString(string(t.Registry)); err != nil {


     

       
2796

       

       
-

       
		return err


     

       
2797

       

       
-

       
	}


     

       
2798

       

       
-

       
	return nil


     

       
2799

       

       
-

       
}


     

       
2800

       

       
-

       



     

       
2801

       

       
-

       
func (t *Pipeline_Dependency) UnmarshalCBOR(r io.Reader) (err error) {


     

       
2802

       

       
-

       
	*t = Pipeline_Dependency{}


     

       
2803

       

       
-

       



     

       
2804

       

       
-

       
	cr := cbg.NewCborReader(r)


     

       
2805

       

       
-

       



     

       
2806

       

       
-

       
	maj, extra, err := cr.ReadHeader()


     

       
2807

       

       
-

       
	if err != nil {


     

       
2808

       

       
-

       
		return err


     

       
2809

       

       
-

       
	}


     

       
2810

       

       
-

       
	defer func() {


     

       
2811

       

       
-

       
		if err == io.EOF {


     

       
2812

       

       
-

       
			err = io.ErrUnexpectedEOF


     

       
2813

       

       
-

       
		}


     

       
2814

       

       
-

       
	}()


     

       
2815

       

       
-

       



     

       
2816

       

       
-

       
	if maj != cbg.MajMap {


     

       
2817

       

       
-

       
		return fmt.Errorf("cbor input should be of type map")


     

       
2818

       

       
-

       
	}


     

       
2819

       

       
-

       



     

       
2820

       

       
-

       
	if extra > cbg.MaxLength {


     

       
2821

       

       
-

       
		return fmt.Errorf("Pipeline_Dependency: map struct too large (%d)", extra)


     

       
2822

       

       
-

       
	}


     

       
2823

       

       
-

       



     

       
2824

       

       
-

       
	n := extra


     

       
2825

       

       
-

       



     

       
2826

       

       
-

       
	nameBuf := make([]byte, 8)


     

       
2827

       

       
-

       
	for i := uint64(0); i < n; i++ {


     

       
2828

       

       
-

       
		nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)


     

       
2829

       

       
-

       
		if err != nil {


     

       
2830

       

       
-

       
			return err


     

       
2831

       

       
-

       
		}


     

       
2832

       

       
-

       



     

       
2833

       

       
-

       
		if !ok {


     

       
2834

       

       
-

       
			// Field doesn't exist on this type, so ignore it


     

       
2835

       

       
-

       
			if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil {


     

       
2836

       

       
-

       
				return err


     

       
2837

       

       
-

       
			}


     

       
2838

       

       
-

       
			continue


     

       
2839

       

       
-

       
		}


     

       
2840

       

       
-

       



     

       
2841

       

       
-

       
		switch string(nameBuf[:nameLen]) {


     

       
2842

       

       
-

       
		// t.Packages ([]string) (slice)


     

       
2843

       

       
-

       
		case "packages":


     

       
2844

       

       
-

       



     

       
2845

       

       
-

       
			maj, extra, err = cr.ReadHeader()


     

       
2846

       

       
-

       
			if err != nil {


     

       
2847

       

       
-

       
				return err


     

       
2848

       

       
-

       
			}


     

       
2849

       

       
-

       



     

       
2850

       

       
-

       
			if extra > 8192 {


     

       
2851

       

       
-

       
				return fmt.Errorf("t.Packages: array too large (%d)", extra)


     

       
2852

       

       
-

       
			}


     

       
2853

       

       
-

       



     

       
2854

       

       
-

       
			if maj != cbg.MajArray {


     

       
2855

       

       
-

       
				return fmt.Errorf("expected cbor array")


     

       
2856

       

       
-

       
			}


     

       
2857

       

       
-

       



     

       
2858

       

       
-

       
			if extra > 0 {


     

       
2859

       

       
-

       
				t.Packages = make([]string, extra)


     

       
2860

       

       
-

       
			}


     

       
2861

       

       
-

       



     

       
2862

       

       
-

       
			for i := 0; i < int(extra); i++ {


     

       
2863

       

       
-

       
				{


     

       
2864

       

       
-

       
					var maj byte


     

       
2865

       

       
-

       
					var extra uint64


     

       
2866

       

       
-

       
					var err error


     

       
2867

       

       
-

       
					_ = maj


     

       
2868

       

       
-

       
					_ = extra


     

       
2869

       

       
-

       
					_ = err


     

       
2870

       

       
-

       



     

       
2871

       

       
-

       
					{


     

       
2872

       

       
-

       
						sval, err := cbg.ReadStringWithMax(cr, 1000000)


     

       
2873

       

       
-

       
						if err != nil {


     

       
2874

       

       
-

       
							return err


     

       
2875

       

       
-

       
						}


     

       
2876

       

       
-

       



     

       
2877

       

       
-

       
						t.Packages[i] = string(sval)


     

       
2878

       

       
-

       
					}


     

       
2879

       

       
-

       



     

       
2880

       

       
-

       
				}


     

       
2881

       

       
-

       
			}


     

       
2882

       

       
-

       
			// t.Registry (string) (string)


     

       
2883

       

       
-

       
		case "registry":


     

       
2884

       

       
-

       



     

       
2885

       

       
-

       
			{


     

       
2886

       

       
-

       
				sval, err := cbg.ReadStringWithMax(cr, 1000000)


     

       
2887

       

       
-

       
				if err != nil {


     

       
2888

       

       
-

       
					return err


     

       
2889

       

       
-

       
				}


     

       
2890

       

       
-

       



     

       
2891

       

       
-

       
				t.Registry = string(sval)


     

       
2892

       

       
-

       
			}


     

       
2893

       

       
-

       



     

       
2894

       

       
-

       
		default:


     

       
2895

       

       
-

       
			// Field doesn't exist on this type, so ignore it


     

       
2896

       

       
-

       
			if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil {


     

       
2897

       

       
-

       
				return err


     

       
2898

       

       
-

       
			}


     

       
2899

       

       
-

       
		}


     

       
2900

       

       
-

       
	}


     

       
2901

       

       
-

       



     

       
2902

       

       
-

       
	return nil


     

       
2903

       

       
-

       
}


     

       
2904

       
2731

       
 

       
func (t *Pipeline_ManualTriggerData) MarshalCBOR(w io.Writer) error {


     

       
2905

       
2732

       
 

       
	if t == nil {


     

       
2906

       
2733

       
 

       
		_, err := w.Write(cbg.CborNull)


     
···

       
3916

       
3743

       
 

       



     

       
3917

       
3744

       
 

       
	return nil


     

       
3918

       
3745

       
 

       
}


     

       
3919

       

       
-

       
func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error {


     

       
3920

       

       
-

       
	if t == nil {


     

       
3921

       

       
-

       
		_, err := w.Write(cbg.CborNull)


     

       
3922

       

       
-

       
		return err


     

       
3923

       

       
-

       
	}


     

       
3924

       

       
-

       



     

       
3925

       

       
-

       
	cw := cbg.NewCborWriter(w)


     

       
3926

       

       
-

       
	fieldCount := 3


     

       
3927

       

       
-

       



     

       
3928

       

       
-

       
	if t.Environment == nil {


     

       
3929

       

       
-

       
		fieldCount--


     

       
3930

       

       
-

       
	}


     

       
3931

       

       
-

       



     

       
3932

       

       
-

       
	if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil {


     

       
3933

       

       
-

       
		return err


     

       
3934

       

       
-

       
	}


     

       
3935

       

       
-

       



     

       
3936

       

       
-

       
	// t.Name (string) (string)


     

       
3937

       

       
-

       
	if len("name") > 1000000 {


     

       
3938

       

       
-

       
		return xerrors.Errorf("Value in field \"name\" was too long")


     

       
3939

       

       
-

       
	}


     

       
3940

       

       
-

       



     

       
3941

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil {


     

       
3942

       

       
-

       
		return err


     

       
3943

       

       
-

       
	}


     

       
3944

       

       
-

       
	if _, err := cw.WriteString(string("name")); err != nil {


     

       
3945

       

       
-

       
		return err


     

       
3946

       

       
-

       
	}


     

       
3947

       

       
-

       



     

       
3948

       

       
-

       
	if len(t.Name) > 1000000 {


     

       
3949

       

       
-

       
		return xerrors.Errorf("Value in field t.Name was too long")


     

       
3950

       

       
-

       
	}


     

       
3951

       

       
-

       



     

       
3952

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil {


     

       
3953

       

       
-

       
		return err


     

       
3954

       

       
-

       
	}


     

       
3955

       

       
-

       
	if _, err := cw.WriteString(string(t.Name)); err != nil {


     

       
3956

       

       
-

       
		return err


     

       
3957

       

       
-

       
	}


     

       
3958

       

       
-

       



     

       
3959

       

       
-

       
	// t.Command (string) (string)


     

       
3960

       

       
-

       
	if len("command") > 1000000 {


     

       
3961

       

       
-

       
		return xerrors.Errorf("Value in field \"command\" was too long")


     

       
3962

       

       
-

       
	}


     

       
3963

       

       
-

       



     

       
3964

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("command"))); err != nil {


     

       
3965

       

       
-

       
		return err


     

       
3966

       

       
-

       
	}


     

       
3967

       

       
-

       
	if _, err := cw.WriteString(string("command")); err != nil {


     

       
3968

       

       
-

       
		return err


     

       
3969

       

       
-

       
	}


     

       
3970

       

       
-

       



     

       
3971

       

       
-

       
	if len(t.Command) > 1000000 {


     

       
3972

       

       
-

       
		return xerrors.Errorf("Value in field t.Command was too long")


     

       
3973

       

       
-

       
	}


     

       
3974

       

       
-

       



     

       
3975

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Command))); err != nil {


     

       
3976

       

       
-

       
		return err


     

       
3977

       

       
-

       
	}


     

       
3978

       

       
-

       
	if _, err := cw.WriteString(string(t.Command)); err != nil {


     

       
3979

       

       
-

       
		return err


     

       
3980

       

       
-

       
	}


     

       
3981

       

       
-

       



     

       
3982

       

       
-

       
	// t.Environment ([]*tangled.Pipeline_Pair) (slice)


     

       
3983

       

       
-

       
	if t.Environment != nil {


     

       
3984

       

       
-

       



     

       
3985

       

       
-

       
		if len("environment") > 1000000 {


     

       
3986

       

       
-

       
			return xerrors.Errorf("Value in field \"environment\" was too long")


     

       
3987

       

       
-

       
		}


     

       
3988

       

       
-

       



     

       
3989

       

       
-

       
		if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("environment"))); err != nil {


     

       
3990

       

       
-

       
			return err


     

       
3991

       

       
-

       
		}


     

       
3992

       

       
-

       
		if _, err := cw.WriteString(string("environment")); err != nil {


     

       
3993

       

       
-

       
			return err


     

       
3994

       

       
-

       
		}


     

       
3995

       

       
-

       



     

       
3996

       

       
-

       
		if len(t.Environment) > 8192 {


     

       
3997

       

       
-

       
			return xerrors.Errorf("Slice value in field t.Environment was too long")


     

       
3998

       

       
-

       
		}


     

       
3999

       

       
-

       



     

       
4000

       

       
-

       
		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Environment))); err != nil {


     

       
4001

       

       
-

       
			return err


     

       
4002

       

       
-

       
		}


     

       
4003

       

       
-

       
		for _, v := range t.Environment {


     

       
4004

       

       
-

       
			if err := v.MarshalCBOR(cw); err != nil {


     

       
4005

       

       
-

       
				return err


     

       
4006

       

       
-

       
			}


     

       
4007

       

       
-

       



     

       
4008

       

       
-

       
		}


     

       
4009

       

       
-

       
	}


     

       
4010

       

       
-

       
	return nil


     

       
4011

       

       
-

       
}


     

       
4012

       

       
-

       



     

       
4013

       

       
-

       
func (t *Pipeline_Step) UnmarshalCBOR(r io.Reader) (err error) {


     

       
4014

       

       
-

       
	*t = Pipeline_Step{}


     

       
4015

       

       
-

       



     

       
4016

       

       
-

       
	cr := cbg.NewCborReader(r)


     

       
4017

       

       
-

       



     

       
4018

       

       
-

       
	maj, extra, err := cr.ReadHeader()


     

       
4019

       

       
-

       
	if err != nil {


     

       
4020

       

       
-

       
		return err


     

       
4021

       

       
-

       
	}


     

       
4022

       

       
-

       
	defer func() {


     

       
4023

       

       
-

       
		if err == io.EOF {


     

       
4024

       

       
-

       
			err = io.ErrUnexpectedEOF


     

       
4025

       

       
-

       
		}


     

       
4026

       

       
-

       
	}()


     

       
4027

       

       
-

       



     

       
4028

       

       
-

       
	if maj != cbg.MajMap {


     

       
4029

       

       
-

       
		return fmt.Errorf("cbor input should be of type map")


     

       
4030

       

       
-

       
	}


     

       
4031

       

       
-

       



     

       
4032

       

       
-

       
	if extra > cbg.MaxLength {


     

       
4033

       

       
-

       
		return fmt.Errorf("Pipeline_Step: map struct too large (%d)", extra)


     

       
4034

       

       
-

       
	}


     

       
4035

       

       
-

       



     

       
4036

       

       
-

       
	n := extra


     

       
4037

       

       
-

       



     

       
4038

       

       
-

       
	nameBuf := make([]byte, 11)


     

       
4039

       

       
-

       
	for i := uint64(0); i < n; i++ {


     

       
4040

       

       
-

       
		nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)


     

       
4041

       

       
-

       
		if err != nil {


     

       
4042

       

       
-

       
			return err


     

       
4043

       

       
-

       
		}


     

       
4044

       

       
-

       



     

       
4045

       

       
-

       
		if !ok {


     

       
4046

       

       
-

       
			// Field doesn't exist on this type, so ignore it


     

       
4047

       

       
-

       
			if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil {


     

       
4048

       

       
-

       
				return err


     

       
4049

       

       
-

       
			}


     

       
4050

       

       
-

       
			continue


     

       
4051

       

       
-

       
		}


     

       
4052

       

       
-

       



     

       
4053

       

       
-

       
		switch string(nameBuf[:nameLen]) {


     

       
4054

       

       
-

       
		// t.Name (string) (string)


     

       
4055

       

       
-

       
		case "name":


     

       
4056

       

       
-

       



     

       
4057

       

       
-

       
			{


     

       
4058

       

       
-

       
				sval, err := cbg.ReadStringWithMax(cr, 1000000)


     

       
4059

       

       
-

       
				if err != nil {


     

       
4060

       

       
-

       
					return err


     

       
4061

       

       
-

       
				}


     

       
4062

       

       
-

       



     

       
4063

       

       
-

       
				t.Name = string(sval)


     

       
4064

       

       
-

       
			}


     

       
4065

       

       
-

       
			// t.Command (string) (string)


     

       
4066

       

       
-

       
		case "command":


     

       
4067

       

       
-

       



     

       
4068

       

       
-

       
			{


     

       
4069

       

       
-

       
				sval, err := cbg.ReadStringWithMax(cr, 1000000)


     

       
4070

       

       
-

       
				if err != nil {


     

       
4071

       

       
-

       
					return err


     

       
4072

       

       
-

       
				}


     

       
4073

       

       
-

       



     

       
4074

       

       
-

       
				t.Command = string(sval)


     

       
4075

       

       
-

       
			}


     

       
4076

       

       
-

       
			// t.Environment ([]*tangled.Pipeline_Pair) (slice)


     

       
4077

       

       
-

       
		case "environment":


     

       
4078

       

       
-

       



     

       
4079

       

       
-

       
			maj, extra, err = cr.ReadHeader()


     

       
4080

       

       
-

       
			if err != nil {


     

       
4081

       

       
-

       
				return err


     

       
4082

       

       
-

       
			}


     

       
4083

       

       
-

       



     

       
4084

       

       
-

       
			if extra > 8192 {


     

       
4085

       

       
-

       
				return fmt.Errorf("t.Environment: array too large (%d)", extra)


     

       
4086

       

       
-

       
			}


     

       
4087

       

       
-

       



     

       
4088

       

       
-

       
			if maj != cbg.MajArray {


     

       
4089

       

       
-

       
				return fmt.Errorf("expected cbor array")


     

       
4090

       

       
-

       
			}


     

       
4091

       

       
-

       



     

       
4092

       

       
-

       
			if extra > 0 {


     

       
4093

       

       
-

       
				t.Environment = make([]*Pipeline_Pair, extra)


     

       
4094

       

       
-

       
			}


     

       
4095

       

       
-

       



     

       
4096

       

       
-

       
			for i := 0; i < int(extra); i++ {


     

       
4097

       

       
-

       
				{


     

       
4098

       

       
-

       
					var maj byte


     

       
4099

       

       
-

       
					var extra uint64


     

       
4100

       

       
-

       
					var err error


     

       
4101

       

       
-

       
					_ = maj


     

       
4102

       

       
-

       
					_ = extra


     

       
4103

       

       
-

       
					_ = err


     

       
4104

       

       
-

       



     

       
4105

       

       
-

       
					{


     

       
4106

       

       
-

       



     

       
4107

       

       
-

       
						b, err := cr.ReadByte()


     

       
4108

       

       
-

       
						if err != nil {


     

       
4109

       

       
-

       
							return err


     

       
4110

       

       
-

       
						}


     

       
4111

       

       
-

       
						if b != cbg.CborNull[0] {


     

       
4112

       

       
-

       
							if err := cr.UnreadByte(); err != nil {


     

       
4113

       

       
-

       
								return err


     

       
4114

       

       
-

       
							}


     

       
4115

       

       
-

       
							t.Environment[i] = new(Pipeline_Pair)


     

       
4116

       

       
-

       
							if err := t.Environment[i].UnmarshalCBOR(cr); err != nil {


     

       
4117

       

       
-

       
								return xerrors.Errorf("unmarshaling t.Environment[i] pointer: %w", err)


     

       
4118

       

       
-

       
							}


     

       
4119

       

       
-

       
						}


     

       
4120

       

       
-

       



     

       
4121

       

       
-

       
					}


     

       
4122

       

       
-

       



     

       
4123

       

       
-

       
				}


     

       
4124

       

       
-

       
			}


     

       
4125

       

       
-

       



     

       
4126

       

       
-

       
		default:


     

       
4127

       

       
-

       
			// Field doesn't exist on this type, so ignore it


     

       
4128

       

       
-

       
			if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil {


     

       
4129

       

       
-

       
				return err


     

       
4130

       

       
-

       
			}


     

       
4131

       

       
-

       
		}


     

       
4132

       

       
-

       
	}


     

       
4133

       

       
-

       



     

       
4134

       

       
-

       
	return nil


     

       
4135

       

       
-

       
}


     

       
4136

       
3746

       
 

       
func (t *Pipeline_TriggerMetadata) MarshalCBOR(w io.Writer) error {


     

       
4137

       
3747

       
 

       
	if t == nil {


     

       
4138

       
3748

       
 

       
		_, err := w.Write(cbg.CborNull)


     
···

       
4609

       
4219

       
 

       



     

       
4610

       
4220

       
 

       
	cw := cbg.NewCborWriter(w)


     

       
4611

       
4221

       
 

       



     

       
4612

       

       
-

       
	if _, err := cw.Write([]byte{165}); err != nil {


     

       

       
4222

       
+

       
	if _, err := cw.Write([]byte{164}); err != nil {


     

       

       
4223

       
+

       
		return err


     

       

       
4224

       
+

       
	}


     

       

       
4225

       
+

       



     

       

       
4226

       
+

       
	// t.Raw (string) (string)


     

       

       
4227

       
+

       
	if len("raw") > 1000000 {


     

       

       
4228

       
+

       
		return xerrors.Errorf("Value in field \"raw\" was too long")


     

       

       
4229

       
+

       
	}


     

       

       
4230

       
+

       



     

       

       
4231

       
+

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("raw"))); err != nil {


     

       

       
4232

       
+

       
		return err


     

       

       
4233

       
+

       
	}


     

       

       
4234

       
+

       
	if _, err := cw.WriteString(string("raw")); err != nil {


     

       

       
4235

       
+

       
		return err


     

       

       
4236

       
+

       
	}


     

       

       
4237

       
+

       



     

       

       
4238

       
+

       
	if len(t.Raw) > 1000000 {


     

       

       
4239

       
+

       
		return xerrors.Errorf("Value in field t.Raw was too long")


     

       

       
4240

       
+

       
	}


     

       

       
4241

       
+

       



     

       

       
4242

       
+

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Raw))); err != nil {


     

       

       
4243

       
+

       
		return err


     

       

       
4244

       
+

       
	}


     

       

       
4245

       
+

       
	if _, err := cw.WriteString(string(t.Raw)); err != nil {


     

       
4613

       
4246

       
 

       
		return err


     

       
4614

       
4247

       
 

       
	}


     

       
4615

       
4248

       
 

       



     
···

       
4652

       
4285

       
 

       
		return err


     

       
4653

       
4286

       
 

       
	}


     

       
4654

       
4287

       
 

       



     

       
4655

       

       
-

       
	// t.Steps ([]*tangled.Pipeline_Step) (slice)


     

       
4656

       

       
-

       
	if len("steps") > 1000000 {


     

       
4657

       

       
-

       
		return xerrors.Errorf("Value in field \"steps\" was too long")


     

       

       
4288

       
+

       
	// t.Engine (string) (string)


     

       

       
4289

       
+

       
	if len("engine") > 1000000 {


     

       

       
4290

       
+

       
		return xerrors.Errorf("Value in field \"engine\" was too long")


     

       
4658

       
4291

       
 

       
	}


     

       
4659

       
4292

       
 

       



     

       
4660

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("steps"))); err != nil {


     

       
4661

       

       
-

       
		return err


     

       
4662

       

       
-

       
	}


     

       
4663

       

       
-

       
	if _, err := cw.WriteString(string("steps")); err != nil {


     

       

       
4293

       
+

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("engine"))); err != nil {


     

       
4664

       
4294

       
 

       
		return err


     

       
4665

       
4295

       
 

       
	}


     

       
4666

       

       
-

       



     

       
4667

       

       
-

       
	if len(t.Steps) > 8192 {


     

       
4668

       

       
-

       
		return xerrors.Errorf("Slice value in field t.Steps was too long")


     

       
4669

       

       
-

       
	}


     

       
4670

       

       
-

       



     

       
4671

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Steps))); err != nil {


     

       
4672

       

       
-

       
		return err


     

       
4673

       

       
-

       
	}


     

       
4674

       

       
-

       
	for _, v := range t.Steps {


     

       
4675

       

       
-

       
		if err := v.MarshalCBOR(cw); err != nil {


     

       
4676

       

       
-

       
			return err


     

       
4677

       

       
-

       
		}


     

       
4678

       

       
-

       



     

       
4679

       

       
-

       
	}


     

       
4680

       

       
-

       



     

       
4681

       

       
-

       
	// t.Environment ([]*tangled.Pipeline_Pair) (slice)


     

       
4682

       

       
-

       
	if len("environment") > 1000000 {


     

       
4683

       

       
-

       
		return xerrors.Errorf("Value in field \"environment\" was too long")


     

       
4684

       

       
-

       
	}


     

       
4685

       

       
-

       



     

       
4686

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("environment"))); err != nil {


     

       
4687

       

       
-

       
		return err


     

       
4688

       

       
-

       
	}


     

       
4689

       

       
-

       
	if _, err := cw.WriteString(string("environment")); err != nil {


     

       
4690

       

       
-

       
		return err


     

       
4691

       

       
-

       
	}


     

       
4692

       

       
-

       



     

       
4693

       

       
-

       
	if len(t.Environment) > 8192 {


     

       
4694

       

       
-

       
		return xerrors.Errorf("Slice value in field t.Environment was too long")


     

       
4695

       

       
-

       
	}


     

       
4696

       

       
-

       



     

       
4697

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Environment))); err != nil {


     

       

       
4296

       
+

       
	if _, err := cw.WriteString(string("engine")); err != nil {


     

       
4698

       
4297

       
 

       
		return err


     

       
4699

       
4298

       
 

       
	}


     

       
4700

       

       
-

       
	for _, v := range t.Environment {


     

       
4701

       

       
-

       
		if err := v.MarshalCBOR(cw); err != nil {


     

       
4702

       

       
-

       
			return err


     

       
4703

       

       
-

       
		}


     

       
4704

       
4299

       
 

       



     

       

       
4300

       
+

       
	if len(t.Engine) > 1000000 {


     

       

       
4301

       
+

       
		return xerrors.Errorf("Value in field t.Engine was too long")


     

       
4705

       
4302

       
 

       
	}


     

       
4706

       
4303

       
 

       



     

       
4707

       

       
-

       
	// t.Dependencies ([]*tangled.Pipeline_Dependency) (slice)


     

       
4708

       

       
-

       
	if len("dependencies") > 1000000 {


     

       
4709

       

       
-

       
		return xerrors.Errorf("Value in field \"dependencies\" was too long")


     

       
4710

       

       
-

       
	}


     

       
4711

       

       
-

       



     

       
4712

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("dependencies"))); err != nil {


     

       

       
4304

       
+

       
	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Engine))); err != nil {


     

       
4713

       
4305

       
 

       
		return err


     

       
4714

       
4306

       
 

       
	}


     

       
4715

       

       
-

       
	if _, err := cw.WriteString(string("dependencies")); err != nil {


     

       

       
4307

       
+

       
	if _, err := cw.WriteString(string(t.Engine)); err != nil {


     

       
4716

       
4308

       
 

       
		return err


     

       
4717

       

       
-

       
	}


     

       
4718

       

       
-

       



     

       
4719

       

       
-

       
	if len(t.Dependencies) > 8192 {


     

       
4720

       

       
-

       
		return xerrors.Errorf("Slice value in field t.Dependencies was too long")


     

       
4721

       

       
-

       
	}


     

       
4722

       

       
-

       



     

       
4723

       

       
-

       
	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Dependencies))); err != nil {


     

       
4724

       

       
-

       
		return err


     

       
4725

       

       
-

       
	}


     

       
4726

       

       
-

       
	for _, v := range t.Dependencies {


     

       
4727

       

       
-

       
		if err := v.MarshalCBOR(cw); err != nil {


     

       
4728

       

       
-

       
			return err


     

       
4729

       

       
-

       
		}


     

       
4730

       

       
-

       



     

       
4731

       
4309

       
 

       
	}


     

       
4732

       
4310

       
 

       
	return nil


     

       
4733

       
4311

       
 

       
}


     
···

       
4757

       
4335

       
 

       



     

       
4758

       
4336

       
 

       
	n := extra


     

       
4759

       
4337

       
 

       



     

       
4760

       

       
-

       
	nameBuf := make([]byte, 12)


     

       

       
4338

       
+

       
	nameBuf := make([]byte, 6)


     

       
4761

       
4339

       
 

       
	for i := uint64(0); i < n; i++ {


     

       
4762

       
4340

       
 

       
		nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)


     

       
4763

       
4341

       
 

       
		if err != nil {


     
···

       
4773

       
4351

       
 

       
		}


     

       
4774

       
4352

       
 

       



     

       
4775

       
4353

       
 

       
		switch string(nameBuf[:nameLen]) {


     

       
4776

       

       
-

       
		// t.Name (string) (string)


     

       

       
4354

       
+

       
		// t.Raw (string) (string)


     

       

       
4355

       
+

       
		case "raw":


     

       

       
4356

       
+

       



     

       

       
4357

       
+

       
			{


     

       

       
4358

       
+

       
				sval, err := cbg.ReadStringWithMax(cr, 1000000)


     

       

       
4359

       
+

       
				if err != nil {


     

       

       
4360

       
+

       
					return err


     

       

       
4361

       
+

       
				}


     

       

       
4362

       
+

       



     

       

       
4363

       
+

       
				t.Raw = string(sval)


     

       

       
4364

       
+

       
			}


     

       

       
4365

       
+

       
			// t.Name (string) (string)


     

       
4777

       
4366

       
 

       
		case "name":


     

       
4778

       
4367

       
 

       



     

       
4779

       
4368

       
 

       
			{


     
···

       
4804

       
4393

       
 

       
				}


     

       
4805

       
4394

       
 

       



     

       
4806

       
4395

       
 

       
			}


     

       
4807

       

       
-

       
			// t.Steps ([]*tangled.Pipeline_Step) (slice)


     

       
4808

       

       
-

       
		case "steps":


     

       

       
4396

       
+

       
			// t.Engine (string) (string)


     

       

       
4397

       
+

       
		case "engine":


     

       
4809

       
4398

       
 

       



     

       
4810

       

       
-

       
			maj, extra, err = cr.ReadHeader()


     

       
4811

       

       
-

       
			if err != nil {


     

       
4812

       

       
-

       
				return err


     

       
4813

       

       
-

       
			}


     

       
4814

       

       
-

       



     

       
4815

       

       
-

       
			if extra > 8192 {


     

       
4816

       

       
-

       
				return fmt.Errorf("t.Steps: array too large (%d)", extra)


     

       
4817

       

       
-

       
			}


     

       
4818

       

       
-

       



     

       
4819

       

       
-

       
			if maj != cbg.MajArray {


     

       
4820

       

       
-

       
				return fmt.Errorf("expected cbor array")


     

       
4821

       

       
-

       
			}


     

       
4822

       

       
-

       



     

       
4823

       

       
-

       
			if extra > 0 {


     

       
4824

       

       
-

       
				t.Steps = make([]*Pipeline_Step, extra)


     

       
4825

       

       
-

       
			}


     

       
4826

       

       
-

       



     

       
4827

       

       
-

       
			for i := 0; i < int(extra); i++ {


     

       
4828

       

       
-

       
				{


     

       
4829

       

       
-

       
					var maj byte


     

       
4830

       

       
-

       
					var extra uint64


     

       
4831

       

       
-

       
					var err error


     

       
4832

       

       
-

       
					_ = maj


     

       
4833

       

       
-

       
					_ = extra


     

       
4834

       

       
-

       
					_ = err


     

       
4835

       

       
-

       



     

       
4836

       

       
-

       
					{


     

       
4837

       

       
-

       



     

       
4838

       

       
-

       
						b, err := cr.ReadByte()


     

       
4839

       

       
-

       
						if err != nil {


     

       
4840

       

       
-

       
							return err


     

       
4841

       

       
-

       
						}


     

       
4842

       

       
-

       
						if b != cbg.CborNull[0] {


     

       
4843

       

       
-

       
							if err := cr.UnreadByte(); err != nil {


     

       
4844

       

       
-

       
								return err


     

       
4845

       

       
-

       
							}


     

       
4846

       

       
-

       
							t.Steps[i] = new(Pipeline_Step)


     

       
4847

       

       
-

       
							if err := t.Steps[i].UnmarshalCBOR(cr); err != nil {


     

       
4848

       

       
-

       
								return xerrors.Errorf("unmarshaling t.Steps[i] pointer: %w", err)


     

       
4849

       

       
-

       
							}


     

       
4850

       

       
-

       
						}


     

       
4851

       

       
-

       



     

       
4852

       

       
-

       
					}


     

       
4853

       

       
-

       



     

       

       
4399

       
+

       
			{


     

       

       
4400

       
+

       
				sval, err := cbg.ReadStringWithMax(cr, 1000000)


     

       

       
4401

       
+

       
				if err != nil {


     

       

       
4402

       
+

       
					return err


     

       
4854

       
4403

       
 

       
				}


     

       
4855

       

       
-

       
			}


     

       
4856

       

       
-

       
			// t.Environment ([]*tangled.Pipeline_Pair) (slice)


     

       
4857

       

       
-

       
		case "environment":


     

       
4858

       
4404

       
 

       



     

       
4859

       

       
-

       
			maj, extra, err = cr.ReadHeader()


     

       
4860

       

       
-

       
			if err != nil {


     

       
4861

       

       
-

       
				return err


     

       
4862

       

       
-

       
			}


     

       
4863

       

       
-

       



     

       
4864

       

       
-

       
			if extra > 8192 {


     

       
4865

       

       
-

       
				return fmt.Errorf("t.Environment: array too large (%d)", extra)


     

       
4866

       

       
-

       
			}


     

       
4867

       

       
-

       



     

       
4868

       

       
-

       
			if maj != cbg.MajArray {


     

       
4869

       

       
-

       
				return fmt.Errorf("expected cbor array")


     

       
4870

       

       
-

       
			}


     

       
4871

       

       
-

       



     

       
4872

       

       
-

       
			if extra > 0 {


     

       
4873

       

       
-

       
				t.Environment = make([]*Pipeline_Pair, extra)


     

       
4874

       

       
-

       
			}


     

       
4875

       

       
-

       



     

       
4876

       

       
-

       
			for i := 0; i < int(extra); i++ {


     

       
4877

       

       
-

       
				{


     

       
4878

       

       
-

       
					var maj byte


     

       
4879

       

       
-

       
					var extra uint64


     

       
4880

       

       
-

       
					var err error


     

       
4881

       

       
-

       
					_ = maj


     

       
4882

       

       
-

       
					_ = extra


     

       
4883

       

       
-

       
					_ = err


     

       
4884

       

       
-

       



     

       
4885

       

       
-

       
					{


     

       
4886

       

       
-

       



     

       
4887

       

       
-

       
						b, err := cr.ReadByte()


     

       
4888

       

       
-

       
						if err != nil {


     

       
4889

       

       
-

       
							return err


     

       
4890

       

       
-

       
						}


     

       
4891

       

       
-

       
						if b != cbg.CborNull[0] {


     

       
4892

       

       
-

       
							if err := cr.UnreadByte(); err != nil {


     

       
4893

       

       
-

       
								return err


     

       
4894

       

       
-

       
							}


     

       
4895

       

       
-

       
							t.Environment[i] = new(Pipeline_Pair)


     

       
4896

       

       
-

       
							if err := t.Environment[i].UnmarshalCBOR(cr); err != nil {


     

       
4897

       

       
-

       
								return xerrors.Errorf("unmarshaling t.Environment[i] pointer: %w", err)


     

       
4898

       

       
-

       
							}


     

       
4899

       

       
-

       
						}


     

       
4900

       

       
-

       



     

       
4901

       

       
-

       
					}


     

       
4902

       

       
-

       



     

       
4903

       

       
-

       
				}


     

       
4904

       

       
-

       
			}


     

       
4905

       

       
-

       
			// t.Dependencies ([]*tangled.Pipeline_Dependency) (slice)


     

       
4906

       

       
-

       
		case "dependencies":


     

       
4907

       

       
-

       



     

       
4908

       

       
-

       
			maj, extra, err = cr.ReadHeader()


     

       
4909

       

       
-

       
			if err != nil {


     

       
4910

       

       
-

       
				return err


     

       
4911

       

       
-

       
			}


     

       
4912

       

       
-

       



     

       
4913

       

       
-

       
			if extra > 8192 {


     

       
4914

       

       
-

       
				return fmt.Errorf("t.Dependencies: array too large (%d)", extra)


     

       
4915

       

       
-

       
			}


     

       
4916

       

       
-

       



     

       
4917

       

       
-

       
			if maj != cbg.MajArray {


     

       
4918

       

       
-

       
				return fmt.Errorf("expected cbor array")


     

       
4919

       

       
-

       
			}


     

       
4920

       

       
-

       



     

       
4921

       

       
-

       
			if extra > 0 {


     

       
4922

       

       
-

       
				t.Dependencies = make([]*Pipeline_Dependency, extra)


     

       
4923

       

       
-

       
			}


     

       
4924

       

       
-

       



     

       
4925

       

       
-

       
			for i := 0; i < int(extra); i++ {


     

       
4926

       

       
-

       
				{


     

       
4927

       

       
-

       
					var maj byte


     

       
4928

       

       
-

       
					var extra uint64


     

       
4929

       

       
-

       
					var err error


     

       
4930

       

       
-

       
					_ = maj


     

       
4931

       

       
-

       
					_ = extra


     

       
4932

       

       
-

       
					_ = err


     

       
4933

       

       
-

       



     

       
4934

       

       
-

       
					{


     

       
4935

       

       
-

       



     

       
4936

       

       
-

       
						b, err := cr.ReadByte()


     

       
4937

       

       
-

       
						if err != nil {


     

       
4938

       

       
-

       
							return err


     

       
4939

       

       
-

       
						}


     

       
4940

       

       
-

       
						if b != cbg.CborNull[0] {


     

       
4941

       

       
-

       
							if err := cr.UnreadByte(); err != nil {


     

       
4942

       

       
-

       
								return err


     

       
4943

       

       
-

       
							}


     

       
4944

       

       
-

       
							t.Dependencies[i] = new(Pipeline_Dependency)


     

       
4945

       

       
-

       
							if err := t.Dependencies[i].UnmarshalCBOR(cr); err != nil {


     

       
4946

       

       
-

       
								return xerrors.Errorf("unmarshaling t.Dependencies[i] pointer: %w", err)


     

       
4947

       

       
-

       
							}


     

       
4948

       

       
-

       
						}


     

       
4949

       

       
-

       



     

       
4950

       

       
-

       
					}


     

       
4951

       

       
-

       



     

       
4952

       

       
-

       
				}


     

       

       
4405

       
+

       
				t.Engine = string(sval)


     

       
4953

       
4406

       
 

       
			}


     

       
4954

       
4407

       
 

       



     

       
4955

       
4408

       
 

       
		default:
+4 -18
api/tangled/tangledpipeline.go 
···

       
29

       
29

       
 

       
	Submodules bool  `json:"submodules" cborgen:"submodules"`


     

       
30

       
30

       
 

       
}


     

       
31

       
31

       
 

       



     

       
32

       

       
-

       
// Pipeline_Dependency is a "dependency" in the sh.tangled.pipeline schema.


     

       
33

       

       
-

       
type Pipeline_Dependency struct {


     

       
34

       

       
-

       
	Packages []string `json:"packages" cborgen:"packages"`


     

       
35

       

       
-

       
	Registry string   `json:"registry" cborgen:"registry"`


     

       
36

       

       
-

       
}


     

       
37

       

       
-

       



     

       
38

       
32

       
 

       
// Pipeline_ManualTriggerData is a "manualTriggerData" in the sh.tangled.pipeline schema.


     

       
39

       
33

       
 

       
type Pipeline_ManualTriggerData struct {


     

       
40

       
34

       
 

       
	Inputs []*Pipeline_Pair `json:"inputs,omitempty" cborgen:"inputs,omitempty"`


     
···

       
61

       
55

       
 

       
	Ref    string `json:"ref" cborgen:"ref"`


     

       
62

       
56

       
 

       
}


     

       
63

       
57

       
 

       



     

       
64

       

       
-

       
// Pipeline_Step is a "step" in the sh.tangled.pipeline schema.


     

       
65

       

       
-

       
type Pipeline_Step struct {


     

       
66

       

       
-

       
	Command     string           `json:"command" cborgen:"command"`


     

       
67

       

       
-

       
	Environment []*Pipeline_Pair `json:"environment,omitempty" cborgen:"environment,omitempty"`


     

       
68

       

       
-

       
	Name        string           `json:"name" cborgen:"name"`


     

       
69

       

       
-

       
}


     

       
70

       

       
-

       



     

       
71

       
58

       
 

       
// Pipeline_TriggerMetadata is a "triggerMetadata" in the sh.tangled.pipeline schema.


     

       
72

       
59

       
 

       
type Pipeline_TriggerMetadata struct {


     

       
73

       
60

       
 

       
	Kind        string                           `json:"kind" cborgen:"kind"`


     
···

       
87

       
74

       
 

       



     

       
88

       
75

       
 

       
// Pipeline_Workflow is a "workflow" in the sh.tangled.pipeline schema.


     

       
89

       
76

       
 

       
type Pipeline_Workflow struct {


     

       
90

       

       
-

       
	Clone        *Pipeline_CloneOpts    `json:"clone" cborgen:"clone"`


     

       
91

       

       
-

       
	Dependencies []*Pipeline_Dependency `json:"dependencies" cborgen:"dependencies"`


     

       
92

       

       
-

       
	Environment  []*Pipeline_Pair       `json:"environment" cborgen:"environment"`


     

       
93

       

       
-

       
	Name         string                 `json:"name" cborgen:"name"`


     

       
94

       

       
-

       
	Steps        []*Pipeline_Step       `json:"steps" cborgen:"steps"`


     

       

       
77

       
+

       
	Clone  *Pipeline_CloneOpts `json:"clone" cborgen:"clone"`


     

       

       
78

       
+

       
	Engine string              `json:"engine" cborgen:"engine"`


     

       

       
79

       
+

       
	Name   string              `json:"name" cborgen:"name"`


     

       

       
80

       
+

       
	Raw    string              `json:"raw" cborgen:"raw"`


     

       
95

       
81

       
 

       
}
-2
cmd/gen.go 
···

       
27

       
27

       
 

       
		tangled.KnotMember{},


     

       
28

       
28

       
 

       
		tangled.Pipeline{},


     

       
29

       
29

       
 

       
		tangled.Pipeline_CloneOpts{},


     

       
30

       

       
-

       
		tangled.Pipeline_Dependency{},


     

       
31

       
30

       
 

       
		tangled.Pipeline_ManualTriggerData{},


     

       
32

       
31

       
 

       
		tangled.Pipeline_Pair{},


     

       
33

       
32

       
 

       
		tangled.Pipeline_PullRequestTriggerData{},


     

       
34

       
33

       
 

       
		tangled.Pipeline_PushTriggerData{},


     

       
35

       
34

       
 

       
		tangled.PipelineStatus{},


     

       
36

       

       
-

       
		tangled.Pipeline_Step{},


     

       
37

       
35

       
 

       
		tangled.Pipeline_TriggerMetadata{},


     

       
38

       
36

       
 

       
		tangled.Pipeline_TriggerRepo{},


     

       
39

       
37

       
 

       
		tangled.Pipeline_Workflow{},
+26 -3
docs/spindle/pipeline.md 
···

       
4

       
4

       
 

       
repo. Generally:


     

       
5

       
5

       
 

       



     

       
6

       
6

       
 

       
* Pipelines are defined in YAML.


     

       
7

       

       
-

       
* Dependencies can be specified from


     

       
8

       

       
-

       
[Nixpkgs](https://search.nixos.org) or custom registries.


     

       
9

       

       
-

       
* Environment variables can be set globally or per-step.


     

       

       
7

       
+

       
* Workflows can run using different *engines*.


     

       

       
8

       
+

       



     

       

       
9

       
+

       
The most barebones workflow looks like this:


     

       

       
10

       
+

       



     

       

       
11

       
+

       
```yaml


     

       

       
12

       
+

       
when:


     

       

       
13

       
+

       
  - event: ["push"]


     

       

       
14

       
+

       
    branch: ["main"]


     

       

       
15

       
+

       



     

       

       
16

       
+

       
engine: "nixery"


     

       

       
17

       
+

       



     

       

       
18

       
+

       
# optional


     

       

       
19

       
+

       
clone:


     

       

       
20

       
+

       
  skip: false


     

       

       
21

       
+

       
  depth: 50


     

       

       
22

       
+

       
  submodules: true


     

       

       
23

       
+

       
```


     

       

       
24

       
+

       



     

       

       
25

       
+

       
The `when` and `engine` fields are required, while every other aspect


     

       

       
26

       
+

       
of how the definition is parsed is up to the engine. Currently, a spindle


     

       

       
27

       
+

       
provides at least one of these built-in engines:


     

       

       
28

       
+

       



     

       

       
29

       
+

       
## `nixery`


     

       

       
30

       
+

       



     

       

       
31

       
+

       
The Nixery engine uses an instance of [Nixery](https://nixery.dev) to run


     

       

       
32

       
+

       
steps that use dependencies from [Nixpkgs](https://github.com/NixOS/nixpkgs).


     

       
10

       
33

       
 

       



     

       
11

       
34

       
 

       
Here's an example that uses all fields:


     

       
12

       
35
+7 -63
lexicons/pipeline/pipeline.json 
···

       
149

       
149

       
 

       
      "type": "object",


     

       
150

       
150

       
 

       
      "required": [


     

       
151

       
151

       
 

       
        "name",


     

       
152

       

       
-

       
        "dependencies",


     

       
153

       

       
-

       
        "steps",


     

       
154

       

       
-

       
        "environment",


     

       
155

       

       
-

       
        "clone"


     

       

       
152

       
+

       
        "engine",


     

       

       
153

       
+

       
        "clone",


     

       

       
154

       
+

       
        "raw"


     

       
156

       
155

       
 

       
      ],


     

       
157

       
156

       
 

       
      "properties": {


     

       
158

       
157

       
 

       
        "name": {


     

       
159

       
158

       
 

       
          "type": "string"


     

       
160

       
159

       
 

       
        },


     

       
161

       

       
-

       
        "dependencies": {


     

       
162

       

       
-

       
          "type": "array",


     

       
163

       

       
-

       
          "items": {


     

       
164

       

       
-

       
            "type": "ref",


     

       
165

       

       
-

       
            "ref": "#dependency"


     

       
166

       

       
-

       
          }


     

       
167

       

       
-

       
        },


     

       
168

       

       
-

       
        "steps": {


     

       
169

       

       
-

       
          "type": "array",


     

       
170

       

       
-

       
          "items": {


     

       
171

       

       
-

       
            "type": "ref",


     

       
172

       

       
-

       
            "ref": "#step"


     

       
173

       

       
-

       
          }


     

       
174

       

       
-

       
        },


     

       
175

       

       
-

       
        "environment": {


     

       
176

       

       
-

       
          "type": "array",


     

       
177

       

       
-

       
          "items": {


     

       
178

       

       
-

       
            "type": "ref",


     

       
179

       

       
-

       
            "ref": "#pair"


     

       
180

       

       
-

       
          }


     

       

       
160

       
+

       
        "engine": {


     

       

       
161

       
+

       
          "type": "string"


     

       
181

       
162

       
 

       
        },


     

       
182

       
163

       
 

       
        "clone": {


     

       
183

       
164

       
 

       
          "type": "ref",


     

       
184

       
165

       
 

       
          "ref": "#cloneOpts"


     

       
185

       

       
-

       
        }


     

       
186

       

       
-

       
      }


     

       
187

       

       
-

       
    },


     

       
188

       

       
-

       
    "dependency": {


     

       
189

       

       
-

       
      "type": "object",


     

       
190

       

       
-

       
      "required": [


     

       
191

       

       
-

       
        "registry",


     

       
192

       

       
-

       
        "packages"


     

       
193

       

       
-

       
      ],


     

       
194

       

       
-

       
      "properties": {


     

       
195

       

       
-

       
        "registry": {


     

       

       
166

       
+

       
        },


     

       

       
167

       
+

       
        "raw": {


     

       
196

       
168

       
 

       
          "type": "string"


     

       
197

       

       
-

       
        },


     

       
198

       

       
-

       
        "packages": {


     

       
199

       

       
-

       
          "type": "array",


     

       
200

       

       
-

       
          "items": {


     

       
201

       

       
-

       
            "type": "string"


     

       
202

       

       
-

       
          }


     

       
203

       
169

       
 

       
        }


     

       
204

       
170

       
 

       
      }


     

       
205

       
171

       
 

       
    },


     
···

       
219

       
185

       
 

       
        },


     

       
220

       
186

       
 

       
        "submodules": {


     

       
221

       
187

       
 

       
          "type": "boolean"


     

       
222

       

       
-

       
        }


     

       
223

       

       
-

       
      }


     

       
224

       

       
-

       
    },


     

       
225

       

       
-

       
    "step": {


     

       
226

       

       
-

       
      "type": "object",


     

       
227

       

       
-

       
      "required": [


     

       
228

       

       
-

       
        "name",


     

       
229

       

       
-

       
        "command"


     

       
230

       

       
-

       
      ],


     

       
231

       

       
-

       
      "properties": {


     

       
232

       

       
-

       
        "name": {


     

       
233

       

       
-

       
          "type": "string"


     

       
234

       

       
-

       
        },


     

       
235

       

       
-

       
        "command": {


     

       
236

       

       
-

       
          "type": "string"


     

       
237

       

       
-

       
        },


     

       
238

       

       
-

       
        "environment": {


     

       
239

       

       
-

       
          "type": "array",


     

       
240

       

       
-

       
          "items": {


     

       
241

       

       
-

       
            "type": "ref",


     

       
242

       

       
-

       
            "ref": "#pair"


     

       
243

       

       
-

       
          }


     

       
244

       
188

       
 

       
        }


     

       
245

       
189

       
 

       
      }


     

       
246

       
190

       
 

       
    },
+2 -2
nix/modules/spindle.nix 
···

       
111

       
111

       
 

       
            "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"


     

       
112

       
112

       
 

       
            "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"


     

       
113

       
113

       
 

       
            "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"


     

       
114

       

       
-

       
            "SPINDLE_PIPELINES_NIXERY=${cfg.pipelines.nixery}"


     

       
115

       

       
-

       
            "SPINDLE_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"


     

       

       
114

       
+

       
            "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}"


     

       

       
115

       
+

       
            "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"


     

       
116

       
116

       
 

       
          ];


     

       
117

       
117

       
 

       
          ExecStart = "${cfg.package}/bin/spindle";


     

       
118

       
118

       
 

       
          Restart = "always";
+4 -4
spindle/config/config.go 
···

       
16

       
16

       
 

       
	Dev               bool    `env:"DEV, default=false"`


     

       
17

       
17

       
 

       
	Owner             string  `env:"OWNER, required"`


     

       
18

       
18

       
 

       
	Secrets           Secrets `env:",prefix=SECRETS_"`


     

       

       
19

       
+

       
	LogDir            string  `env:"LOG_DIR, default=/var/log/spindle"`


     

       
19

       
20

       
 

       
}


     

       
20

       
21

       
 

       



     

       
21

       
22

       
 

       
func (s Server) Did() syntax.DID {


     
···

       
32

       
33

       
 

       
	Mount     string `env:"MOUNT, default=spindle"`


     

       
33

       
34

       
 

       
}


     

       
34

       
35

       
 

       



     

       
35

       

       
-

       
type Pipelines struct {


     

       

       
36

       
+

       
type NixeryPipelines struct {


     

       
36

       
37

       
 

       
	Nixery          string `env:"NIXERY, default=nixery.tangled.sh"`


     

       
37

       
38

       
 

       
	WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"`


     

       
38

       

       
-

       
	LogDir          string `env:"LOG_DIR, default=/var/log/spindle"`


     

       
39

       
39

       
 

       
}


     

       
40

       
40

       
 

       



     

       
41

       
41

       
 

       
type Config struct {


     

       
42

       

       
-

       
	Server    Server    `env:",prefix=SPINDLE_SERVER_"`


     

       
43

       

       
-

       
	Pipelines Pipelines `env:",prefix=SPINDLE_PIPELINES_"`


     

       

       
42

       
+

       
	Server          Server          `env:",prefix=SPINDLE_SERVER_"`


     

       

       
43

       
+

       
	NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"`


     

       
44

       
44

       
 

       
}


     

       
45

       
45

       
 

       



     

       
46

       
46

       
 

       
func Load(ctx context.Context) (*Config, error) {
+1 -1
spindle/engine/ansi_stripper.go spindle/engines/nixery/ansi_stripper.go 
···

       
1

       

       
-

       
package engine


     

       

       
1

       
+

       
package nixery


     

       
2

       
2

       
 

       



     

       
3

       
3

       
 

       
import (


     

       
4

       
4

       
 

       
	"io"
+64 -416
spindle/engine/engine.go 
···

       
4

       
4

       
 

       
	"context"


     

       
5

       
5

       
 

       
	"errors"


     

       
6

       
6

       
 

       
	"fmt"


     

       
7

       

       
-

       
	"io"


     

       
8

       
7

       
 

       
	"log/slog"


     

       
9

       

       
-

       
	"os"


     

       
10

       

       
-

       
	"strings"


     

       
11

       

       
-

       
	"sync"


     

       
12

       

       
-

       
	"time"


     

       
13

       
8

       
 

       



     

       
14

       
9

       
 

       
	securejoin "github.com/cyphar/filepath-securejoin"


     

       
15

       

       
-

       
	"github.com/docker/docker/api/types/container"


     

       
16

       

       
-

       
	"github.com/docker/docker/api/types/image"


     

       
17

       

       
-

       
	"github.com/docker/docker/api/types/mount"


     

       
18

       

       
-

       
	"github.com/docker/docker/api/types/network"


     

       
19

       

       
-

       
	"github.com/docker/docker/api/types/volume"


     

       
20

       

       
-

       
	"github.com/docker/docker/client"


     

       
21

       

       
-

       
	"github.com/docker/docker/pkg/stdcopy"


     

       
22

       
10

       
 

       
	"golang.org/x/sync/errgroup"


     

       
23

       

       
-

       
	"tangled.sh/tangled.sh/core/log"


     

       
24

       
11

       
 

       
	"tangled.sh/tangled.sh/core/notifier"


     

       
25

       
12

       
 

       
	"tangled.sh/tangled.sh/core/spindle/config"


     

       
26

       
13

       
 

       
	"tangled.sh/tangled.sh/core/spindle/db"


     
···

       
28

       
15

       
 

       
	"tangled.sh/tangled.sh/core/spindle/secrets"


     

       
29

       
16

       
 

       
)


     

       
30

       
17

       
 

       



     

       
31

       

       
-

       
const (


     

       
32

       

       
-

       
	workspaceDir = "/tangled/workspace"


     

       

       
18

       
+

       
var (


     

       

       
19

       
+

       
	ErrTimedOut       = errors.New("timed out")


     

       

       
20

       
+

       
	ErrWorkflowFailed = errors.New("workflow failed")


     

       
33

       
21

       
 

       
)


     

       
34

       
22

       
 

       



     

       
35

       

       
-

       
type cleanupFunc func(context.Context) error


     

       
36

       

       
-

       



     

       
37

       

       
-

       
type Engine struct {


     

       
38

       

       
-

       
	docker client.APIClient


     

       
39

       

       
-

       
	l      *slog.Logger


     

       
40

       

       
-

       
	db     *db.DB


     

       
41

       

       
-

       
	n      *notifier.Notifier


     

       
42

       

       
-

       
	cfg    *config.Config


     

       
43

       

       
-

       
	vault  secrets.Manager


     

       
44

       

       
-

       



     

       
45

       

       
-

       
	cleanupMu sync.Mutex


     

       
46

       

       
-

       
	cleanup   map[string][]cleanupFunc


     

       
47

       

       
-

       
}


     

       
48

       

       
-

       



     

       
49

       

       
-

       
func New(ctx context.Context, cfg *config.Config, db *db.DB, n *notifier.Notifier, vault secrets.Manager) (*Engine, error) {


     

       
50

       

       
-

       
	dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())


     

       
51

       

       
-

       
	if err != nil {


     

       
52

       

       
-

       
		return nil, err


     

       
53

       

       
-

       
	}


     

       
54

       

       
-

       



     

       
55

       

       
-

       
	l := log.FromContext(ctx).With("component", "spindle")


     

       
56

       

       
-

       



     

       
57

       

       
-

       
	e := &Engine{


     

       
58

       

       
-

       
		docker: dcli,


     

       
59

       

       
-

       
		l:      l,


     

       
60

       

       
-

       
		db:     db,


     

       
61

       

       
-

       
		n:      n,


     

       
62

       

       
-

       
		cfg:    cfg,


     

       
63

       

       
-

       
		vault:  vault,


     

       
64

       

       
-

       
	}


     

       
65

       

       
-

       



     

       
66

       

       
-

       
	e.cleanup = make(map[string][]cleanupFunc)


     

       
67

       

       
-

       



     

       
68

       

       
-

       
	return e, nil


     

       
69

       

       
-

       
}


     

       
70

       

       
-

       



     

       
71

       

       
-

       
func (e *Engine) StartWorkflows(ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) {


     

       
72

       

       
-

       
	e.l.Info("starting all workflows in parallel", "pipeline", pipelineId)


     

       

       
23

       
+

       
func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) {


     

       

       
24

       
+

       
	l.Info("starting all workflows in parallel", "pipeline", pipelineId)


     

       
73

       
25

       
 

       



     

       
74

       
26

       
 

       
	// extract secrets


     

       
75

       
27

       
 

       
	var allSecrets []secrets.UnlockedSecret


     

       
76

       
28

       
 

       
	if didSlashRepo, err := securejoin.SecureJoin(pipeline.RepoOwner, pipeline.RepoName); err == nil {


     

       
77

       

       
-

       
		if res, err := e.vault.GetSecretsUnlocked(ctx, secrets.DidSlashRepo(didSlashRepo)); err == nil {


     

       

       
29

       
+

       
		if res, err := vault.GetSecretsUnlocked(ctx, secrets.DidSlashRepo(didSlashRepo)); err == nil {


     

       
78

       
30

       
 

       
			allSecrets = res


     

       
79

       
31

       
 

       
		}


     

       
80

       
32

       
 

       
	}


     

       
81

       
33

       
 

       



     

       
82

       

       
-

       
	workflowTimeoutStr := e.cfg.Pipelines.WorkflowTimeout


     

       
83

       

       
-

       
	workflowTimeout, err := time.ParseDuration(workflowTimeoutStr)


     

       
84

       

       
-

       
	if err != nil {


     

       
85

       

       
-

       
		e.l.Error("failed to parse workflow timeout", "error", err, "timeout", workflowTimeoutStr)


     

       
86

       

       
-

       
		workflowTimeout = 5 * time.Minute


     

       
87

       

       
-

       
	}


     

       
88

       

       
-

       
	e.l.Info("using workflow timeout", "timeout", workflowTimeout)


     

       
89

       

       
-

       



     

       
90

       
34

       
 

       
	eg, ctx := errgroup.WithContext(ctx)


     

       
91

       

       
-

       
	for _, w := range pipeline.Workflows {


     

       
92

       

       
-

       
		eg.Go(func() error {


     

       
93

       

       
-

       
			wid := models.WorkflowId{


     

       
94

       

       
-

       
				PipelineId: pipelineId,


     

       
95

       

       
-

       
				Name:       w.Name,


     

       
96

       

       
-

       
			}


     

       
97

       

       
-

       



     

       
98

       

       
-

       
			err := e.db.StatusRunning(wid, e.n)


     

       
99

       

       
-

       
			if err != nil {


     

       
100

       

       
-

       
				return err


     

       
101

       

       
-

       
			}


     

       

       
35

       
+

       
	for eng, wfs := range pipeline.Workflows {


     

       

       
36

       
+

       
		workflowTimeout := eng.WorkflowTimeout()


     

       

       
37

       
+

       
		l.Info("using workflow timeout", "timeout", workflowTimeout)


     

       
102

       
38

       
 

       



     

       
103

       

       
-

       
			err = e.SetupWorkflow(ctx, wid)


     

       
104

       

       
-

       
			if err != nil {


     

       
105

       

       
-

       
				e.l.Error("setting up worklow", "wid", wid, "err", err)


     

       
106

       

       
-

       
				return err


     

       
107

       

       
-

       
			}


     

       
108

       

       
-

       
			defer e.DestroyWorkflow(ctx, wid)


     

       
109

       

       
-

       



     

       
110

       

       
-

       
			reader, err := e.docker.ImagePull(ctx, w.Image, image.PullOptions{})


     

       
111

       

       
-

       
			if err != nil {


     

       
112

       

       
-

       
				e.l.Error("pipeline image pull failed!", "image", w.Image, "workflowId", wid, "error", err.Error())


     

       

       
39

       
+

       
		for _, w := range wfs {


     

       

       
40

       
+

       
			eg.Go(func() error {


     

       

       
41

       
+

       
				wid := models.WorkflowId{


     

       

       
42

       
+

       
					PipelineId: pipelineId,


     

       

       
43

       
+

       
					Name:       w.Name,


     

       

       
44

       
+

       
				}


     

       
113

       
45

       
 

       



     

       
114

       

       
-

       
				err := e.db.StatusFailed(wid, err.Error(), -1, e.n)


     

       

       
46

       
+

       
				err := db.StatusRunning(wid, n)


     

       
115

       
47

       
 

       
				if err != nil {


     

       
116

       
48

       
 

       
					return err


     

       
117

       
49

       
 

       
				}


     

       
118

       
50

       
 

       



     

       
119

       

       
-

       
				return fmt.Errorf("pulling image: %w", err)


     

       
120

       

       
-

       
			}


     

       
121

       

       
-

       
			defer reader.Close()


     

       
122

       

       
-

       
			io.Copy(os.Stdout, reader)


     

       
123

       

       
-

       



     

       
124

       

       
-

       
			ctx, cancel := context.WithTimeout(ctx, workflowTimeout)


     

       
125

       

       
-

       
			defer cancel()


     

       

       
51

       
+

       
				err = eng.SetupWorkflow(ctx, wid, &w)


     

       

       
52

       
+

       
				if err != nil {


     

       

       
53

       
+

       
					// TODO(winter): Should this always set StatusFailed?


     

       

       
54

       
+

       
					// In the original, we only do in a subset of cases.


     

       

       
55

       
+

       
					l.Error("setting up worklow", "wid", wid, "err", err)


     

       
126

       
56

       
 

       



     

       
127

       

       
-

       
			err = e.StartSteps(ctx, wid, w, allSecrets)


     

       
128

       

       
-

       
			if err != nil {


     

       
129

       

       
-

       
				if errors.Is(err, ErrTimedOut) {


     

       
130

       

       
-

       
					dbErr := e.db.StatusTimeout(wid, e.n)


     

       
131

       

       
-

       
					if dbErr != nil {


     

       
132

       

       
-

       
						return dbErr


     

       
133

       

       
-

       
					}


     

       
134

       

       
-

       
				} else {


     

       
135

       

       
-

       
					dbErr := e.db.StatusFailed(wid, err.Error(), -1, e.n)


     

       

       
57

       
+

       
					dbErr := db.StatusFailed(wid, err.Error(), -1, n)


     

       
136

       
58

       
 

       
					if dbErr != nil {


     

       
137

       
59

       
 

       
						return dbErr


     

       
138

       
60

       
 

       
					}


     

       

       
61

       
+

       
					return err


     

       
139

       
62

       
 

       
				}


     

       
140

       

       
-

       



     

       
141

       

       
-

       
				return fmt.Errorf("starting steps image: %w", err)


     

       
142

       

       
-

       
			}


     

       
143

       

       
-

       



     

       
144

       

       
-

       
			err = e.db.StatusSuccess(wid, e.n)


     

       
145

       

       
-

       
			if err != nil {


     

       
146

       

       
-

       
				return err


     

       
147

       

       
-

       
			}


     

       
148

       

       
-

       



     

       
149

       

       
-

       
			return nil


     

       
150

       

       
-

       
		})


     

       
151

       

       
-

       
	}


     

       
152

       

       
-

       



     

       
153

       

       
-

       
	if err = eg.Wait(); err != nil {


     

       
154

       

       
-

       
		e.l.Error("failed to run one or more workflows", "err", err)


     

       
155

       

       
-

       
	} else {


     

       
156

       

       
-

       
		e.l.Error("successfully ran full pipeline")


     

       
157

       

       
-

       
	}


     

       
158

       

       
-

       
}


     

       
159

       

       
-

       



     

       
160

       

       
-

       
// SetupWorkflow sets up a new network for the workflow and volumes for


     

       
161

       

       
-

       
// the workspace and Nix store. These are persisted across steps and are


     

       
162

       

       
-

       
// destroyed at the end of the workflow.


     

       
163

       

       
-

       
func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId) error {


     

       
164

       

       
-

       
	e.l.Info("setting up workflow", "workflow", wid)


     

       
165

       

       
-

       



     

       
166

       

       
-

       
	_, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{


     

       
167

       

       
-

       
		Name:   workspaceVolume(wid),


     

       
168

       

       
-

       
		Driver: "local",


     

       
169

       

       
-

       
	})


     

       
170

       

       
-

       
	if err != nil {


     

       
171

       

       
-

       
		return err


     

       
172

       

       
-

       
	}


     

       
173

       

       
-

       
	e.registerCleanup(wid, func(ctx context.Context) error {


     

       
174

       

       
-

       
		return e.docker.VolumeRemove(ctx, workspaceVolume(wid), true)


     

       
175

       

       
-

       
	})


     

       
176

       

       
-

       



     

       
177

       

       
-

       
	_, err = e.docker.VolumeCreate(ctx, volume.CreateOptions{


     

       
178

       

       
-

       
		Name:   nixVolume(wid),


     

       
179

       

       
-

       
		Driver: "local",


     

       
180

       

       
-

       
	})


     

       
181

       

       
-

       
	if err != nil {


     

       
182

       

       
-

       
		return err


     

       
183

       

       
-

       
	}


     

       
184

       

       
-

       
	e.registerCleanup(wid, func(ctx context.Context) error {


     

       
185

       

       
-

       
		return e.docker.VolumeRemove(ctx, nixVolume(wid), true)


     

       
186

       

       
-

       
	})


     

       
187

       

       
-

       



     

       
188

       

       
-

       
	_, err = e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{


     

       
189

       

       
-

       
		Driver: "bridge",


     

       
190

       

       
-

       
	})


     

       
191

       

       
-

       
	if err != nil {


     

       
192

       

       
-

       
		return err


     

       
193

       

       
-

       
	}


     

       
194

       

       
-

       
	e.registerCleanup(wid, func(ctx context.Context) error {


     

       
195

       

       
-

       
		return e.docker.NetworkRemove(ctx, networkName(wid))


     

       
196

       

       
-

       
	})


     

       
197

       

       
-

       



     

       
198

       

       
-

       
	return nil


     

       
199

       

       
-

       
}


     

       
200

       

       
-

       



     

       
201

       

       
-

       
// StartSteps starts all steps sequentially with the same base image.


     

       
202

       

       
-

       
// ONLY marks pipeline as failed if container's exit code is non-zero.


     

       
203

       

       
-

       
// All other errors are bubbled up.


     

       
204

       

       
-

       
// Fixed version of the step execution logic


     

       
205

       

       
-

       
func (e *Engine) StartSteps(ctx context.Context, wid models.WorkflowId, w models.Workflow, secrets []secrets.UnlockedSecret) error {


     

       
206

       

       
-

       
	workflowEnvs := ConstructEnvs(w.Environment)


     

       
207

       

       
-

       
	for _, s := range secrets {


     

       
208

       

       
-

       
		workflowEnvs.AddEnv(s.Key, s.Value)


     

       
209

       

       
-

       
	}


     

       
210

       

       
-

       



     

       
211

       

       
-

       
	for stepIdx, step := range w.Steps {


     

       
212

       

       
-

       
		select {


     

       
213

       

       
-

       
		case <-ctx.Done():


     

       
214

       

       
-

       
			return ctx.Err()


     

       
215

       

       
-

       
		default:


     

       
216

       

       
-

       
		}


     

       
217

       

       
-

       



     

       
218

       

       
-

       
		envs := append(EnvVars(nil), workflowEnvs...)


     

       
219

       

       
-

       
		for k, v := range step.Environment {


     

       
220

       

       
-

       
			envs.AddEnv(k, v)


     

       
221

       

       
-

       
		}


     

       
222

       

       
-

       
		envs.AddEnv("HOME", workspaceDir)


     

       
223

       

       
-

       
		e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice())


     

       
224

       

       
-

       



     

       
225

       

       
-

       
		hostConfig := hostConfig(wid)


     

       
226

       

       
-

       
		resp, err := e.docker.ContainerCreate(ctx, &container.Config{


     

       
227

       

       
-

       
			Image:      w.Image,


     

       
228

       

       
-

       
			Cmd:        []string{"bash", "-c", step.Command},


     

       
229

       

       
-

       
			WorkingDir: workspaceDir,


     

       
230

       

       
-

       
			Tty:        false,


     

       
231

       

       
-

       
			Hostname:   "spindle",


     

       
232

       

       
-

       
			Env:        envs.Slice(),


     

       
233

       

       
-

       
		}, hostConfig, nil, nil, "")


     

       
234

       

       
-

       
		defer e.DestroyStep(ctx, resp.ID)


     

       
235

       

       
-

       
		if err != nil {


     

       
236

       

       
-

       
			return fmt.Errorf("creating container: %w", err)


     

       
237

       

       
-

       
		}


     

       
238

       

       
-

       



     

       
239

       

       
-

       
		err = e.docker.NetworkConnect(ctx, networkName(wid), resp.ID, nil)


     

       
240

       

       
-

       
		if err != nil {


     

       
241

       

       
-

       
			return fmt.Errorf("connecting network: %w", err)


     

       
242

       

       
-

       
		}


     

       
243

       

       
-

       



     

       
244

       

       
-

       
		err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{})


     

       
245

       

       
-

       
		if err != nil {


     

       
246

       

       
-

       
			return err


     

       
247

       

       
-

       
		}


     

       
248

       

       
-

       
		e.l.Info("started container", "name", resp.ID, "step", step.Name)


     

       
249

       

       
-

       



     

       
250

       

       
-

       
		// start tailing logs in background


     

       
251

       

       
-

       
		tailDone := make(chan error, 1)


     

       
252

       

       
-

       
		go func() {


     

       
253

       

       
-

       
			tailDone <- e.TailStep(ctx, resp.ID, wid, stepIdx, step)


     

       
254

       

       
-

       
		}()


     

       
255

       

       
-

       



     

       
256

       

       
-

       
		// wait for container completion or timeout


     

       
257

       

       
-

       
		waitDone := make(chan struct{})


     

       
258

       

       
-

       
		var state *container.State


     

       
259

       

       
-

       
		var waitErr error


     

       
260

       

       
-

       



     

       
261

       

       
-

       
		go func() {


     

       
262

       

       
-

       
			defer close(waitDone)


     

       
263

       

       
-

       
			state, waitErr = e.WaitStep(ctx, resp.ID)


     

       
264

       

       
-

       
		}()


     

       
265

       

       
-

       



     

       
266

       

       
-

       
		select {


     

       
267

       

       
-

       
		case <-waitDone:


     

       
268

       

       
-

       



     

       
269

       

       
-

       
			// wait for tailing to complete


     

       
270

       

       
-

       
			<-tailDone


     

       
271

       

       
-

       



     

       
272

       

       
-

       
		case <-ctx.Done():


     

       
273

       

       
-

       
			e.l.Warn("step timed out; killing container", "container", resp.ID, "step", step.Name)


     

       
274

       

       
-

       
			err = e.DestroyStep(context.Background(), resp.ID)


     

       
275

       

       
-

       
			if err != nil {


     

       
276

       

       
-

       
				e.l.Error("failed to destroy step", "container", resp.ID, "error", err)


     

       
277

       

       
-

       
			}


     

       
278

       

       
-

       



     

       
279

       

       
-

       
			// wait for both goroutines to finish


     

       
280

       

       
-

       
			<-waitDone


     

       
281

       

       
-

       
			<-tailDone


     

       
282

       

       
-

       



     

       
283

       

       
-

       
			return ErrTimedOut


     

       
284

       

       
-

       
		}


     

       
285

       

       
-

       



     

       
286

       

       
-

       
		select {


     

       
287

       

       
-

       
		case <-ctx.Done():


     

       
288

       

       
-

       
			return ctx.Err()


     

       
289

       

       
-

       
		default:


     

       
290

       

       
-

       
		}


     

       
291

       

       
-

       



     

       
292

       

       
-

       
		if waitErr != nil {


     

       
293

       

       
-

       
			return waitErr


     

       
294

       

       
-

       
		}


     

       
295

       

       
-

       



     

       
296

       

       
-

       
		err = e.DestroyStep(ctx, resp.ID)


     

       
297

       

       
-

       
		if err != nil {


     

       
298

       

       
-

       
			return err


     

       
299

       

       
-

       
		}


     

       
300

       

       
-

       



     

       
301

       

       
-

       
		if state.ExitCode != 0 {


     

       
302

       

       
-

       
			e.l.Error("workflow failed!", "workflow_id", wid.String(), "error", state.Error, "exit_code", state.ExitCode, "oom_killed", state.OOMKilled)


     

       
303

       

       
-

       
			if state.OOMKilled {


     

       
304

       

       
-

       
				return ErrOOMKilled


     

       
305

       

       
-

       
			}


     

       
306

       

       
-

       
			return ErrWorkflowFailed


     

       
307

       

       
-

       
		}


     

       
308

       

       
-

       
	}


     

       
309

       

       
-

       



     

       
310

       

       
-

       
	return nil


     

       
311

       

       
-

       
}


     

       
312

       

       
-

       



     

       
313

       

       
-

       
func (e *Engine) WaitStep(ctx context.Context, containerID string) (*container.State, error) {


     

       
314

       

       
-

       
	wait, errCh := e.docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning)


     

       
315

       

       
-

       
	select {


     

       
316

       

       
-

       
	case err := <-errCh:


     

       
317

       

       
-

       
		if err != nil {


     

       
318

       

       
-

       
			return nil, err


     

       
319

       

       
-

       
		}


     

       
320

       

       
-

       
	case <-wait:


     

       
321

       

       
-

       
	}


     

       
322

       

       
-

       



     

       
323

       

       
-

       
	e.l.Info("waited for container", "name", containerID)


     

       
324

       

       
-

       



     

       
325

       

       
-

       
	info, err := e.docker.ContainerInspect(ctx, containerID)


     

       
326

       

       
-

       
	if err != nil {


     

       
327

       

       
-

       
		return nil, err


     

       
328

       

       
-

       
	}


     

       
329

       

       
-

       



     

       
330

       

       
-

       
	return info.State, nil


     

       
331

       

       
-

       
}


     

       
332

       

       
-

       



     

       
333

       

       
-

       
func (e *Engine) TailStep(ctx context.Context, containerID string, wid models.WorkflowId, stepIdx int, step models.Step) error {


     

       
334

       

       
-

       
	wfLogger, err := NewWorkflowLogger(e.cfg.Pipelines.LogDir, wid)


     

       
335

       

       
-

       
	if err != nil {


     

       
336

       

       
-

       
		e.l.Warn("failed to setup step logger; logs will not be persisted", "error", err)


     

       
337

       

       
-

       
		return err


     

       
338

       

       
-

       
	}


     

       
339

       

       
-

       
	defer wfLogger.Close()


     

       
340

       

       
-

       



     

       
341

       

       
-

       
	ctl := wfLogger.ControlWriter(stepIdx, step)


     

       
342

       

       
-

       
	ctl.Write([]byte(step.Name))


     

       
343

       

       
-

       



     

       
344

       

       
-

       
	logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{


     

       
345

       

       
-

       
		Follow:     true,


     

       
346

       

       
-

       
		ShowStdout: true,


     

       
347

       

       
-

       
		ShowStderr: true,


     

       
348

       

       
-

       
		Details:    false,


     

       
349

       

       
-

       
		Timestamps: false,


     

       
350

       

       
-

       
	})


     

       
351

       

       
-

       
	if err != nil {


     

       
352

       

       
-

       
		return err


     

       
353

       

       
-

       
	}


     

       
354

       

       
-

       



     

       
355

       

       
-

       
	_, err = stdcopy.StdCopy(


     

       
356

       

       
-

       
		wfLogger.DataWriter("stdout"),


     

       
357

       

       
-

       
		wfLogger.DataWriter("stderr"),


     

       
358

       

       
-

       
		logs,


     

       
359

       

       
-

       
	)


     

       
360

       

       
-

       
	if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) {


     

       
361

       

       
-

       
		return fmt.Errorf("failed to copy logs: %w", err)


     

       
362

       

       
-

       
	}


     

       

       
63

       
+

       
				defer eng.DestroyWorkflow(ctx, wid)


     

       
363

       
64

       
 

       



     

       
364

       

       
-

       
	return nil


     

       
365

       

       
-

       
}


     

       

       
65

       
+

       
				wfLogger, err := models.NewWorkflowLogger(cfg.Server.LogDir, wid)


     

       

       
66

       
+

       
				if err != nil {


     

       

       
67

       
+

       
					l.Warn("failed to setup step logger; logs will not be persisted", "error", err)


     

       

       
68

       
+

       
					wfLogger = nil


     

       

       
69

       
+

       
				} else {


     

       

       
70

       
+

       
					defer wfLogger.Close()


     

       

       
71

       
+

       
				}


     

       
366

       
72

       
 

       



     

       
367

       

       
-

       
func (e *Engine) DestroyStep(ctx context.Context, containerID string) error {


     

       
368

       

       
-

       
	err := e.docker.ContainerKill(ctx, containerID, "9") // SIGKILL


     

       
369

       

       
-

       
	if err != nil && !isErrContainerNotFoundOrNotRunning(err) {


     

       
370

       

       
-

       
		return err


     

       
371

       

       
-

       
	}


     

       

       
73

       
+

       
				ctx, cancel := context.WithTimeout(ctx, workflowTimeout)


     

       

       
74

       
+

       
				defer cancel()


     

       
372

       
75

       
 

       



     

       
373

       

       
-

       
	if err := e.docker.ContainerRemove(ctx, containerID, container.RemoveOptions{


     

       
374

       

       
-

       
		RemoveVolumes: true,


     

       
375

       

       
-

       
		RemoveLinks:   false,


     

       
376

       

       
-

       
		Force:         false,


     

       
377

       

       
-

       
	}); err != nil && !isErrContainerNotFoundOrNotRunning(err) {


     

       
378

       

       
-

       
		return err


     

       
379

       

       
-

       
	}


     

       

       
76

       
+

       
				for stepIdx, step := range w.Steps {


     

       

       
77

       
+

       
					if wfLogger != nil {


     

       

       
78

       
+

       
						ctl := wfLogger.ControlWriter(stepIdx, step)


     

       

       
79

       
+

       
						ctl.Write([]byte(step.Name()))


     

       

       
80

       
+

       
					}


     

       
380

       
81

       
 

       



     

       
381

       

       
-

       
	return nil


     

       
382

       

       
-

       
}


     

       

       
82

       
+

       
					err = eng.RunStep(ctx, wid, &w, stepIdx, allSecrets, wfLogger)


     

       

       
83

       
+

       
					if err != nil {


     

       

       
84

       
+

       
						if errors.Is(err, ErrTimedOut) {


     

       

       
85

       
+

       
							dbErr := db.StatusTimeout(wid, n)


     

       

       
86

       
+

       
							if dbErr != nil {


     

       

       
87

       
+

       
								return dbErr


     

       

       
88

       
+

       
							}


     

       

       
89

       
+

       
						} else {


     

       

       
90

       
+

       
							dbErr := db.StatusFailed(wid, err.Error(), -1, n)


     

       

       
91

       
+

       
							if dbErr != nil {


     

       

       
92

       
+

       
								return dbErr


     

       

       
93

       
+

       
							}


     

       

       
94

       
+

       
						}


     

       
383

       
95

       
 

       



     

       
384

       

       
-

       
func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error {


     

       
385

       

       
-

       
	e.cleanupMu.Lock()


     

       
386

       

       
-

       
	key := wid.String()


     

       

       
96

       
+

       
						return fmt.Errorf("starting steps image: %w", err)


     

       

       
97

       
+

       
					}


     

       

       
98

       
+

       
				}


     

       
387

       
99

       
 

       



     

       
388

       

       
-

       
	fns := e.cleanup[key]


     

       
389

       

       
-

       
	delete(e.cleanup, key)


     

       
390

       

       
-

       
	e.cleanupMu.Unlock()


     

       

       
100

       
+

       
				err = db.StatusSuccess(wid, n)


     

       

       
101

       
+

       
				if err != nil {


     

       

       
102

       
+

       
					return err


     

       

       
103

       
+

       
				}


     

       
391

       
104

       
 

       



     

       
392

       

       
-

       
	for _, fn := range fns {


     

       
393

       

       
-

       
		if err := fn(ctx); err != nil {


     

       
394

       

       
-

       
			e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err)


     

       

       
105

       
+

       
				return nil


     

       

       
106

       
+

       
			})


     

       
395

       
107

       
 

       
		}


     

       
396

       
108

       
 

       
	}


     

       
397

       

       
-

       
	return nil


     

       
398

       

       
-

       
}


     

       
399

       
109

       
 

       



     

       
400

       

       
-

       
func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) {


     

       
401

       

       
-

       
	e.cleanupMu.Lock()


     

       
402

       

       
-

       
	defer e.cleanupMu.Unlock()


     

       
403

       

       
-

       



     

       
404

       

       
-

       
	key := wid.String()


     

       
405

       

       
-

       
	e.cleanup[key] = append(e.cleanup[key], fn)


     

       
406

       

       
-

       
}


     

       
407

       

       
-

       



     

       
408

       

       
-

       
func workspaceVolume(wid models.WorkflowId) string {


     

       
409

       

       
-

       
	return fmt.Sprintf("workspace-%s", wid)


     

       
410

       

       
-

       
}


     

       
411

       

       
-

       



     

       
412

       

       
-

       
func nixVolume(wid models.WorkflowId) string {


     

       
413

       

       
-

       
	return fmt.Sprintf("nix-%s", wid)


     

       
414

       

       
-

       
}


     

       
415

       

       
-

       



     

       
416

       

       
-

       
func networkName(wid models.WorkflowId) string {


     

       
417

       

       
-

       
	return fmt.Sprintf("workflow-network-%s", wid)


     

       
418

       

       
-

       
}


     

       
419

       

       
-

       



     

       
420

       

       
-

       
func hostConfig(wid models.WorkflowId) *container.HostConfig {


     

       
421

       

       
-

       
	hostConfig := &container.HostConfig{


     

       
422

       

       
-

       
		Mounts: []mount.Mount{


     

       
423

       

       
-

       
			{


     

       
424

       

       
-

       
				Type:   mount.TypeVolume,


     

       
425

       

       
-

       
				Source: workspaceVolume(wid),


     

       
426

       

       
-

       
				Target: workspaceDir,


     

       
427

       

       
-

       
			},


     

       
428

       

       
-

       
			{


     

       
429

       

       
-

       
				Type:   mount.TypeVolume,


     

       
430

       

       
-

       
				Source: nixVolume(wid),


     

       
431

       

       
-

       
				Target: "/nix",


     

       
432

       

       
-

       
			},


     

       
433

       

       
-

       
			{


     

       
434

       

       
-

       
				Type:     mount.TypeTmpfs,


     

       
435

       

       
-

       
				Target:   "/tmp",


     

       
436

       

       
-

       
				ReadOnly: false,


     

       
437

       

       
-

       
				TmpfsOptions: &mount.TmpfsOptions{


     

       
438

       

       
-

       
					Mode: 0o1777, // world-writeable sticky bit


     

       
439

       

       
-

       
					Options: [][]string{


     

       
440

       

       
-

       
						{"exec"},


     

       
441

       

       
-

       
					},


     

       
442

       

       
-

       
				},


     

       
443

       

       
-

       
			},


     

       
444

       

       
-

       
			{


     

       
445

       

       
-

       
				Type:   mount.TypeVolume,


     

       
446

       

       
-

       
				Source: "etc-nix-" + wid.String(),


     

       
447

       

       
-

       
				Target: "/etc/nix",


     

       
448

       

       
-

       
			},


     

       
449

       

       
-

       
		},


     

       
450

       

       
-

       
		ReadonlyRootfs: false,


     

       
451

       

       
-

       
		CapDrop:        []string{"ALL"},


     

       
452

       

       
-

       
		CapAdd:         []string{"CAP_DAC_OVERRIDE"},


     

       
453

       

       
-

       
		SecurityOpt:    []string{"no-new-privileges"},


     

       
454

       

       
-

       
		ExtraHosts:     []string{"host.docker.internal:host-gateway"},


     

       

       
110

       
+

       
	if err := eg.Wait(); err != nil {


     

       

       
111

       
+

       
		l.Error("failed to run one or more workflows", "err", err)


     

       

       
112

       
+

       
	} else {


     

       

       
113

       
+

       
		l.Error("successfully ran full pipeline")


     

       
455

       
114

       
 

       
	}


     

       
456

       

       
-

       



     

       
457

       

       
-

       
	return hostConfig


     

       
458

       

       
-

       
}


     

       
459

       

       
-

       



     

       
460

       

       
-

       
// thanks woodpecker


     

       
461

       

       
-

       
func isErrContainerNotFoundOrNotRunning(err error) bool {


     

       
462

       

       
-

       
	// Error response from daemon: Cannot kill container: ...: No such container: ...


     

       
463

       

       
-

       
	// Error response from daemon: Cannot kill container: ...: Container ... is not running"


     

       
464

       

       
-

       
	// Error response from podman daemon: can only kill running containers. ... is in state exited


     

       
465

       

       
-

       
	// Error: No such container: ...


     

       
466

       

       
-

       
	return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running") || strings.Contains(err.Error(), "can only kill running containers"))


     

       
467

       
115

       
 

       
}
+1 -1
spindle/engine/envs.go spindle/engines/nixery/envs.go 
···

       
1

       

       
-

       
package engine


     

       

       
1

       
+

       
package nixery


     

       
2

       
2

       
 

       



     

       
3

       
3

       
 

       
import (


     

       
4

       
4

       
 

       
	"fmt"
+1 -1
spindle/engine/envs_test.go spindle/engines/nixery/envs_test.go 
···

       
1

       

       
-

       
package engine


     

       

       
1

       
+

       
package nixery


     

       
2

       
2

       
 

       



     

       
3

       
3

       
 

       
import (


     

       
4

       
4

       
 

       
	"testing"
-9
spindle/engine/errors.go 
···

       
1

       

       
-

       
package engine


     

       
2

       

       
-

       



     

       
3

       

       
-

       
import "errors"


     

       
4

       

       
-

       



     

       
5

       

       
-

       
var (


     

       
6

       

       
-

       
	ErrOOMKilled      = errors.New("oom killed")


     

       
7

       

       
-

       
	ErrTimedOut       = errors.New("timed out")


     

       
8

       

       
-

       
	ErrWorkflowFailed = errors.New("workflow failed")


     

       
9

       

       
-

       
)
+8 -10
spindle/engine/logger.go spindle/models/logger.go 
···

       
1

       

       
-

       
package engine


     

       

       
1

       
+

       
package models


     

       
2

       
2

       
 

       



     

       
3

       
3

       
 

       
import (


     

       
4

       
4

       
 

       
	"encoding/json"


     
···

       
7

       
7

       
 

       
	"os"


     

       
8

       
8

       
 

       
	"path/filepath"


     

       
9

       
9

       
 

       
	"strings"


     

       
10

       

       
-

       



     

       
11

       

       
-

       
	"tangled.sh/tangled.sh/core/spindle/models"


     

       
12

       
10

       
 

       
)


     

       
13

       
11

       
 

       



     

       
14

       
12

       
 

       
type WorkflowLogger struct {


     
···

       
16

       
14

       
 

       
	encoder *json.Encoder


     

       
17

       
15

       
 

       
}


     

       
18

       
16

       
 

       



     

       
19

       

       
-

       
func NewWorkflowLogger(baseDir string, wid models.WorkflowId) (*WorkflowLogger, error) {


     

       

       
17

       
+

       
func NewWorkflowLogger(baseDir string, wid WorkflowId) (*WorkflowLogger, error) {


     

       
20

       
18

       
 

       
	path := LogFilePath(baseDir, wid)


     

       
21

       
19

       
 

       



     

       
22

       
20

       
 

       
	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)


     
···

       
30

       
28

       
 

       
	}, nil


     

       
31

       
29

       
 

       
}


     

       
32

       
30

       
 

       



     

       
33

       

       
-

       
func LogFilePath(baseDir string, workflowID models.WorkflowId) string {


     

       

       
31

       
+

       
func LogFilePath(baseDir string, workflowID WorkflowId) string {


     

       
34

       
32

       
 

       
	logFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.log", workflowID.String()))


     

       
35

       
33

       
 

       
	return logFilePath


     

       
36

       
34

       
 

       
}


     
···

       
47

       
45

       
 

       
	}


     

       
48

       
46

       
 

       
}


     

       
49

       
47

       
 

       



     

       
50

       

       
-

       
func (l *WorkflowLogger) ControlWriter(idx int, step models.Step) io.Writer {


     

       

       
48

       
+

       
func (l *WorkflowLogger) ControlWriter(idx int, step Step) io.Writer {


     

       
51

       
49

       
 

       
	return &controlWriter{


     

       
52

       
50

       
 

       
		logger: l,


     

       
53

       
51

       
 

       
		idx:    idx,


     
···

       
62

       
60

       
 

       



     

       
63

       
61

       
 

       
func (w *dataWriter) Write(p []byte) (int, error) {


     

       
64

       
62

       
 

       
	line := strings.TrimRight(string(p), "\r\n")


     

       
65

       

       
-

       
	entry := models.NewDataLogLine(line, w.stream)


     

       

       
63

       
+

       
	entry := NewDataLogLine(line, w.stream)


     

       
66

       
64

       
 

       
	if err := w.logger.encoder.Encode(entry); err != nil {


     

       
67

       
65

       
 

       
		return 0, err


     

       
68

       
66

       
 

       
	}


     
···

       
72

       
70

       
 

       
type controlWriter struct {


     

       
73

       
71

       
 

       
	logger *WorkflowLogger


     

       
74

       
72

       
 

       
	idx    int


     

       
75

       

       
-

       
	step   models.Step


     

       

       
73

       
+

       
	step   Step


     

       
76

       
74

       
 

       
}


     

       
77

       
75

       
 

       



     

       
78

       
76

       
 

       
func (w *controlWriter) Write(_ []byte) (int, error) {


     

       
79

       

       
-

       
	entry := models.NewControlLogLine(w.idx, w.step)


     

       

       
77

       
+

       
	entry := NewControlLogLine(w.idx, w.step)


     

       
80

       
78

       
 

       
	if err := w.logger.encoder.Encode(entry); err != nil {


     

       
81

       
79

       
 

       
		return 0, err


     

       
82

       
80

       
 

       
	}


     

       
83

       

       
-

       
	return len(w.step.Name), nil


     

       

       
81

       
+

       
	return len(w.step.Name()), nil


     

       
84

       
82

       
 

       
}
+476
spindle/engines/nixery/engine.go 
···

       

       
1

       
+

       
package nixery


     

       

       
2

       
+

       



     

       

       
3

       
+

       
import (


     

       

       
4

       
+

       
	"context"


     

       

       
5

       
+

       
	"errors"


     

       

       
6

       
+

       
	"fmt"


     

       

       
7

       
+

       
	"io"


     

       

       
8

       
+

       
	"log/slog"


     

       

       
9

       
+

       
	"os"


     

       

       
10

       
+

       
	"path"


     

       

       
11

       
+

       
	"strings"


     

       

       
12

       
+

       
	"sync"


     

       

       
13

       
+

       
	"time"


     

       

       
14

       
+

       



     

       

       
15

       
+

       
	"github.com/docker/docker/api/types/container"


     

       

       
16

       
+

       
	"github.com/docker/docker/api/types/image"


     

       

       
17

       
+

       
	"github.com/docker/docker/api/types/mount"


     

       

       
18

       
+

       
	"github.com/docker/docker/api/types/network"


     

       

       
19

       
+

       
	"github.com/docker/docker/api/types/volume"


     

       

       
20

       
+

       
	"github.com/docker/docker/client"


     

       

       
21

       
+

       
	"github.com/docker/docker/pkg/stdcopy"


     

       

       
22

       
+

       
	"gopkg.in/yaml.v3"


     

       

       
23

       
+

       
	"tangled.sh/tangled.sh/core/api/tangled"


     

       

       
24

       
+

       
	"tangled.sh/tangled.sh/core/log"


     

       

       
25

       
+

       
	"tangled.sh/tangled.sh/core/spindle/config"


     

       

       
26

       
+

       
	"tangled.sh/tangled.sh/core/spindle/engine"


     

       

       
27

       
+

       
	"tangled.sh/tangled.sh/core/spindle/models"


     

       

       
28

       
+

       
	"tangled.sh/tangled.sh/core/spindle/secrets"


     

       

       
29

       
+

       
)


     

       

       
30

       
+

       



     

       

       
31

       
+

       
const (


     

       

       
32

       
+

       
	workspaceDir = "/tangled/workspace"


     

       

       
33

       
+

       
)


     

       

       
34

       
+

       



     

       

       
35

       
+

       
type cleanupFunc func(context.Context) error


     

       

       
36

       
+

       



     

       

       
37

       
+

       
type Engine struct {


     

       

       
38

       
+

       
	docker client.APIClient


     

       

       
39

       
+

       
	l      *slog.Logger


     

       

       
40

       
+

       
	cfg    *config.Config


     

       

       
41

       
+

       



     

       

       
42

       
+

       
	cleanupMu sync.Mutex


     

       

       
43

       
+

       
	cleanup   map[string][]cleanupFunc


     

       

       
44

       
+

       
}


     

       

       
45

       
+

       



     

       

       
46

       
+

       
type Step struct {


     

       

       
47

       
+

       
	name        string


     

       

       
48

       
+

       
	kind        models.StepKind


     

       

       
49

       
+

       
	command     string


     

       

       
50

       
+

       
	environment map[string]string


     

       

       
51

       
+

       
}


     

       

       
52

       
+

       



     

       

       
53

       
+

       
func (s Step) Name() string {


     

       

       
54

       
+

       
	return s.name


     

       

       
55

       
+

       
}


     

       

       
56

       
+

       



     

       

       
57

       
+

       
func (s Step) Command() string {


     

       

       
58

       
+

       
	return s.command


     

       

       
59

       
+

       
}


     

       

       
60

       
+

       



     

       

       
61

       
+

       
func (s Step) Kind() models.StepKind {


     

       

       
62

       
+

       
	return s.kind


     

       

       
63

       
+

       
}


     

       

       
64

       
+

       



     

       

       
65

       
+

       
// setupSteps get added to start of Steps


     

       

       
66

       
+

       
type setupSteps []models.Step


     

       

       
67

       
+

       



     

       

       
68

       
+

       
// addStep adds a step to the beginning of the workflow's steps.


     

       

       
69

       
+

       
func (ss *setupSteps) addStep(step models.Step) {


     

       

       
70

       
+

       
	*ss = append(*ss, step)


     

       

       
71

       
+

       
}


     

       

       
72

       
+

       



     

       

       
73

       
+

       
type addlFields struct {


     

       

       
74

       
+

       
	image string


     

       

       
75

       
+

       
	env   map[string]string


     

       

       
76

       
+

       
}


     

       

       
77

       
+

       



     

       

       
78

       
+

       
func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) {


     

       

       
79

       
+

       
	swf := &models.Workflow{}


     

       

       
80

       
+

       
	addl := addlFields{}


     

       

       
81

       
+

       



     

       

       
82

       
+

       
	dwf := &struct {


     

       

       
83

       
+

       
		Steps []struct {


     

       

       
84

       
+

       
			Command     string            `yaml:"command"`


     

       

       
85

       
+

       
			Name        string            `yaml:"name"`


     

       

       
86

       
+

       
			Environment map[string]string `yaml:"environment"`


     

       

       
87

       
+

       
		} `yaml:"steps"`


     

       

       
88

       
+

       
		Dependencies map[string][]string `yaml:"dependencies"`


     

       

       
89

       
+

       
		Environment  map[string]string   `yaml:"environment"`


     

       

       
90

       
+

       
	}{}


     

       

       
91

       
+

       
	err := yaml.Unmarshal([]byte(twf.Raw), &dwf)


     

       

       
92

       
+

       
	if err != nil {


     

       

       
93

       
+

       
		return nil, err


     

       

       
94

       
+

       
	}


     

       

       
95

       
+

       



     

       

       
96

       
+

       
	for _, dstep := range dwf.Steps {


     

       

       
97

       
+

       
		sstep := Step{}


     

       

       
98

       
+

       
		sstep.environment = dstep.Environment


     

       

       
99

       
+

       
		sstep.command = dstep.Command


     

       

       
100

       
+

       
		sstep.name = dstep.Name


     

       

       
101

       
+

       
		sstep.kind = models.StepKindUser


     

       

       
102

       
+

       
		swf.Steps = append(swf.Steps, sstep)


     

       

       
103

       
+

       
	}


     

       

       
104

       
+

       
	swf.Name = twf.Name


     

       

       
105

       
+

       
	addl.env = dwf.Environment


     

       

       
106

       
+

       
	addl.image = workflowImage(dwf.Dependencies, e.cfg.NixeryPipelines.Nixery)


     

       

       
107

       
+

       



     

       

       
108

       
+

       
	setup := &setupSteps{}


     

       

       
109

       
+

       



     

       

       
110

       
+

       
	setup.addStep(nixConfStep())


     

       

       
111

       
+

       
	setup.addStep(cloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev))


     

       

       
112

       
+

       
	// this step could be empty


     

       

       
113

       
+

       
	if s := dependencyStep(dwf.Dependencies); s != nil {


     

       

       
114

       
+

       
		setup.addStep(*s)


     

       

       
115

       
+

       
	}


     

       

       
116

       
+

       



     

       

       
117

       
+

       
	// append setup steps in order to the start of workflow steps


     

       

       
118

       
+

       
	swf.Steps = append(*setup, swf.Steps...)


     

       

       
119

       
+

       
	swf.Data = addl


     

       

       
120

       
+

       



     

       

       
121

       
+

       
	return swf, nil


     

       

       
122

       
+

       
}


     

       

       
123

       
+

       



     

       

       
124

       
+

       
func (e *Engine) WorkflowTimeout() time.Duration {


     

       

       
125

       
+

       
	workflowTimeoutStr := e.cfg.NixeryPipelines.WorkflowTimeout


     

       

       
126

       
+

       
	workflowTimeout, err := time.ParseDuration(workflowTimeoutStr)


     

       

       
127

       
+

       
	if err != nil {


     

       

       
128

       
+

       
		e.l.Error("failed to parse workflow timeout", "error", err, "timeout", workflowTimeoutStr)


     

       

       
129

       
+

       
		workflowTimeout = 5 * time.Minute


     

       

       
130

       
+

       
	}


     

       

       
131

       
+

       



     

       

       
132

       
+

       
	return workflowTimeout


     

       

       
133

       
+

       
}


     

       

       
134

       
+

       



     

       

       
135

       
+

       
func workflowImage(deps map[string][]string, nixery string) string {


     

       

       
136

       
+

       
	var dependencies string


     

       

       
137

       
+

       
	for reg, ds := range deps {


     

       

       
138

       
+

       
		if reg == "nixpkgs" {


     

       

       
139

       
+

       
			dependencies = path.Join(ds...)


     

       

       
140

       
+

       
		}


     

       

       
141

       
+

       
	}


     

       

       
142

       
+

       



     

       

       
143

       
+

       
	// load defaults from somewhere else


     

       

       
144

       
+

       
	dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix")


     

       

       
145

       
+

       



     

       

       
146

       
+

       
	return path.Join(nixery, dependencies)


     

       

       
147

       
+

       
}


     

       

       
148

       
+

       



     

       

       
149

       
+

       
func New(ctx context.Context, cfg *config.Config) (*Engine, error) {


     

       

       
150

       
+

       
	dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())


     

       

       
151

       
+

       
	if err != nil {


     

       

       
152

       
+

       
		return nil, err


     

       

       
153

       
+

       
	}


     

       

       
154

       
+

       



     

       

       
155

       
+

       
	l := log.FromContext(ctx).With("component", "spindle")


     

       

       
156

       
+

       



     

       

       
157

       
+

       
	e := &Engine{


     

       

       
158

       
+

       
		docker: dcli,


     

       

       
159

       
+

       
		l:      l,


     

       

       
160

       
+

       
		cfg:    cfg,


     

       

       
161

       
+

       
	}


     

       

       
162

       
+

       



     

       

       
163

       
+

       
	e.cleanup = make(map[string][]cleanupFunc)


     

       

       
164

       
+

       



     

       

       
165

       
+

       
	return e, nil


     

       

       
166

       
+

       
}


     

       

       
167

       
+

       



     

       

       
168

       
+

       
// SetupWorkflow sets up a new network for the workflow and volumes for


     

       

       
169

       
+

       
// the workspace and Nix store. These are persisted across steps and are


     

       

       
170

       
+

       
// destroyed at the end of the workflow.


     

       

       
171

       
+

       
func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) error {


     

       

       
172

       
+

       
	e.l.Info("setting up workflow", "workflow", wid)


     

       

       
173

       
+

       



     

       

       
174

       
+

       
	_, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{


     

       

       
175

       
+

       
		Name:   workspaceVolume(wid),


     

       

       
176

       
+

       
		Driver: "local",


     

       

       
177

       
+

       
	})


     

       

       
178

       
+

       
	if err != nil {


     

       

       
179

       
+

       
		return err


     

       

       
180

       
+

       
	}


     

       

       
181

       
+

       
	e.registerCleanup(wid, func(ctx context.Context) error {


     

       

       
182

       
+

       
		return e.docker.VolumeRemove(ctx, workspaceVolume(wid), true)


     

       

       
183

       
+

       
	})


     

       

       
184

       
+

       



     

       

       
185

       
+

       
	_, err = e.docker.VolumeCreate(ctx, volume.CreateOptions{


     

       

       
186

       
+

       
		Name:   nixVolume(wid),


     

       

       
187

       
+

       
		Driver: "local",


     

       

       
188

       
+

       
	})


     

       

       
189

       
+

       
	if err != nil {


     

       

       
190

       
+

       
		return err


     

       

       
191

       
+

       
	}


     

       

       
192

       
+

       
	e.registerCleanup(wid, func(ctx context.Context) error {


     

       

       
193

       
+

       
		return e.docker.VolumeRemove(ctx, nixVolume(wid), true)


     

       

       
194

       
+

       
	})


     

       

       
195

       
+

       



     

       

       
196

       
+

       
	_, err = e.docker.NetworkCreate(ctx, networkName(wid), network.CreateOptions{


     

       

       
197

       
+

       
		Driver: "bridge",


     

       

       
198

       
+

       
	})


     

       

       
199

       
+

       
	if err != nil {


     

       

       
200

       
+

       
		return err


     

       

       
201

       
+

       
	}


     

       

       
202

       
+

       
	e.registerCleanup(wid, func(ctx context.Context) error {


     

       

       
203

       
+

       
		return e.docker.NetworkRemove(ctx, networkName(wid))


     

       

       
204

       
+

       
	})


     

       

       
205

       
+

       



     

       

       
206

       
+

       
	addl := wf.Data.(addlFields)


     

       

       
207

       
+

       



     

       

       
208

       
+

       
	reader, err := e.docker.ImagePull(ctx, addl.image, image.PullOptions{})


     

       

       
209

       
+

       
	if err != nil {


     

       

       
210

       
+

       
		e.l.Error("pipeline image pull failed!", "image", addl.image, "workflowId", wid, "error", err.Error())


     

       

       
211

       
+

       



     

       

       
212

       
+

       
		return fmt.Errorf("pulling image: %w", err)


     

       

       
213

       
+

       
	}


     

       

       
214

       
+

       
	defer reader.Close()


     

       

       
215

       
+

       
	io.Copy(os.Stdout, reader)


     

       

       
216

       
+

       



     

       

       
217

       
+

       
	return nil


     

       

       
218

       
+

       
}


     

       

       
219

       
+

       



     

       

       
220

       
+

       
func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *models.WorkflowLogger) error {


     

       

       
221

       
+

       
	workflowEnvs := ConstructEnvs(w.Data.(addlFields).env)


     

       

       
222

       
+

       
	for _, s := range secrets {


     

       

       
223

       
+

       
		workflowEnvs.AddEnv(s.Key, s.Value)


     

       

       
224

       
+

       
	}


     

       

       
225

       
+

       



     

       

       
226

       
+

       
	step := w.Steps[idx].(Step)


     

       

       
227

       
+

       



     

       

       
228

       
+

       
	select {


     

       

       
229

       
+

       
	case <-ctx.Done():


     

       

       
230

       
+

       
		return ctx.Err()


     

       

       
231

       
+

       
	default:


     

       

       
232

       
+

       
	}


     

       

       
233

       
+

       



     

       

       
234

       
+

       
	envs := append(EnvVars(nil), workflowEnvs...)


     

       

       
235

       
+

       
	for k, v := range step.environment {


     

       

       
236

       
+

       
		envs.AddEnv(k, v)


     

       

       
237

       
+

       
	}


     

       

       
238

       
+

       
	envs.AddEnv("HOME", workspaceDir)


     

       

       
239

       
+

       
	e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice())


     

       

       
240

       
+

       



     

       

       
241

       
+

       
	hostConfig := hostConfig(wid)


     

       

       
242

       
+

       
	resp, err := e.docker.ContainerCreate(ctx, &container.Config{


     

       

       
243

       
+

       
		Image:      w.Data.(addlFields).image,


     

       

       
244

       
+

       
		Cmd:        []string{"bash", "-c", step.command},


     

       

       
245

       
+

       
		WorkingDir: workspaceDir,


     

       

       
246

       
+

       
		Tty:        false,


     

       

       
247

       
+

       
		Hostname:   "spindle",


     

       

       
248

       
+

       
		Env:        envs.Slice(),


     

       

       
249

       
+

       
	}, hostConfig, nil, nil, "")


     

       

       
250

       
+

       
	defer e.DestroyStep(ctx, resp.ID)


     

       

       
251

       
+

       
	if err != nil {


     

       

       
252

       
+

       
		return fmt.Errorf("creating container: %w", err)


     

       

       
253

       
+

       
	}


     

       

       
254

       
+

       



     

       

       
255

       
+

       
	err = e.docker.NetworkConnect(ctx, networkName(wid), resp.ID, nil)


     

       

       
256

       
+

       
	if err != nil {


     

       

       
257

       
+

       
		return fmt.Errorf("connecting network: %w", err)


     

       

       
258

       
+

       
	}


     

       

       
259

       
+

       



     

       

       
260

       
+

       
	err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{})


     

       

       
261

       
+

       
	if err != nil {


     

       

       
262

       
+

       
		return err


     

       

       
263

       
+

       
	}


     

       

       
264

       
+

       
	e.l.Info("started container", "name", resp.ID, "step", step.Name)


     

       

       
265

       
+

       



     

       

       
266

       
+

       
	// start tailing logs in background


     

       

       
267

       
+

       
	tailDone := make(chan error, 1)


     

       

       
268

       
+

       
	go func() {


     

       

       
269

       
+

       
		tailDone <- e.tailStep(ctx, wfLogger, resp.ID, wid, idx, step)


     

       

       
270

       
+

       
	}()


     

       

       
271

       
+

       



     

       

       
272

       
+

       
	// wait for container completion or timeout


     

       

       
273

       
+

       
	waitDone := make(chan struct{})


     

       

       
274

       
+

       
	var state *container.State


     

       

       
275

       
+

       
	var waitErr error


     

       

       
276

       
+

       



     

       

       
277

       
+

       
	go func() {


     

       

       
278

       
+

       
		defer close(waitDone)


     

       

       
279

       
+

       
		state, waitErr = e.WaitStep(ctx, resp.ID)


     

       

       
280

       
+

       
	}()


     

       

       
281

       
+

       



     

       

       
282

       
+

       
	select {


     

       

       
283

       
+

       
	case <-waitDone:


     

       

       
284

       
+

       



     

       

       
285

       
+

       
		// wait for tailing to complete


     

       

       
286

       
+

       
		<-tailDone


     

       

       
287

       
+

       



     

       

       
288

       
+

       
	case <-ctx.Done():


     

       

       
289

       
+

       
		e.l.Warn("step timed out; killing container", "container", resp.ID, "step", step.Name)


     

       

       
290

       
+

       
		err = e.DestroyStep(context.Background(), resp.ID)


     

       

       
291

       
+

       
		if err != nil {


     

       

       
292

       
+

       
			e.l.Error("failed to destroy step", "container", resp.ID, "error", err)


     

       

       
293

       
+

       
		}


     

       

       
294

       
+

       



     

       

       
295

       
+

       
		// wait for both goroutines to finish


     

       

       
296

       
+

       
		<-waitDone


     

       

       
297

       
+

       
		<-tailDone


     

       

       
298

       
+

       



     

       

       
299

       
+

       
		return engine.ErrTimedOut


     

       

       
300

       
+

       
	}


     

       

       
301

       
+

       



     

       

       
302

       
+

       
	select {


     

       

       
303

       
+

       
	case <-ctx.Done():


     

       

       
304

       
+

       
		return ctx.Err()


     

       

       
305

       
+

       
	default:


     

       

       
306

       
+

       
	}


     

       

       
307

       
+

       



     

       

       
308

       
+

       
	if waitErr != nil {


     

       

       
309

       
+

       
		return waitErr


     

       

       
310

       
+

       
	}


     

       

       
311

       
+

       



     

       

       
312

       
+

       
	err = e.DestroyStep(ctx, resp.ID)


     

       

       
313

       
+

       
	if err != nil {


     

       

       
314

       
+

       
		return err


     

       

       
315

       
+

       
	}


     

       

       
316

       
+

       



     

       

       
317

       
+

       
	if state.ExitCode != 0 {


     

       

       
318

       
+

       
		e.l.Error("workflow failed!", "workflow_id", wid.String(), "error", state.Error, "exit_code", state.ExitCode, "oom_killed", state.OOMKilled)


     

       

       
319

       
+

       
		if state.OOMKilled {


     

       

       
320

       
+

       
			return ErrOOMKilled


     

       

       
321

       
+

       
		}


     

       

       
322

       
+

       
		return engine.ErrWorkflowFailed


     

       

       
323

       
+

       
	}


     

       

       
324

       
+

       



     

       

       
325

       
+

       
	return nil


     

       

       
326

       
+

       
}


     

       

       
327

       
+

       



     

       

       
328

       
+

       
func (e *Engine) WaitStep(ctx context.Context, containerID string) (*container.State, error) {


     

       

       
329

       
+

       
	wait, errCh := e.docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning)


     

       

       
330

       
+

       
	select {


     

       

       
331

       
+

       
	case err := <-errCh:


     

       

       
332

       
+

       
		if err != nil {


     

       

       
333

       
+

       
			return nil, err


     

       

       
334

       
+

       
		}


     

       

       
335

       
+

       
	case <-wait:


     

       

       
336

       
+

       
	}


     

       

       
337

       
+

       



     

       

       
338

       
+

       
	e.l.Info("waited for container", "name", containerID)


     

       

       
339

       
+

       



     

       

       
340

       
+

       
	info, err := e.docker.ContainerInspect(ctx, containerID)


     

       

       
341

       
+

       
	if err != nil {


     

       

       
342

       
+

       
		return nil, err


     

       

       
343

       
+

       
	}


     

       

       
344

       
+

       



     

       

       
345

       
+

       
	return info.State, nil


     

       

       
346

       
+

       
}


     

       

       
347

       
+

       



     

       

       
348

       
+

       
func (e *Engine) tailStep(ctx context.Context, wfLogger *models.WorkflowLogger, containerID string, wid models.WorkflowId, stepIdx int, step models.Step) error {


     

       

       
349

       
+

       
	if wfLogger == nil {


     

       

       
350

       
+

       
		return nil


     

       

       
351

       
+

       
	}


     

       

       
352

       
+

       



     

       

       
353

       
+

       
	logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{


     

       

       
354

       
+

       
		Follow:     true,


     

       

       
355

       
+

       
		ShowStdout: true,


     

       

       
356

       
+

       
		ShowStderr: true,


     

       

       
357

       
+

       
		Details:    false,


     

       

       
358

       
+

       
		Timestamps: false,


     

       

       
359

       
+

       
	})


     

       

       
360

       
+

       
	if err != nil {


     

       

       
361

       
+

       
		return err


     

       

       
362

       
+

       
	}


     

       

       
363

       
+

       



     

       

       
364

       
+

       
	_, err = stdcopy.StdCopy(


     

       

       
365

       
+

       
		wfLogger.DataWriter("stdout"),


     

       

       
366

       
+

       
		wfLogger.DataWriter("stderr"),


     

       

       
367

       
+

       
		logs,


     

       

       
368

       
+

       
	)


     

       

       
369

       
+

       
	if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) {


     

       

       
370

       
+

       
		return fmt.Errorf("failed to copy logs: %w", err)


     

       

       
371

       
+

       
	}


     

       

       
372

       
+

       



     

       

       
373

       
+

       
	return nil


     

       

       
374

       
+

       
}


     

       

       
375

       
+

       



     

       

       
376

       
+

       
func (e *Engine) DestroyStep(ctx context.Context, containerID string) error {


     

       

       
377

       
+

       
	err := e.docker.ContainerKill(ctx, containerID, "9") // SIGKILL


     

       

       
378

       
+

       
	if err != nil && !isErrContainerNotFoundOrNotRunning(err) {


     

       

       
379

       
+

       
		return err


     

       

       
380

       
+

       
	}


     

       

       
381

       
+

       



     

       

       
382

       
+

       
	if err := e.docker.ContainerRemove(ctx, containerID, container.RemoveOptions{


     

       

       
383

       
+

       
		RemoveVolumes: true,


     

       

       
384

       
+

       
		RemoveLinks:   false,


     

       

       
385

       
+

       
		Force:         false,


     

       

       
386

       
+

       
	}); err != nil && !isErrContainerNotFoundOrNotRunning(err) {


     

       

       
387

       
+

       
		return err


     

       

       
388

       
+

       
	}


     

       

       
389

       
+

       



     

       

       
390

       
+

       
	return nil


     

       

       
391

       
+

       
}


     

       

       
392

       
+

       



     

       

       
393

       
+

       
func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error {


     

       

       
394

       
+

       
	e.cleanupMu.Lock()


     

       

       
395

       
+

       
	key := wid.String()


     

       

       
396

       
+

       



     

       

       
397

       
+

       
	fns := e.cleanup[key]


     

       

       
398

       
+

       
	delete(e.cleanup, key)


     

       

       
399

       
+

       
	e.cleanupMu.Unlock()


     

       

       
400

       
+

       



     

       

       
401

       
+

       
	for _, fn := range fns {


     

       

       
402

       
+

       
		if err := fn(ctx); err != nil {


     

       

       
403

       
+

       
			e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err)


     

       

       
404

       
+

       
		}


     

       

       
405

       
+

       
	}


     

       

       
406

       
+

       
	return nil


     

       

       
407

       
+

       
}


     

       

       
408

       
+

       



     

       

       
409

       
+

       
func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) {


     

       

       
410

       
+

       
	e.cleanupMu.Lock()


     

       

       
411

       
+

       
	defer e.cleanupMu.Unlock()


     

       

       
412

       
+

       



     

       

       
413

       
+

       
	key := wid.String()


     

       

       
414

       
+

       
	e.cleanup[key] = append(e.cleanup[key], fn)


     

       

       
415

       
+

       
}


     

       

       
416

       
+

       



     

       

       
417

       
+

       
func workspaceVolume(wid models.WorkflowId) string {


     

       

       
418

       
+

       
	return fmt.Sprintf("workspace-%s", wid)


     

       

       
419

       
+

       
}


     

       

       
420

       
+

       



     

       

       
421

       
+

       
func nixVolume(wid models.WorkflowId) string {


     

       

       
422

       
+

       
	return fmt.Sprintf("nix-%s", wid)


     

       

       
423

       
+

       
}


     

       

       
424

       
+

       



     

       

       
425

       
+

       
func networkName(wid models.WorkflowId) string {


     

       

       
426

       
+

       
	return fmt.Sprintf("workflow-network-%s", wid)


     

       

       
427

       
+

       
}


     

       

       
428

       
+

       



     

       

       
429

       
+

       
func hostConfig(wid models.WorkflowId) *container.HostConfig {


     

       

       
430

       
+

       
	hostConfig := &container.HostConfig{


     

       

       
431

       
+

       
		Mounts: []mount.Mount{


     

       

       
432

       
+

       
			{


     

       

       
433

       
+

       
				Type:   mount.TypeVolume,


     

       

       
434

       
+

       
				Source: workspaceVolume(wid),


     

       

       
435

       
+

       
				Target: workspaceDir,


     

       

       
436

       
+

       
			},


     

       

       
437

       
+

       
			{


     

       

       
438

       
+

       
				Type:   mount.TypeVolume,


     

       

       
439

       
+

       
				Source: nixVolume(wid),


     

       

       
440

       
+

       
				Target: "/nix",


     

       

       
441

       
+

       
			},


     

       

       
442

       
+

       
			{


     

       

       
443

       
+

       
				Type:     mount.TypeTmpfs,


     

       

       
444

       
+

       
				Target:   "/tmp",


     

       

       
445

       
+

       
				ReadOnly: false,


     

       

       
446

       
+

       
				TmpfsOptions: &mount.TmpfsOptions{


     

       

       
447

       
+

       
					Mode: 0o1777, // world-writeable sticky bit


     

       

       
448

       
+

       
					Options: [][]string{


     

       

       
449

       
+

       
						{"exec"},


     

       

       
450

       
+

       
					},


     

       

       
451

       
+

       
				},


     

       

       
452

       
+

       
			},


     

       

       
453

       
+

       
			{


     

       

       
454

       
+

       
				Type:   mount.TypeVolume,


     

       

       
455

       
+

       
				Source: "etc-nix-" + wid.String(),


     

       

       
456

       
+

       
				Target: "/etc/nix",


     

       

       
457

       
+

       
			},


     

       

       
458

       
+

       
		},


     

       

       
459

       
+

       
		ReadonlyRootfs: false,


     

       

       
460

       
+

       
		CapDrop:        []string{"ALL"},


     

       

       
461

       
+

       
		CapAdd:         []string{"CAP_DAC_OVERRIDE"},


     

       

       
462

       
+

       
		SecurityOpt:    []string{"no-new-privileges"},


     

       

       
463

       
+

       
		ExtraHosts:     []string{"host.docker.internal:host-gateway"},


     

       

       
464

       
+

       
	}


     

       

       
465

       
+

       



     

       

       
466

       
+

       
	return hostConfig


     

       

       
467

       
+

       
}


     

       

       
468

       
+

       



     

       

       
469

       
+

       
// thanks woodpecker


     

       

       
470

       
+

       
func isErrContainerNotFoundOrNotRunning(err error) bool {


     

       

       
471

       
+

       
	// Error response from daemon: Cannot kill container: ...: No such container: ...


     

       

       
472

       
+

       
	// Error response from daemon: Cannot kill container: ...: Container ... is not running"


     

       

       
473

       
+

       
	// Error response from podman daemon: can only kill running containers. ... is in state exited


     

       

       
474

       
+

       
	// Error: No such container: ...


     

       

       
475

       
+

       
	return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running") || strings.Contains(err.Error(), "can only kill running containers"))


     

       

       
476

       
+

       
}
+7
spindle/engines/nixery/errors.go 
···

       

       
1

       
+

       
package nixery


     

       

       
2

       
+

       



     

       

       
3

       
+

       
import "errors"


     

       

       
4

       
+

       



     

       

       
5

       
+

       
var (


     

       

       
6

       
+

       
	ErrOOMKilled = errors.New("oom killed")


     

       

       
7

       
+

       
)
+17
spindle/models/engine.go 
···

       

       
1

       
+

       
package models


     

       

       
2

       
+

       



     

       

       
3

       
+

       
import (


     

       

       
4

       
+

       
	"context"


     

       

       
5

       
+

       
	"time"


     

       

       
6

       
+

       



     

       

       
7

       
+

       
	"tangled.sh/tangled.sh/core/api/tangled"


     

       

       
8

       
+

       
	"tangled.sh/tangled.sh/core/spindle/secrets"


     

       

       
9

       
+

       
)


     

       

       
10

       
+

       



     

       

       
11

       
+

       
type Engine interface {


     

       

       
12

       
+

       
	InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*Workflow, error)


     

       

       
13

       
+

       
	SetupWorkflow(ctx context.Context, wid WorkflowId, wf *Workflow) error


     

       

       
14

       
+

       
	WorkflowTimeout() time.Duration


     

       

       
15

       
+

       
	DestroyWorkflow(ctx context.Context, wid WorkflowId) error


     

       

       
16

       
+

       
	RunStep(ctx context.Context, wid WorkflowId, w *Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *WorkflowLogger) error


     

       

       
17

       
+

       
}
+3 -3
spindle/models/models.go 
···

       
104

       
104

       
 

       
func NewControlLogLine(idx int, step Step) LogLine {


     

       
105

       
105

       
 

       
	return LogLine{


     

       
106

       
106

       
 

       
		Kind:        LogKindControl,


     

       
107

       

       
-

       
		Content:     step.Name,


     

       

       
107

       
+

       
		Content:     step.Name(),


     

       
108

       
108

       
 

       
		StepId:      idx,


     

       
109

       

       
-

       
		StepKind:    step.Kind,


     

       
110

       

       
-

       
		StepCommand: step.Command,


     

       

       
109

       
+

       
		StepKind:    step.Kind(),


     

       

       
110

       
+

       
		StepCommand: step.Command(),


     

       
111

       
111

       
 

       
	}


     

       
112

       
112

       
 

       
}
+8 -103
spindle/models/pipeline.go 
···

       
1

       
1

       
 

       
package models


     

       
2

       
2

       
 

       



     

       
3

       

       
-

       
import (


     

       
4

       

       
-

       
	"path"


     

       
5

       

       
-

       



     

       
6

       

       
-

       
	"tangled.sh/tangled.sh/core/api/tangled"


     

       
7

       

       
-

       
	"tangled.sh/tangled.sh/core/spindle/config"


     

       
8

       

       
-

       
)


     

       
9

       

       
-

       



     

       
10

       
3

       
 

       
type Pipeline struct {


     

       
11

       
4

       
 

       
	RepoOwner string


     

       
12

       
5

       
 

       
	RepoName  string


     

       
13

       

       
-

       
	Workflows []Workflow


     

       

       
6

       
+

       
	Workflows map[Engine][]Workflow


     

       
14

       
7

       
 

       
}


     

       
15

       
8

       
 

       



     

       
16

       

       
-

       
type Step struct {


     

       
17

       

       
-

       
	Command     string


     

       
18

       

       
-

       
	Name        string


     

       
19

       

       
-

       
	Environment map[string]string


     

       
20

       

       
-

       
	Kind        StepKind


     

       

       
9

       
+

       
type Step interface {


     

       

       
10

       
+

       
	Name() string


     

       

       
11

       
+

       
	Command() string


     

       

       
12

       
+

       
	Kind() StepKind


     

       
21

       
13

       
 

       
}


     

       
22

       
14

       
 

       



     

       
23

       
15

       
 

       
type StepKind int


     
···

       
30

       
22

       
 

       
)


     

       
31

       
23

       
 

       



     

       
32

       
24

       
 

       
type Workflow struct {


     

       
33

       

       
-

       
	Steps       []Step


     

       
34

       

       
-

       
	Environment map[string]string


     

       
35

       

       
-

       
	Name        string


     

       
36

       

       
-

       
	Image       string


     

       
37

       

       
-

       
}


     

       
38

       

       
-

       



     

       
39

       

       
-

       
// setupSteps get added to start of Steps


     

       
40

       

       
-

       
type setupSteps []Step


     

       
41

       

       
-

       



     

       
42

       

       
-

       
// addStep adds a step to the beginning of the workflow's steps.


     

       
43

       

       
-

       
func (ss *setupSteps) addStep(step Step) {


     

       
44

       

       
-

       
	*ss = append(*ss, step)


     

       
45

       

       
-

       
}


     

       
46

       

       
-

       



     

       
47

       

       
-

       
// ToPipeline converts a tangled.Pipeline into a model.Pipeline.


     

       
48

       

       
-

       
// In the process, dependencies are resolved: nixpkgs deps


     

       
49

       

       
-

       
// are constructed atop nixery and set as the Workflow.Image,


     

       
50

       

       
-

       
// and ones from custom registries


     

       
51

       

       
-

       
func ToPipeline(pl tangled.Pipeline, cfg config.Config) *Pipeline {


     

       
52

       

       
-

       
	workflows := []Workflow{}


     

       
53

       

       
-

       



     

       
54

       

       
-

       
	for _, twf := range pl.Workflows {


     

       
55

       

       
-

       
		swf := &Workflow{}


     

       
56

       

       
-

       
		for _, tstep := range twf.Steps {


     

       
57

       

       
-

       
			sstep := Step{}


     

       
58

       

       
-

       
			sstep.Environment = stepEnvToMap(tstep.Environment)


     

       
59

       

       
-

       
			sstep.Command = tstep.Command


     

       
60

       

       
-

       
			sstep.Name = tstep.Name


     

       
61

       

       
-

       
			sstep.Kind = StepKindUser


     

       
62

       

       
-

       
			swf.Steps = append(swf.Steps, sstep)


     

       
63

       

       
-

       
		}


     

       
64

       

       
-

       
		swf.Name = twf.Name


     

       
65

       

       
-

       
		swf.Environment = workflowEnvToMap(twf.Environment)


     

       
66

       

       
-

       
		swf.Image = workflowImage(twf.Dependencies, cfg.Pipelines.Nixery)


     

       
67

       

       
-

       



     

       
68

       

       
-

       
		setup := &setupSteps{}


     

       
69

       

       
-

       



     

       
70

       

       
-

       
		setup.addStep(nixConfStep())


     

       
71

       

       
-

       
		setup.addStep(cloneStep(*twf, *pl.TriggerMetadata, cfg.Server.Dev))


     

       
72

       

       
-

       
		// this step could be empty


     

       
73

       

       
-

       
		if s := dependencyStep(*twf); s != nil {


     

       
74

       

       
-

       
			setup.addStep(*s)


     

       
75

       

       
-

       
		}


     

       
76

       

       
-

       



     

       
77

       

       
-

       
		// append setup steps in order to the start of workflow steps


     

       
78

       

       
-

       
		swf.Steps = append(*setup, swf.Steps...)


     

       
79

       

       
-

       



     

       
80

       

       
-

       
		workflows = append(workflows, *swf)


     

       
81

       

       
-

       
	}


     

       
82

       

       
-

       
	repoOwner := pl.TriggerMetadata.Repo.Did


     

       
83

       

       
-

       
	repoName := pl.TriggerMetadata.Repo.Repo


     

       
84

       

       
-

       
	return &Pipeline{


     

       
85

       

       
-

       
		RepoOwner: repoOwner,


     

       
86

       

       
-

       
		RepoName:  repoName,


     

       
87

       

       
-

       
		Workflows: workflows,


     

       
88

       

       
-

       
	}


     

       
89

       

       
-

       
}


     

       
90

       

       
-

       



     

       
91

       

       
-

       
func workflowEnvToMap(envs []*tangled.Pipeline_Pair) map[string]string {


     

       
92

       

       
-

       
	envMap := map[string]string{}


     

       
93

       

       
-

       
	for _, env := range envs {


     

       
94

       

       
-

       
		if env != nil {


     

       
95

       

       
-

       
			envMap[env.Key] = env.Value


     

       
96

       

       
-

       
		}


     

       
97

       

       
-

       
	}


     

       
98

       

       
-

       
	return envMap


     

       
99

       

       
-

       
}


     

       
100

       

       
-

       



     

       
101

       

       
-

       
func stepEnvToMap(envs []*tangled.Pipeline_Pair) map[string]string {


     

       
102

       

       
-

       
	envMap := map[string]string{}


     

       
103

       

       
-

       
	for _, env := range envs {


     

       
104

       

       
-

       
		if env != nil {


     

       
105

       

       
-

       
			envMap[env.Key] = env.Value


     

       
106

       

       
-

       
		}


     

       
107

       

       
-

       
	}


     

       
108

       

       
-

       
	return envMap


     

       
109

       

       
-

       
}


     

       
110

       

       
-

       



     

       
111

       

       
-

       
func workflowImage(deps []*tangled.Pipeline_Dependency, nixery string) string {


     

       
112

       

       
-

       
	var dependencies string


     

       
113

       

       
-

       
	for _, d := range deps {


     

       
114

       

       
-

       
		if d.Registry == "nixpkgs" {


     

       
115

       

       
-

       
			dependencies = path.Join(d.Packages...)


     

       
116

       

       
-

       
		}


     

       
117

       

       
-

       
	}


     

       
118

       

       
-

       



     

       
119

       

       
-

       
	// load defaults from somewhere else


     

       
120

       

       
-

       
	dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix")


     

       
121

       

       
-

       



     

       
122

       

       
-

       
	return path.Join(nixery, dependencies)


     

       

       
25

       
+

       
	Steps []Step


     

       

       
26

       
+

       
	Name  string


     

       

       
27

       
+

       
	Data  any


     

       
123

       
28

       
 

       
}
+10 -13
spindle/models/setup_steps.go spindle/engines/nixery/setup_steps.go 
···

       
1

       

       
-

       
package models


     

       

       
1

       
+

       
package nixery


     

       
2

       
2

       
 

       



     

       
3

       
3

       
 

       
import (


     

       
4

       
4

       
 

       
	"fmt"


     
···

       
13

       
13

       
 

       
	setupCmd := `echo 'extra-experimental-features = nix-command flakes' >> /etc/nix/nix.conf


     

       
14

       
14

       
 

       
echo 'build-users-group = ' >> /etc/nix/nix.conf`


     

       
15

       
15

       
 

       
	return Step{


     

       
16

       

       
-

       
		Command: setupCmd,


     

       
17

       

       
-

       
		Name:    "Configure Nix",


     

       

       
16

       
+

       
		command: setupCmd,


     

       

       
17

       
+

       
		name:    "Configure Nix",


     

       
18

       
18

       
 

       
	}


     

       
19

       
19

       
 

       
}


     

       
20

       
20

       
 

       



     
···

       
81

       
81

       
 

       
	commands = append(commands, "git checkout FETCH_HEAD")


     

       
82

       
82

       
 

       



     

       
83

       
83

       
 

       
	cloneStep := Step{


     

       
84

       

       
-

       
		Command: strings.Join(commands, "\n"),


     

       
85

       

       
-

       
		Name:    "Clone repository into workspace",


     

       

       
84

       
+

       
		command: strings.Join(commands, "\n"),


     

       

       
85

       
+

       
		name:    "Clone repository into workspace",


     

       
86

       
86

       
 

       
	}


     

       
87

       
87

       
 

       
	return cloneStep


     

       
88

       
88

       
 

       
}


     
···

       
91

       
91

       
 

       
// For dependencies using a custom registry (i.e. not nixpkgs), it collects


     

       
92

       
92

       
 

       
// all packages and adds a single 'nix profile install' step to the


     

       
93

       
93

       
 

       
// beginning of the workflow's step list.


     

       
94

       

       
-

       
func dependencyStep(twf tangled.Pipeline_Workflow) *Step {


     

       

       
94

       
+

       
func dependencyStep(deps map[string][]string) *Step {


     

       
95

       
95

       
 

       
	var customPackages []string


     

       
96

       
96

       
 

       



     

       
97

       

       
-

       
	for _, d := range twf.Dependencies {


     

       
98

       

       
-

       
		registry := d.Registry


     

       
99

       

       
-

       
		packages := d.Packages


     

       
100

       

       
-

       



     

       

       
97

       
+

       
	for registry, packages := range deps {


     

       
101

       
98

       
 

       
		if registry == "nixpkgs" {


     

       
102

       
99

       
 

       
			continue


     

       
103

       
100

       
 

       
		}


     
···

       
115

       
112

       
 

       
		installCmd := "nix --extra-experimental-features nix-command --extra-experimental-features flakes profile install"


     

       
116

       
113

       
 

       
		cmd := fmt.Sprintf("%s %s", installCmd, strings.Join(customPackages, " "))


     

       
117

       
114

       
 

       
		installStep := Step{


     

       
118

       

       
-

       
			Command: cmd,


     

       
119

       

       
-

       
			Name:    "Install custom dependencies",


     

       
120

       

       
-

       
			Environment: map[string]string{


     

       

       
115

       
+

       
			command: cmd,


     

       

       
116

       
+

       
			name:    "Install custom dependencies",


     

       

       
117

       
+

       
			environment: map[string]string{


     

       
121

       
118

       
 

       
				"NIX_NO_COLOR":               "1",


     

       
122

       
119

       
 

       
				"NIX_SHOW_DOWNLOAD_PROGRESS": "0",


     

       
123

       
120

       
 

       
			},
+38 -8
spindle/server.go 
···

       
20

       
20

       
 

       
	"tangled.sh/tangled.sh/core/spindle/config"


     

       
21

       
21

       
 

       
	"tangled.sh/tangled.sh/core/spindle/db"


     

       
22

       
22

       
 

       
	"tangled.sh/tangled.sh/core/spindle/engine"


     

       

       
23

       
+

       
	"tangled.sh/tangled.sh/core/spindle/engines/nixery"


     

       
23

       
24

       
 

       
	"tangled.sh/tangled.sh/core/spindle/models"


     

       
24

       
25

       
 

       
	"tangled.sh/tangled.sh/core/spindle/queue"


     

       
25

       
26

       
 

       
	"tangled.sh/tangled.sh/core/spindle/secrets"


     
···

       
39

       
40

       
 

       
	e     *rbac.Enforcer


     

       
40

       
41

       
 

       
	l     *slog.Logger


     

       
41

       
42

       
 

       
	n     *notifier.Notifier


     

       
42

       

       
-

       
	eng   *engine.Engine


     

       

       
43

       
+

       
	engs  map[string]models.Engine


     

       
43

       
44

       
 

       
	jq    *queue.Queue


     

       
44

       
45

       
 

       
	cfg   *config.Config


     

       
45

       
46

       
 

       
	ks    *eventconsumer.Consumer


     
···

       
93

       
94

       
 

       
		return fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider)


     

       
94

       
95

       
 

       
	}


     

       
95

       
96

       
 

       



     

       
96

       

       
-

       
	eng, err := engine.New(ctx, cfg, d, &n, vault)


     

       

       
97

       
+

       
	nixeryEng, err := nixery.New(ctx, cfg)


     

       
97

       
98

       
 

       
	if err != nil {


     

       
98

       
99

       
 

       
		return err


     

       
99

       
100

       
 

       
	}


     
···

       
128

       
129

       
 

       
		db:    d,


     

       
129

       
130

       
 

       
		l:     logger,


     

       
130

       
131

       
 

       
		n:     &n,


     

       
131

       

       
-

       
		eng:   eng,


     

       

       
132

       
+

       
		engs:  map[string]models.Engine{"nixery": nixeryEng},


     

       
132

       
133

       
 

       
		jq:    jq,


     

       
133

       
134

       
 

       
		cfg:   cfg,


     

       
134

       
135

       
 

       
		res:   resolver,


     
···

       
216

       
217

       
 

       
		Logger:   logger,


     

       
217

       
218

       
 

       
		Db:       s.db,


     

       
218

       
219

       
 

       
		Enforcer: s.e,


     

       
219

       

       
-

       
		Engine:   s.eng,


     

       

       
220

       
+

       
		Engines:  s.engs,


     

       
220

       
221

       
 

       
		Config:   s.cfg,


     

       
221

       
222

       
 

       
		Resolver: s.res,


     

       
222

       
223

       
 

       
		Vault:    s.vault,


     
···

       
261

       
262

       
 

       
			Rkey: msg.Rkey,


     

       
262

       
263

       
 

       
		}


     

       
263

       
264

       
 

       



     

       

       
265

       
+

       
		workflows := make(map[models.Engine][]models.Workflow)


     

       

       
266

       
+

       



     

       
264

       
267

       
 

       
		for _, w := range tpl.Workflows {


     

       
265

       
268

       
 

       
			if w != nil {


     

       
266

       

       
-

       
				err := s.db.StatusPending(models.WorkflowId{


     

       

       
269

       
+

       
				if _, ok := s.engs[w.Engine]; !ok {


     

       

       
270

       
+

       
					err = s.db.StatusFailed(models.WorkflowId{


     

       

       
271

       
+

       
						PipelineId: pipelineId,


     

       

       
272

       
+

       
						Name:       w.Name,


     

       

       
273

       
+

       
					}, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n)


     

       

       
274

       
+

       
					if err != nil {


     

       

       
275

       
+

       
						return err


     

       

       
276

       
+

       
					}


     

       

       
277

       
+

       



     

       

       
278

       
+

       
					continue


     

       

       
279

       
+

       
				}


     

       

       
280

       
+

       



     

       

       
281

       
+

       
				eng := s.engs[w.Engine]


     

       

       
282

       
+

       



     

       

       
283

       
+

       
				if _, ok := workflows[eng]; !ok {


     

       

       
284

       
+

       
					workflows[eng] = []models.Workflow{}


     

       

       
285

       
+

       
				}


     

       

       
286

       
+

       



     

       

       
287

       
+

       
				ewf, err := s.engs[w.Engine].InitWorkflow(*w, tpl)


     

       

       
288

       
+

       
				if err != nil {


     

       

       
289

       
+

       
					return err


     

       

       
290

       
+

       
				}


     

       

       
291

       
+

       



     

       

       
292

       
+

       
				workflows[eng] = append(workflows[eng], *ewf)


     

       

       
293

       
+

       



     

       

       
294

       
+

       
				err = s.db.StatusPending(models.WorkflowId{


     

       
267

       
295

       
 

       
					PipelineId: pipelineId,


     

       
268

       
296

       
 

       
					Name:       w.Name,


     

       
269

       
297

       
 

       
				}, s.n)


     
···

       
273

       
301

       
 

       
			}


     

       
274

       
302

       
 

       
		}


     

       
275

       
303

       
 

       



     

       
276

       

       
-

       
		spl := models.ToPipeline(tpl, *s.cfg)


     

       
277

       

       
-

       



     

       
278

       
304

       
 

       
		ok := s.jq.Enqueue(queue.Job{


     

       
279

       
305

       
 

       
			Run: func() error {


     

       
280

       

       
-

       
				s.eng.StartWorkflows(ctx, spl, pipelineId)


     

       

       
306

       
+

       
				engine.StartWorkflows(s.l, s.vault, s.cfg, s.db, s.n, ctx, &models.Pipeline{


     

       

       
307

       
+

       
					RepoOwner: tpl.TriggerMetadata.Repo.Did,


     

       

       
308

       
+

       
					RepoName:  tpl.TriggerMetadata.Repo.Repo,


     

       

       
309

       
+

       
					Workflows: workflows,


     

       

       
310

       
+

       
				}, pipelineId)


     

       
281

       
311

       
 

       
				return nil


     

       
282

       
312

       
 

       
			},


     

       
283

       
313

       
 

       
			OnFail: func(jobError error) {
+1 -2
spindle/stream.go 
···

       
9

       
9

       
 

       
	"strconv"


     

       
10

       
10

       
 

       
	"time"


     

       
11

       
11

       
 

       



     

       
12

       

       
-

       
	"tangled.sh/tangled.sh/core/spindle/engine"


     

       
13

       
12

       
 

       
	"tangled.sh/tangled.sh/core/spindle/models"


     

       
14

       
13

       
 

       



     

       
15

       
14

       
 

       
	"github.com/go-chi/chi/v5"


     
···

       
143

       
142

       
 

       
	}


     

       
144

       
143

       
 

       
	isFinished := models.StatusKind(status.Status).IsFinish()


     

       
145

       
144

       
 

       



     

       
146

       

       
-

       
	filePath := engine.LogFilePath(s.cfg.Pipelines.LogDir, wid)


     

       

       
145

       
+

       
	filePath := models.LogFilePath(s.cfg.Server.LogDir, wid)


     

       
147

       
146

       
 

       



     

       
148

       
147

       
 

       
	config := tail.Config{


     

       
149

       
148

       
 

       
		Follow:    !isFinished,
+2 -2
spindle/xrpc/xrpc.go 
···

       
17

       
17

       
 

       
	"tangled.sh/tangled.sh/core/rbac"


     

       
18

       
18

       
 

       
	"tangled.sh/tangled.sh/core/spindle/config"


     

       
19

       
19

       
 

       
	"tangled.sh/tangled.sh/core/spindle/db"


     

       
20

       

       
-

       
	"tangled.sh/tangled.sh/core/spindle/engine"


     

       

       
20

       
+

       
	"tangled.sh/tangled.sh/core/spindle/models"


     

       
21

       
21

       
 

       
	"tangled.sh/tangled.sh/core/spindle/secrets"


     

       
22

       
22

       
 

       
)


     

       
23

       
23

       
 

       



     
···

       
27

       
27

       
 

       
	Logger   *slog.Logger


     

       
28

       
28

       
 

       
	Db       *db.DB


     

       
29

       
29

       
 

       
	Enforcer *rbac.Enforcer


     

       
30

       

       
-

       
	Engine   *engine.Engine


     

       

       
30

       
+

       
	Engines  map[string]models.Engine


     

       
31

       
31

       
 

       
	Config   *config.Config


     

       
32

       
32

       
 

       
	Resolver *idresolver.Resolver


     

       
33

       
33

       
 

       
	Vault    secrets.Manager
+17 -36
workflow/compile.go 
···

       
1

       
1

       
 

       
package workflow


     

       
2

       
2

       
 

       



     

       
3

       
3

       
 

       
import (


     

       

       
4

       
+

       
	"errors"


     

       
4

       
5

       
 

       
	"fmt"


     

       
5

       
6

       
 

       



     

       
6

       
7

       
 

       
	"tangled.sh/tangled.sh/core/api/tangled"


     
···

       
63

       
64

       
 

       
	return fmt.Sprintf("warning: %s: %s: %s", w.Path, w.Type, w.Reason)


     

       
64

       
65

       
 

       
}


     

       
65

       
66

       
 

       



     

       

       
67

       
+

       
var (


     

       

       
68

       
+

       
	MissingEngine error = errors.New("missing engine")


     

       

       
69

       
+

       
)


     

       

       
70

       
+

       



     

       
66

       
71

       
 

       
type WarningKind string


     

       
67

       
72

       
 

       



     

       
68

       
73

       
 

       
var (


     
···

       
95

       
100

       
 

       
	for _, wf := range p {


     

       
96

       
101

       
 

       
		cw := compiler.compileWorkflow(wf)


     

       
97

       
102

       
 

       



     

       
98

       

       
-

       
		// empty workflows are not added to the pipeline


     

       
99

       

       
-

       
		if len(cw.Steps) == 0 {


     

       

       
103

       
+

       
		if cw == nil {


     

       
100

       
104

       
 

       
			continue


     

       
101

       
105

       
 

       
		}


     

       
102

       
106

       
 

       



     

       
103

       

       
-

       
		cp.Workflows = append(cp.Workflows, &cw)


     

       

       
107

       
+

       
		cp.Workflows = append(cp.Workflows, cw)


     

       
104

       
108

       
 

       
	}


     

       
105

       
109

       
 

       



     

       
106

       
110

       
 

       
	return cp


     

       
107

       
111

       
 

       
}


     

       
108

       
112

       
 

       



     

       
109

       

       
-

       
func (compiler *Compiler) compileWorkflow(w Workflow) tangled.Pipeline_Workflow {


     

       
110

       

       
-

       
	cw := tangled.Pipeline_Workflow{}


     

       

       
113

       
+

       
func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow {


     

       

       
114

       
+

       
	cw := &tangled.Pipeline_Workflow{}


     

       
111

       
115

       
 

       



     

       
112

       
116

       
 

       
	if !w.Match(compiler.Trigger) {


     

       
113

       
117

       
 

       
		compiler.Diagnostics.AddWarning(


     
···

       
115

       
119

       
 

       
			WorkflowSkipped,


     

       
116

       
120

       
 

       
			fmt.Sprintf("did not match trigger %s", compiler.Trigger.Kind),


     

       
117

       
121

       
 

       
		)


     

       
118

       

       
-

       
		return cw


     

       
119

       

       
-

       
	}


     

       
120

       

       
-

       



     

       
121

       

       
-

       
	if len(w.Steps) == 0 {


     

       
122

       

       
-

       
		compiler.Diagnostics.AddWarning(


     

       
123

       

       
-

       
			w.Name,


     

       
124

       

       
-

       
			WorkflowSkipped,


     

       
125

       

       
-

       
			"empty workflow",


     

       
126

       

       
-

       
		)


     

       
127

       

       
-

       
		return cw


     

       

       
122

       
+

       
		return nil


     

       
128

       
123

       
 

       
	}


     

       
129

       
124

       
 

       



     

       
130

       
125

       
 

       
	// validate clone options


     

       
131

       
126

       
 

       
	compiler.analyzeCloneOptions(w)


     

       
132

       
127

       
 

       



     

       
133

       
128

       
 

       
	cw.Name = w.Name


     

       
134

       

       
-

       
	cw.Dependencies = w.Dependencies.AsRecord()


     

       
135

       

       
-

       
	for _, s := range w.Steps {


     

       
136

       

       
-

       
		step := tangled.Pipeline_Step{


     

       
137

       

       
-

       
			Command: s.Command,


     

       
138

       

       
-

       
			Name:    s.Name,


     

       
139

       

       
-

       
		}


     

       
140

       

       
-

       
		for k, v := range s.Environment {


     

       
141

       

       
-

       
			e := &tangled.Pipeline_Pair{


     

       
142

       

       
-

       
				Key:   k,


     

       
143

       

       
-

       
				Value: v,


     

       
144

       

       
-

       
			}


     

       
145

       

       
-

       
			step.Environment = append(step.Environment, e)


     

       
146

       

       
-

       
		}


     

       
147

       

       
-

       
		cw.Steps = append(cw.Steps, &step)


     

       

       
129

       
+

       



     

       

       
130

       
+

       
	if w.Engine == "" {


     

       

       
131

       
+

       
		compiler.Diagnostics.AddError(w.Name, MissingEngine)


     

       

       
132

       
+

       
		return nil


     

       
148

       
133

       
 

       
	}


     

       
149

       

       
-

       
	for k, v := range w.Environment {


     

       
150

       

       
-

       
		e := &tangled.Pipeline_Pair{


     

       
151

       

       
-

       
			Key:   k,


     

       
152

       

       
-

       
			Value: v,


     

       
153

       

       
-

       
		}


     

       
154

       

       
-

       
		cw.Environment = append(cw.Environment, e)


     

       
155

       

       
-

       
	}


     

       

       
134

       
+

       



     

       

       
135

       
+

       
	cw.Engine = w.Engine


     

       

       
136

       
+

       
	cw.Raw = w.Raw


     

       
156

       
137

       
 

       



     

       
157

       
138

       
 

       
	o := w.CloneOpts.AsRecord()


     

       
158

       
139

       
 

       
	cw.Clone = &o
+23 -29
workflow/compile_test.go 
···

       
26

       
26

       
 

       



     

       
27

       
27

       
 

       
func TestCompileWorkflow_MatchingWorkflowWithSteps(t *testing.T) {


     

       
28

       
28

       
 

       
	wf := Workflow{


     

       
29

       

       
-

       
		Name: ".tangled/workflows/test.yml",


     

       
30

       

       
-

       
		When: when,


     

       
31

       

       
-

       
		Steps: []Step{


     

       
32

       

       
-

       
			{Name: "Test", Command: "go test ./..."},


     

       
33

       

       
-

       
		},


     

       

       
29

       
+

       
		Name:      ".tangled/workflows/test.yml",


     

       

       
30

       
+

       
		Engine:    "nixery",


     

       

       
31

       
+

       
		When:      when,


     

       
34

       
32

       
 

       
		CloneOpts: CloneOpts{}, // default true


     

       
35

       
33

       
 

       
	}


     

       
36

       
34

       
 

       



     
···

       
43

       
41

       
 

       
	assert.False(t, c.Diagnostics.IsErr())


     

       
44

       
42

       
 

       
}


     

       
45

       
43

       
 

       



     

       
46

       

       
-

       
func TestCompileWorkflow_EmptySteps(t *testing.T) {


     

       
47

       

       
-

       
	wf := Workflow{


     

       
48

       

       
-

       
		Name:  ".tangled/workflows/empty.yml",


     

       
49

       

       
-

       
		When:  when,


     

       
50

       

       
-

       
		Steps: []Step{}, // no steps


     

       
51

       

       
-

       
	}


     

       
52

       

       
-

       



     

       
53

       

       
-

       
	c := Compiler{Trigger: trigger}


     

       
54

       

       
-

       
	cp := c.Compile([]Workflow{wf})


     

       
55

       

       
-

       



     

       
56

       

       
-

       
	assert.Len(t, cp.Workflows, 0)


     

       
57

       

       
-

       
	assert.Len(t, c.Diagnostics.Warnings, 1)


     

       
58

       

       
-

       
	assert.Equal(t, WorkflowSkipped, c.Diagnostics.Warnings[0].Type)


     

       
59

       

       
-

       
}


     

       
60

       

       
-

       



     

       
61

       
44

       
 

       
func TestCompileWorkflow_TriggerMismatch(t *testing.T) {


     

       
62

       
45

       
 

       
	wf := Workflow{


     

       
63

       

       
-

       
		Name: ".tangled/workflows/mismatch.yml",


     

       

       
46

       
+

       
		Name:   ".tangled/workflows/mismatch.yml",


     

       

       
47

       
+

       
		Engine: "nixery",


     

       
64

       
48

       
 

       
		When: []Constraint{


     

       
65

       
49

       
 

       
			{


     

       
66

       
50

       
 

       
				Event:  []string{"push"},


     

       
67

       
51

       
 

       
				Branch: []string{"master"}, // different branch


     

       
68

       
52

       
 

       
			},


     

       
69

       
53

       
 

       
		},


     

       
70

       

       
-

       
		Steps: []Step{


     

       
71

       

       
-

       
			{Name: "Lint", Command: "golint ./..."},


     

       
72

       

       
-

       
		},


     

       
73

       
54

       
 

       
	}


     

       
74

       
55

       
 

       



     

       
75

       
56

       
 

       
	c := Compiler{Trigger: trigger}


     
···

       
82

       
63

       
 

       



     

       
83

       
64

       
 

       
func TestCompileWorkflow_CloneFalseWithShallowTrue(t *testing.T) {


     

       
84

       
65

       
 

       
	wf := Workflow{


     

       
85

       

       
-

       
		Name: ".tangled/workflows/clone_skip.yml",


     

       
86

       

       
-

       
		When: when,


     

       
87

       

       
-

       
		Steps: []Step{


     

       
88

       

       
-

       
			{Name: "Skip", Command: "echo skip"},


     

       
89

       

       
-

       
		},


     

       

       
66

       
+

       
		Name:   ".tangled/workflows/clone_skip.yml",


     

       

       
67

       
+

       
		Engine: "nixery",


     

       

       
68

       
+

       
		When:   when,


     

       
90

       
69

       
 

       
		CloneOpts: CloneOpts{


     

       
91

       
70

       
 

       
			Skip:  true,


     

       
92

       
71

       
 

       
			Depth: 1,


     
···

       
101

       
80

       
 

       
	assert.Len(t, c.Diagnostics.Warnings, 1)


     

       
102

       
81

       
 

       
	assert.Equal(t, InvalidConfiguration, c.Diagnostics.Warnings[0].Type)


     

       
103

       
82

       
 

       
}


     

       

       
83

       
+

       



     

       

       
84

       
+

       
func TestCompileWorkflow_MissingEngine(t *testing.T) {


     

       

       
85

       
+

       
	wf := Workflow{


     

       

       
86

       
+

       
		Name:   ".tangled/workflows/missing_engine.yml",


     

       

       
87

       
+

       
		When:   when,


     

       

       
88

       
+

       
		Engine: "",


     

       

       
89

       
+

       
	}


     

       

       
90

       
+

       



     

       

       
91

       
+

       
	c := Compiler{Trigger: trigger}


     

       

       
92

       
+

       
	cp := c.Compile([]Workflow{wf})


     

       

       
93

       
+

       



     

       

       
94

       
+

       
	assert.Len(t, cp.Workflows, 0)


     

       

       
95

       
+

       
	assert.Len(t, c.Diagnostics.Errors, 1)


     

       

       
96

       
+

       
	assert.Equal(t, MissingEngine, c.Diagnostics.Errors[0].Error)


     

       

       
97

       
+

       
}
+6 -33
workflow/def.go 
···

       
24

       
24

       
 

       



     

       
25

       
25

       
 

       
	// this is simply a structural representation of the workflow file


     

       
26

       
26

       
 

       
	Workflow struct {


     

       
27

       

       
-

       
		Name         string            `yaml:"-"` // name of the workflow file


     

       
28

       

       
-

       
		When         []Constraint      `yaml:"when"`


     

       
29

       

       
-

       
		Dependencies Dependencies      `yaml:"dependencies"`


     

       
30

       

       
-

       
		Steps        []Step            `yaml:"steps"`


     

       
31

       

       
-

       
		Environment  map[string]string `yaml:"environment"`


     

       
32

       

       
-

       
		CloneOpts    CloneOpts         `yaml:"clone"`


     

       

       
27

       
+

       
		Name      string       `yaml:"-"` // name of the workflow file


     

       

       
28

       
+

       
		Engine    string       `yaml:"engine"`


     

       

       
29

       
+

       
		When      []Constraint `yaml:"when"`


     

       

       
30

       
+

       
		CloneOpts CloneOpts    `yaml:"clone"`


     

       

       
31

       
+

       
		Raw       string       `yaml:"-"`


     

       
33

       
32

       
 

       
	}


     

       
34

       
33

       
 

       



     

       
35

       
34

       
 

       
	Constraint struct {


     

       
36

       
35

       
 

       
		Event  StringList `yaml:"event"`


     

       
37

       
36

       
 

       
		Branch StringList `yaml:"branch"` // this is optional, and only applied on "push" events


     

       
38

       
37

       
 

       
	}


     

       
39

       

       
-

       



     

       
40

       

       
-

       
	Dependencies map[string][]string


     

       
41

       
38

       
 

       



     

       
42

       
39

       
 

       
	CloneOpts struct {


     

       
43

       
40

       
 

       
		Skip              bool `yaml:"skip"`


     

       
44

       
41

       
 

       
		Depth             int  `yaml:"depth"`


     

       
45

       
42

       
 

       
		IncludeSubmodules bool `yaml:"submodules"`


     

       
46

       

       
-

       
	}


     

       
47

       

       
-

       



     

       
48

       

       
-

       
	Step struct {


     

       
49

       

       
-

       
		Name        string            `yaml:"name"`


     

       
50

       

       
-

       
		Command     string            `yaml:"command"`


     

       
51

       

       
-

       
		Environment map[string]string `yaml:"environment"`


     

       
52

       
43

       
 

       
	}


     

       
53

       
44

       
 

       



     

       
54

       
45

       
 

       
	StringList []string


     
···

       
77

       
68

       
 

       
	}


     

       
78

       
69

       
 

       



     

       
79

       
70

       
 

       
	wf.Name = name


     

       

       
71

       
+

       
	wf.Raw = string(contents)


     

       
80

       
72

       
 

       



     

       
81

       
73

       
 

       
	return wf, nil


     

       
82

       
74

       
 

       
}


     
···

       
173

       
165

       
 

       
	}


     

       
174

       
166

       
 

       



     

       
175

       
167

       
 

       
	return errors.New("failed to unmarshal StringOrSlice")


     

       
176

       

       
-

       
}


     

       
177

       

       
-

       



     

       
178

       

       
-

       
// conversion utilities to atproto records


     

       
179

       

       
-

       
func (d Dependencies) AsRecord() []*tangled.Pipeline_Dependency {


     

       
180

       

       
-

       
	var deps []*tangled.Pipeline_Dependency


     

       
181

       

       
-

       
	for registry, packages := range d {


     

       
182

       

       
-

       
		deps = append(deps, &tangled.Pipeline_Dependency{


     

       
183

       

       
-

       
			Registry: registry,


     

       
184

       

       
-

       
			Packages: packages,


     

       
185

       

       
-

       
		})


     

       
186

       

       
-

       
	}


     

       
187

       

       
-

       
	return deps


     

       
188

       

       
-

       
}


     

       
189

       

       
-

       



     

       
190

       

       
-

       
func (s Step) AsRecord() tangled.Pipeline_Step {


     

       
191

       

       
-

       
	return tangled.Pipeline_Step{


     

       
192

       

       
-

       
		Command: s.Command,


     

       
193

       

       
-

       
		Name:    s.Name,


     

       
194

       

       
-

       
	}


     

       
195

       
168

       
 

       
}


     

       
196

       
169

       
 

       



     

       
197

       
170

       
 

       
func (c CloneOpts) AsRecord() tangled.Pipeline_CloneOpts {
+1 -86
workflow/def_test.go 
···

       
10

       
10

       
 

       
	yamlData := `


     

       
11

       
11

       
 

       
when:


     

       
12

       
12

       
 

       
  - event: ["push", "pull_request"]


     

       
13

       

       
-

       
    branch: ["main", "develop"]


     

       
14

       

       
-

       



     

       
15

       

       
-

       
dependencies:


     

       
16

       

       
-

       
  nixpkgs:


     

       
17

       

       
-

       
    - go


     

       
18

       

       
-

       
    - git


     

       
19

       

       
-

       
    - curl


     

       
20

       

       
-

       



     

       
21

       

       
-

       
steps:


     

       
22

       

       
-

       
  - name: "Test"


     

       
23

       

       
-

       
    command: |


     

       
24

       

       
-

       
       go test ./...`


     

       

       
13

       
+

       
    branch: ["main", "develop"]`


     

       
25

       
14

       
 

       



     

       
26

       
15

       
 

       
	wf, err := FromFile("test.yml", []byte(yamlData))


     

       
27

       
16

       
 

       
	assert.NoError(t, err, "YAML should unmarshal without error")


     
···

       
30

       
19

       
 

       
	assert.ElementsMatch(t, []string{"main", "develop"}, wf.When[0].Branch)


     

       
31

       
20

       
 

       
	assert.ElementsMatch(t, []string{"push", "pull_request"}, wf.When[0].Event)


     

       
32

       
21

       
 

       



     

       
33

       

       
-

       
	assert.Len(t, wf.Steps, 1)


     

       
34

       

       
-

       
	assert.Equal(t, "Test", wf.Steps[0].Name)


     

       
35

       

       
-

       
	assert.Equal(t, "go test ./...", wf.Steps[0].Command)


     

       
36

       

       
-

       



     

       
37

       

       
-

       
	pkgs, ok := wf.Dependencies["nixpkgs"]


     

       
38

       

       
-

       
	assert.True(t, ok, "`nixpkgs` should be present in dependencies")


     

       
39

       

       
-

       
	assert.ElementsMatch(t, []string{"go", "git", "curl"}, pkgs)


     

       
40

       

       
-

       



     

       
41

       
22

       
 

       
	assert.False(t, wf.CloneOpts.Skip, "Skip should default to false")


     

       
42

       
23

       
 

       
}


     

       
43

       
24

       
 

       



     

       
44

       

       
-

       
func TestUnmarshalCustomRegistry(t *testing.T) {


     

       
45

       

       
-

       
	yamlData := `


     

       
46

       

       
-

       
when:


     

       
47

       

       
-

       
  - event: push


     

       
48

       

       
-

       
    branch: main


     

       
49

       

       
-

       



     

       
50

       

       
-

       
dependencies:


     

       
51

       

       
-

       
  git+https://tangled.sh/@oppi.li/tbsp:


     

       
52

       

       
-

       
    - tbsp


     

       
53

       

       
-

       
  git+https://git.peppe.rs/languages/statix:


     

       
54

       

       
-

       
    - statix


     

       
55

       

       
-

       



     

       
56

       

       
-

       
steps:


     

       
57

       

       
-

       
  - name: "Check"


     

       
58

       

       
-

       
    command: |


     

       
59

       

       
-

       
       statix check`


     

       
60

       

       
-

       



     

       
61

       

       
-

       
	wf, err := FromFile("test.yml", []byte(yamlData))


     

       
62

       

       
-

       
	assert.NoError(t, err, "YAML should unmarshal without error")


     

       
63

       

       
-

       



     

       
64

       

       
-

       
	assert.ElementsMatch(t, []string{"push"}, wf.When[0].Event)


     

       
65

       

       
-

       
	assert.ElementsMatch(t, []string{"main"}, wf.When[0].Branch)


     

       
66

       

       
-

       



     

       
67

       

       
-

       
	assert.ElementsMatch(t, []string{"tbsp"}, wf.Dependencies["git+https://tangled.sh/@oppi.li/tbsp"])


     

       
68

       

       
-

       
	assert.ElementsMatch(t, []string{"statix"}, wf.Dependencies["git+https://git.peppe.rs/languages/statix"])


     

       
69

       

       
-

       
}


     

       
70

       

       
-

       



     

       
71

       
25

       
 

       
func TestUnmarshalCloneFalse(t *testing.T) {


     

       
72

       
26

       
 

       
	yamlData := `


     

       
73

       
27

       
 

       
when:


     
···

       
75

       
29

       
 

       



     

       
76

       
30

       
 

       
clone:


     

       
77

       
31

       
 

       
  skip: true


     

       
78

       

       
-

       



     

       
79

       

       
-

       
dependencies:


     

       
80

       

       
-

       
  nixpkgs:


     

       
81

       

       
-

       
    - python3


     

       
82

       

       
-

       



     

       
83

       

       
-

       
steps:


     

       
84

       

       
-

       
  - name: Notify


     

       
85

       

       
-

       
    command: |


     

       
86

       

       
-

       
      python3 ./notify.py


     

       
87

       
32

       
 

       
`


     

       
88

       
33

       
 

       



     

       
89

       
34

       
 

       
	wf, err := FromFile("test.yml", []byte(yamlData))


     
···

       
93

       
38

       
 

       



     

       
94

       
39

       
 

       
	assert.True(t, wf.CloneOpts.Skip, "Skip should be false")


     

       
95

       
40

       
 

       
}


     

       
96

       

       
-

       



     

       
97

       

       
-

       
func TestUnmarshalEnv(t *testing.T) {


     

       
98

       

       
-

       
	yamlData := `


     

       
99

       

       
-

       
when:


     

       
100

       

       
-

       
  - event: ["pull_request_close"]


     

       
101

       

       
-

       



     

       
102

       

       
-

       
clone:


     

       
103

       

       
-

       
  skip: false


     

       
104

       

       
-

       



     

       
105

       

       
-

       
environment:


     

       
106

       

       
-

       
  HOME: /home/foo bar/baz


     

       
107

       

       
-

       
  CGO_ENABLED: 1


     

       
108

       

       
-

       



     

       
109

       

       
-

       
steps:


     

       
110

       

       
-

       
  - name: Something


     

       
111

       

       
-

       
    command: echo "hello"


     

       
112

       

       
-

       
    environment:


     

       
113

       

       
-

       
      FOO: bar


     

       
114

       

       
-

       
      BAZ: qux


     

       
115

       

       
-

       
`


     

       
116

       

       
-

       



     

       
117

       

       
-

       
	wf, err := FromFile("test.yml", []byte(yamlData))


     

       
118

       

       
-

       
	assert.NoError(t, err)


     

       
119

       

       
-

       



     

       
120

       

       
-

       
	assert.Len(t, wf.Environment, 2)


     

       
121

       

       
-

       
	assert.Equal(t, "/home/foo bar/baz", wf.Environment["HOME"])


     

       
122

       

       
-

       
	assert.Equal(t, "1", wf.Environment["CGO_ENABLED"])


     

       
123

       

       
-

       
	assert.Equal(t, "bar", wf.Steps[0].Environment["FOO"])


     

       
124

       

       
-

       
	assert.Equal(t, "qux", wf.Steps[0].Environment["BAZ"])


     

       
125

       

       
-

       
}