back round 0 view raw

workflow: move workflow parsing into compiler #487

merged

opened by

oppi.li 4 months ago targeting master from push-vnxxyxursomy

this simplifies the error collection logic a lot.

Signed-off-by: oppiliappan me@oppi.li

options

unified split

Changed files

+65 -49

knotserver

ingester.go

internal.go

workflow

compile.go

+6 -10

knotserver/ingester.go

···

       152
       152
        
       		return err

     

       153
       153
        
       	}

     

       154
       154
        
       

     

       155
       155
       -
       	var pipeline workflow.Pipeline

     

       155
       155
       +
       	var pipeline workflow.RawPipeline

     

       156
       156
        
       	for _, e := range workflowDir {

     

       157
       157
        
       		if !e.IsFile {

     

       158
       158
        
       			continue

     
···

       164
       164
        
       			continue

     

       165
       165
        
       		}

     

       166
       166
        
       

     

       167
       167
       -
       		wf, err := workflow.FromFile(e.Name, contents)

     

       168
       168
       -
       		if err != nil {

     

       169
       169
       -
       			// TODO: log here, respond to client that is pushing

     

       170
       170
       -
       			h.l.Error("failed to parse workflow", "err", err, "path", fpath)

     

       171
       171
       -
       			continue

     

       172
       172
       -
       		}

     

       173
       173
       -
       

     

       174
       174
       -
       		pipeline = append(pipeline, wf)

     

       167
       167
       +
       		pipeline = append(pipeline, workflow.RawWorkflow{

     

       168
       168
       +
       			Name:     e.Name,

     

       169
       169
       +
       			Contents: contents,

     

       170
       170
       +
       		})

     

       175
       171
        
       	}

     

       176
       172
        
       

     

       177
       173
        
       	trigger := tangled.Pipeline_PullRequestTriggerData{

     
···

       193
       189
        
       		},

     

       194
       190
        
       	}

     

       195
       191
        
       

     

       196
       196
       -
       	cp := compiler.Compile(pipeline)

     

       192
       192
       +
       	cp := compiler.Compile(compiler.Parse(pipeline))

     

       197
       193
        
       	eventJson, err := json.Marshal(cp)

     

       198
       194
        
       	if err != nil {

     

       199
       195
        
       		return err

+14 -34

knotserver/internal.go

···

       200
       200
        
       		return err

     

       201
       201
        
       	}

     

       202
       202
        
       

     

       203
       203
       -
       	pipelineParseErrors := []string{}

     

       204
       204
       -
       

     

       205
       205
       -
       	var pipeline workflow.Pipeline

     

       203
       203
       +
       	var pipeline workflow.RawPipeline

     

       206
       204
        
       	for _, e := range workflowDir {

     

       207
       205
        
       		if !e.IsFile {

     

       208
       206
        
       			continue

     
···

       214
       212
        
       			continue

     

       215
       213
        
       		}

     

       216
       214
        
       

     

       217
       217
       -
       		wf, err := workflow.FromFile(e.Name, contents)

     

       218
       218
       -
       		if err != nil {

     

       219
       219
       -
       			h.l.Error("failed to parse workflow", "err", err, "path", fpath)

     

       220
       220
       -
       			pipelineParseErrors = append(pipelineParseErrors, fmt.Sprintf("- at %s: %s\n", fpath, err))

     

       221
       221
       -
       			continue

     

       222
       222
       -
       		}

     

       223
       223
       -
       

     

       224
       224
       -
       		pipeline = append(pipeline, wf)

     

       215
       215
       +
       		pipeline = append(pipeline, workflow.RawWorkflow{

     

       216
       216
       +
       			Name:     e.Name,

     

       217
       217
       +
       			Contents: contents,

     

       218
       218
       +
       		})

     

       225
       219
        
       	}

     

       226
       220
        
       

     

       227
       221
        
       	trigger := tangled.Pipeline_PushTriggerData{

     
···

       242
       236
        
       		},

     

       243
       237
        
       	}

     

       244
       238
        
       

     

       245
       245
       -
       	cp := compiler.Compile(pipeline)

     

       239
       239
       +
       	cp := compiler.Compile(compiler.Parse(pipeline))

     

       246
       240
        
       	eventJson, err := json.Marshal(cp)

     

       247
       241
        
       	if err != nil {

     

       248
       242
        
       		return err

     

       249
       243
        
       	}

     

       250
       244
        
       

     

       251
       245
        
       	if pushOptions.verboseCi {

     

       252
       252
       -
       		hasDiagnostics := false

     

       253
       253
       -
       		if len(pipelineParseErrors) > 0 {

     

       254
       254
       -
       			hasDiagnostics = true

     

       255
       255
       -
       			*clientMsgs = append(*clientMsgs, "error: failed to parse workflow(s):")

     

       256
       256
       -
       			for _, error := range pipelineParseErrors {

     

       257
       257
       -
       				*clientMsgs = append(*clientMsgs, error)

     

       258
       258
       -
       			}

     

       259
       259
       -
       		}

     

       260
       260
       -
       		if len(compiler.Diagnostics.Errors) > 0 {

     

       261
       261
       -
       			hasDiagnostics = true

     

       262
       262
       -
       			*clientMsgs = append(*clientMsgs, "error(s) on pipeline:")

     

       263
       263
       -
       			for _, error := range compiler.Diagnostics.Errors {

     

       264
       264
       -
       				*clientMsgs = append(*clientMsgs, fmt.Sprintf("- %s:", error))

     

       265
       265
       -
       			}

     

       246
       246
       +
       		if compiler.Diagnostics.IsEmpty() {

     

       247
       247
       +
       			*clientMsgs = append(*clientMsgs, "success: pipeline compiled with no diagnostics")

     

       266
       248
        
       		}

     

       267
       267
       -
       		if len(compiler.Diagnostics.Warnings) > 0 {

     

       268
       268
       -
       			hasDiagnostics = true

     

       269
       269
       -
       			*clientMsgs = append(*clientMsgs, "warning(s) on pipeline:")

     

       270
       270
       -
       			for _, warning := range compiler.Diagnostics.Warnings {

     

       271
       271
       -
       				*clientMsgs = append(*clientMsgs, fmt.Sprintf("- at %s: %s: %s", warning.Path, warning.Type, warning.Reason))

     

       272
       272
       -
       			}

     

       249
       249
       +
       

     

       250
       250
       +
       		for _, e := range compiler.Diagnostics.Errors {

     

       251
       251
       +
       			*clientMsgs = append(*clientMsgs, e.String())

     

       273
       252
        
       		}

     

       274
       274
       -
       		if !hasDiagnostics {

     

       275
       275
       -
       			*clientMsgs = append(*clientMsgs, "success: pipeline compiled with no diagnostics")

     

       253
       253
       +
       

     

       254
       254
       +
       		for _, w := range compiler.Diagnostics.Warnings {

     

       255
       255
       +
       			*clientMsgs = append(*clientMsgs, w.String())

     

       276
       256
        
       		}

     

       277
       257
        
       	}

     

       278
       258

+45 -5

workflow/compile.go

···

       6
       6
        
       	"tangled.sh/tangled.sh/core/api/tangled"

     

       7
       7
        
       )

     

       8
       8
        
       

     

       9
       9
       +
       type RawWorkflow struct {

     

       10
       10
       +
       	Name     string

     

       11
       11
       +
       	Contents []byte

     

       12
       12
       +
       }

     

       13
       13
       +
       

     

       14
       14
       +
       type RawPipeline = []RawWorkflow

     

       15
       15
       +
       

     

       9
       16
        
       type Compiler struct {

     

       10
       17
        
       	Trigger     tangled.Pipeline_TriggerMetadata

     

       11
       18
        
       	Diagnostics Diagnostics

     

       12
       19
        
       }

     

       13
       20
        
       

     

       14
       21
        
       type Diagnostics struct {

     

       15
       15
       -
       	Errors   []error

     

       22
       22
       +
       	Errors   []Error

     

       16
       23
        
       	Warnings []Warning

     

       17
       24
        
       }

     

       18
       25
        
       

     

       26
       26
       +
       func (d *Diagnostics) IsEmpty() bool {

     

       27
       27
       +
       	return len(d.Errors) == 0 && len(d.Warnings) == 0

     

       28
       28
       +
       }

     

       29
       29
       +
       

     

       19
       30
        
       func (d *Diagnostics) Combine(o Diagnostics) {

     

       20
       31
        
       	d.Errors = append(d.Errors, o.Errors...)

     

       21
       32
        
       	d.Warnings = append(d.Warnings, o.Warnings...)

     
···

       25
       36
        
       	d.Warnings = append(d.Warnings, Warning{path, kind, reason})

     

       26
       37
        
       }

     

       27
       38
        
       

     

       28
       28
       -
       func (d *Diagnostics) AddError(err error) {

     

       29
       29
       -
       	d.Errors = append(d.Errors, err)

     

       39
       39
       +
       func (d *Diagnostics) AddError(path string, err error) {

     

       40
       40
       +
       	d.Errors = append(d.Errors, Error{path, err})

     

       30
       41
        
       }

     

       31
       42
        
       

     

       32
       43
        
       func (d Diagnostics) IsErr() bool {

     

       33
       44
        
       	return len(d.Errors) != 0

     

       34
       45
        
       }

     

       35
       46
        
       

     

       47
       47
       +
       type Error struct {

     

       48
       48
       +
       	Path  string

     

       49
       49
       +
       	Error error

     

       50
       50
       +
       }

     

       51
       51
       +
       

     

       52
       52
       +
       func (e Error) String() string {

     

       53
       53
       +
       	return fmt.Sprintf("error: %s: %s", e.Path, e.Error.Error())

     

       54
       54
       +
       }

     

       55
       55
       +
       

     

       36
       56
        
       type Warning struct {

     

       37
       57
        
       	Path   string

     

       38
       58
        
       	Type   WarningKind

     

       39
       59
        
       	Reason string

     

       40
       60
        
       }

     

       41
       61
        
       

     

       62
       62
       +
       func (w Warning) String() string {

     

       63
       63
       +
       	return fmt.Sprintf("warning: %s: %s: %s", w.Path, w.Type, w.Reason)

     

       64
       64
       +
       }

     

       65
       65
       +
       

     

       42
       66
        
       type WarningKind string

     

       43
       67
        
       

     

       44
       68
        
       var (

     
···

       46
       70
        
       	InvalidConfiguration WarningKind = "invalid configuration"

     

       47
       71
        
       )

     

       48
       72
        
       

     

       73
       73
       +
       func (compiler *Compiler) Parse(p RawPipeline) Pipeline {

     

       74
       74
       +
       	var pp Pipeline

     

       75
       75
       +
       

     

       76
       76
       +
       	for _, w := range p {

     

       77
       77
       +
       		wf, err := FromFile(w.Name, w.Contents)

     

       78
       78
       +
       		if err != nil {

     

       79
       79
       +
       			compiler.Diagnostics.AddError(w.Name, err)

     

       80
       80
       +
       			continue

     

       81
       81
       +
       		}

     

       82
       82
       +
       

     

       83
       83
       +
       		pp = append(pp, wf)

     

       84
       84
       +
       	}

     

       85
       85
       +
       

     

       86
       86
       +
       	return pp

     

       87
       87
       +
       }

     

       88
       88
       +
       

     

       49
       89
        
       // convert a repositories' workflow files into a fully compiled pipeline that runners accept

     

       50
       90
        
       func (compiler *Compiler) Compile(p Pipeline) tangled.Pipeline {

     

       51
       91
        
       	cp := tangled.Pipeline{

     

       52
       92
        
       		TriggerMetadata: &compiler.Trigger,

     

       53
       93
        
       	}

     

       54
       94
        
       

     

       55
       55
       -
       	for _, w := range p {

     

       56
       56
       -
       		cw := compiler.compileWorkflow(w)

     

       95
       95
       +
       	for _, wf := range p {

     

       96
       96
       +
       		cw := compiler.compileWorkflow(wf)

     

       57
       97
        
       

     

       58
       98
        
       		// empty workflows are not added to the pipeline

     

       59
       99
        
       		if len(cw.Steps) == 0 {