From 2e4a6be8407d5991f31835be45559728bdceb7ab Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Wed, 13 Aug 2025 14:41:06 +0100 Subject: [PATCH] workflow: move workflow parsing into compiler Change-Id: vnxxyxursomypxxzplroozxmmnnrrrow this simplifies the error collection logic a lot. Signed-off-by: oppiliappan --- knotserver/ingester.go | 16 +++++--------- knotserver/internal.go | 48 ++++++++++++---------------------------- workflow/compile.go | 50 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 65 insertions(+), 49 deletions(-) diff --git a/knotserver/ingester.go b/knotserver/ingester.go index 1f1cc218..c58b6e0e 100644 --- a/knotserver/ingester.go +++ b/knotserver/ingester.go @@ -152,7 +152,7 @@ func (h *Handle) processPull(ctx context.Context, did string, record tangled.Rep return err } - var pipeline workflow.Pipeline + var pipeline workflow.RawPipeline for _, e := range workflowDir { if !e.IsFile { continue @@ -164,14 +164,10 @@ func (h *Handle) processPull(ctx context.Context, did string, record tangled.Rep continue } - wf, err := workflow.FromFile(e.Name, contents) - if err != nil { - // TODO: log here, respond to client that is pushing - h.l.Error("failed to parse workflow", "err", err, "path", fpath) - continue - } - - pipeline = append(pipeline, wf) + pipeline = append(pipeline, workflow.RawWorkflow{ + Name: e.Name, + Contents: contents, + }) } trigger := tangled.Pipeline_PullRequestTriggerData{ @@ -193,7 +189,7 @@ func (h *Handle) processPull(ctx context.Context, did string, record tangled.Rep }, } - cp := compiler.Compile(pipeline) + cp := compiler.Compile(compiler.Parse(pipeline)) eventJson, err := json.Marshal(cp) if err != nil { return err diff --git a/knotserver/internal.go b/knotserver/internal.go index 1b5a67bf..21473bf7 100644 --- a/knotserver/internal.go +++ b/knotserver/internal.go @@ -200,9 +200,7 @@ func (h *InternalHandle) triggerPipeline(clientMsgs *[]string, line git.PostRece return err } - pipelineParseErrors := []string{} - - var pipeline workflow.Pipeline + var pipeline workflow.RawPipeline for _, e := range workflowDir { if !e.IsFile { continue @@ -214,14 +212,10 @@ func (h *InternalHandle) triggerPipeline(clientMsgs *[]string, line git.PostRece continue } - wf, err := workflow.FromFile(e.Name, contents) - if err != nil { - h.l.Error("failed to parse workflow", "err", err, "path", fpath) - pipelineParseErrors = append(pipelineParseErrors, fmt.Sprintf("- at %s: %s\n", fpath, err)) - continue - } - - pipeline = append(pipeline, wf) + pipeline = append(pipeline, workflow.RawWorkflow{ + Name: e.Name, + Contents: contents, + }) } trigger := tangled.Pipeline_PushTriggerData{ @@ -242,37 +236,23 @@ func (h *InternalHandle) triggerPipeline(clientMsgs *[]string, line git.PostRece }, } - cp := compiler.Compile(pipeline) + cp := compiler.Compile(compiler.Parse(pipeline)) eventJson, err := json.Marshal(cp) if err != nil { return err } if pushOptions.verboseCi { - hasDiagnostics := false - if len(pipelineParseErrors) > 0 { - hasDiagnostics = true - *clientMsgs = append(*clientMsgs, "error: failed to parse workflow(s):") - for _, error := range pipelineParseErrors { - *clientMsgs = append(*clientMsgs, error) - } - } - if len(compiler.Diagnostics.Errors) > 0 { - hasDiagnostics = true - *clientMsgs = append(*clientMsgs, "error(s) on pipeline:") - for _, error := range compiler.Diagnostics.Errors { - *clientMsgs = append(*clientMsgs, fmt.Sprintf("- %s:", error)) - } + if compiler.Diagnostics.IsEmpty() { + *clientMsgs = append(*clientMsgs, "success: pipeline compiled with no diagnostics") } - if len(compiler.Diagnostics.Warnings) > 0 { - hasDiagnostics = true - *clientMsgs = append(*clientMsgs, "warning(s) on pipeline:") - for _, warning := range compiler.Diagnostics.Warnings { - *clientMsgs = append(*clientMsgs, fmt.Sprintf("- at %s: %s: %s", warning.Path, warning.Type, warning.Reason)) - } + + for _, e := range compiler.Diagnostics.Errors { + *clientMsgs = append(*clientMsgs, e.String()) } - if !hasDiagnostics { - *clientMsgs = append(*clientMsgs, "success: pipeline compiled with no diagnostics") + + for _, w := range compiler.Diagnostics.Warnings { + *clientMsgs = append(*clientMsgs, w.String()) } } diff --git a/workflow/compile.go b/workflow/compile.go index 057ee59d..e45dede1 100644 --- a/workflow/compile.go +++ b/workflow/compile.go @@ -6,16 +6,27 @@ import ( "tangled.sh/tangled.sh/core/api/tangled" ) +type RawWorkflow struct { + Name string + Contents []byte +} + +type RawPipeline = []RawWorkflow + type Compiler struct { Trigger tangled.Pipeline_TriggerMetadata Diagnostics Diagnostics } type Diagnostics struct { - Errors []error + Errors []Error Warnings []Warning } +func (d *Diagnostics) IsEmpty() bool { + return len(d.Errors) == 0 && len(d.Warnings) == 0 +} + func (d *Diagnostics) Combine(o Diagnostics) { d.Errors = append(d.Errors, o.Errors...) d.Warnings = append(d.Warnings, o.Warnings...) @@ -25,20 +36,33 @@ func (d *Diagnostics) AddWarning(path string, kind WarningKind, reason string) { d.Warnings = append(d.Warnings, Warning{path, kind, reason}) } -func (d *Diagnostics) AddError(err error) { - d.Errors = append(d.Errors, err) +func (d *Diagnostics) AddError(path string, err error) { + d.Errors = append(d.Errors, Error{path, err}) } func (d Diagnostics) IsErr() bool { return len(d.Errors) != 0 } +type Error struct { + Path string + Error error +} + +func (e Error) String() string { + return fmt.Sprintf("error: %s: %s", e.Path, e.Error.Error()) +} + type Warning struct { Path string Type WarningKind Reason string } +func (w Warning) String() string { + return fmt.Sprintf("warning: %s: %s: %s", w.Path, w.Type, w.Reason) +} + type WarningKind string var ( @@ -46,14 +70,30 @@ var ( InvalidConfiguration WarningKind = "invalid configuration" ) +func (compiler *Compiler) Parse(p RawPipeline) Pipeline { + var pp Pipeline + + for _, w := range p { + wf, err := FromFile(w.Name, w.Contents) + if err != nil { + compiler.Diagnostics.AddError(w.Name, err) + continue + } + + pp = append(pp, wf) + } + + return pp +} + // convert a repositories' workflow files into a fully compiled pipeline that runners accept func (compiler *Compiler) Compile(p Pipeline) tangled.Pipeline { cp := tangled.Pipeline{ TriggerMetadata: &compiler.Trigger, } - for _, w := range p { - cw := compiler.compileWorkflow(w) + for _, wf := range p { + cw := compiler.compileWorkflow(wf) // empty workflows are not added to the pipeline if len(cw.Steps) == 0 { -- 2.43.0