back round 1 view raw

spindle: emit control log lines for start and end of steps #730

merged

opened by

oppi.li 1 month ago targeting master from push-lstutzylzylk

Signed-off-by: oppiliappan me@oppi.li

options

unified split

Changed files

+52 -23

spindle

engine

engine.go

engines

nixery

engine.go

models

logger.go

models.go

+13 -3

spindle/engine/engine.go

···

       79
       79
        
       				defer cancel()

     

       80
       80
        
       

     

       81
       81
        
       				for stepIdx, step := range w.Steps {

     

       82
       82
       +
       					// log start of step

     

       82
       83
        
       					if wfLogger != nil {

     

       83
       83
       -
       						ctl := wfLogger.ControlWriter(stepIdx, step)

     

       84
       84
       -
       						ctl.Write([]byte(step.Name()))

     

       84
       84
       +
       						wfLogger.

     

       85
       85
       +
       							ControlWriter(stepIdx, step, models.StepStatusStart).

     

       86
       86
       +
       							Write([]byte{0})

     

       85
       87
        
       					}

     

       86
       88
        
       

     

       87
       89
        
       					err = eng.RunStep(ctx, wid, &w, stepIdx, allSecrets, wfLogger)

     

       90
       90
       +
       

     

       91
       91
       +
       					// log end of step

     

       92
       92
       +
       					if wfLogger != nil {

     

       93
       93
       +
       						wfLogger.

     

       94
       94
       +
       							ControlWriter(stepIdx, step, models.StepStatusEnd).

     

       95
       95
       +
       							Write([]byte{0})

     

       96
       96
       +
       					}

     

       97
       97
       +
       

     

       88
       98
        
       					if err != nil {

     

       89
       99
        
       						if errors.Is(err, ErrTimedOut) {

     

       90
       100
        
       							dbErr := db.StatusTimeout(wid, n)

     
···

       115
       125
        
       	if err := eg.Wait(); err != nil {

     

       116
       126
        
       		l.Error("failed to run one or more workflows", "err", err)

     

       117
       127
        
       	} else {

     

       118
       118
       -
       		l.Error("successfully ran full pipeline")

     

       128
       128
       +
       		l.Info("successfully ran full pipeline")

     

       119
       129
        
       	}

     

       120
       130
        
       }

+2 -2

spindle/engines/nixery/engine.go

···

       381
       381
        
       	defer logs.Close()

     

       382
       382
        
       

     

       383
       383
        
       	_, err = stdcopy.StdCopy(

     

       384
       384
       -
       		wfLogger.DataWriter("stdout"),

     

       385
       385
       -
       		wfLogger.DataWriter("stderr"),

     

       384
       384
       +
       		wfLogger.DataWriter(stepIdx, "stdout"),

     

       385
       385
       +
       		wfLogger.DataWriter(stepIdx, "stderr"),

     

       386
       386
        
       		logs.Reader,

     

       387
       387
        
       	)

     

       388
       388
        
       	if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) {

+14 -10

spindle/models/logger.go

···

       37
       37
        
       	return l.file.Close()

     

       38
       38
        
       }

     

       39
       39
        
       

     

       40
       40
       -
       func (l *WorkflowLogger) DataWriter(stream string) io.Writer {

     

       40
       40
       +
       func (l *WorkflowLogger) DataWriter(idx int, stream string) io.Writer {

     

       41
       41
        
       	return &dataWriter{

     

       42
       42
        
       		logger: l,

     

       43
       43
       +
       		idx:    idx,

     

       43
       44
        
       		stream: stream,

     

       44
       45
        
       	}

     

       45
       46
        
       }

     

       46
       47
        
       

     

       47
       47
       -
       func (l *WorkflowLogger) ControlWriter(idx int, step Step) io.Writer {

     

       48
       48
       +
       func (l *WorkflowLogger) ControlWriter(idx int, step Step, stepStatus StepStatus) io.Writer {

     

       48
       49
        
       	return &controlWriter{

     

       49
       49
       -
       		logger: l,

     

       50
       50
       -
       		idx:    idx,

     

       51
       51
       -
       		step:   step,

     

       50
       50
       +
       		logger:     l,

     

       51
       51
       +
       		idx:        idx,

     

       52
       52
       +
       		step:       step,

     

       53
       53
       +
       		stepStatus: stepStatus,

     

       52
       54
        
       	}

     

       53
       55
        
       }

     

       54
       56
        
       

     

       55
       57
        
       type dataWriter struct {

     

       56
       58
        
       	logger *WorkflowLogger

     

       59
       59
       +
       	idx    int

     

       57
       60
        
       	stream string

     

       58
       61
        
       }

     

       59
       62
        
       

     

       60
       63
        
       func (w *dataWriter) Write(p []byte) (int, error) {

     

       61
       64
        
       	line := strings.TrimRight(string(p), "\r\n")

     

       62
       62
       -
       	entry := NewDataLogLine(line, w.stream)

     

       65
       65
       +
       	entry := NewDataLogLine(w.idx, line, w.stream)

     

       63
       66
        
       	if err := w.logger.encoder.Encode(entry); err != nil {

     

       64
       67
        
       		return 0, err

     

       65
       68
        
       	}

     
···

       67
       70
        
       }

     

       68
       71
        
       

     

       69
       72
        
       type controlWriter struct {

     

       70
       70
       -
       	logger *WorkflowLogger

     

       71
       71
       -
       	idx    int

     

       72
       72
       -
       	step   Step

     

       73
       73
       +
       	logger     *WorkflowLogger

     

       74
       74
       +
       	idx        int

     

       75
       75
       +
       	step       Step

     

       76
       76
       +
       	stepStatus StepStatus

     

       73
       77
        
       }

     

       74
       78
        
       

     

       75
       79
        
       func (w *controlWriter) Write(_ []byte) (int, error) {

     

       76
       76
       -
       	entry := NewControlLogLine(w.idx, w.step)

     

       80
       80
       +
       	entry := NewControlLogLine(w.idx, w.step, w.stepStatus)

     

       77
       81
        
       	if err := w.logger.encoder.Encode(entry); err != nil {

     

       78
       82
        
       		return 0, err

     

       79
       83
        
       	}

+23 -8

spindle/models/models.go

···

       4
       4
        
       	"fmt"

     

       5
       5
        
       	"regexp"

     

       6
       6
        
       	"slices"

     

       7
       7
       +
       	"time"

     

       7
       8
        
       

     

       8
       9
        
       	"tangled.org/core/api/tangled"

     

       9
       10
        
       

     
···

       76
       77
        
       var (

     

       77
       78
        
       	// step log data

     

       78
       79
        
       	LogKindData LogKind = "data"

     

       79
       79
       -
       	// indicates start/end of a step

     

       80
       80
       +
       	// indicates status of a step

     

       80
       81
        
       	LogKindControl LogKind = "control"

     

       81
       82
        
       )

     

       82
       83
        
       

     

       84
       84
       +
       // step status indicator in control log lines

     

       85
       85
       +
       type StepStatus string

     

       86
       86
       +
       

     

       87
       87
       +
       var (

     

       88
       88
       +
       	StepStatusStart StepStatus = "start"

     

       89
       89
       +
       	StepStatusEnd   StepStatus = "end"

     

       90
       90
       +
       )

     

       91
       91
       +
       

     

       83
       92
        
       type LogLine struct {

     

       84
       84
       -
       	Kind    LogKind `json:"kind"`

     

       85
       85
       -
       	Content string  `json:"content"`

     

       93
       93
       +
       	Kind    LogKind   `json:"kind"`

     

       94
       94
       +
       	Content string    `json:"content"`

     

       95
       95
       +
       	Time    time.Time `json:"time"`

     

       96
       96
       +
       	StepId  int       `json:"step_id"`

     

       86
       97
        
       

     

       87
       98
        
       	// fields if kind is "data"

     

       88
       99
        
       	Stream string `json:"stream,omitempty"`

     

       89
       100
        
       

     

       90
       101
        
       	// fields if kind is "control"

     

       91
       91
       -
       	StepId      int      `json:"step_id,omitempty"`

     

       92
       92
       -
       	StepKind    StepKind `json:"step_kind,omitempty"`

     

       93
       93
       -
       	StepCommand string   `json:"step_command,omitempty"`

     

       102
       102
       +
       	StepStatus  StepStatus `json:"step_status,omitempty"`

     

       103
       103
       +
       	StepKind    StepKind   `json:"step_kind,omitempty"`

     

       104
       104
       +
       	StepCommand string     `json:"step_command,omitempty"`

     

       94
       105
        
       }

     

       95
       106
        
       

     

       96
       96
       -
       func NewDataLogLine(content, stream string) LogLine {

     

       107
       107
       +
       func NewDataLogLine(idx int, content, stream string) LogLine {

     

       97
       108
        
       	return LogLine{

     

       98
       109
        
       		Kind:    LogKindData,

     

       110
       110
       +
       		Time:    time.Now(),

     

       99
       111
        
       		Content: content,

     

       112
       112
       +
       		StepId:  idx,

     

       100
       113
        
       		Stream:  stream,

     

       101
       114
        
       	}

     

       102
       115
        
       }

     

       103
       116
        
       

     

       104
       104
       -
       func NewControlLogLine(idx int, step Step) LogLine {

     

       117
       117
       +
       func NewControlLogLine(idx int, step Step, status StepStatus) LogLine {

     

       105
       118
        
       	return LogLine{

     

       106
       119
        
       		Kind:        LogKindControl,

     

       120
       120
       +
       		Time:        time.Now(),

     

       107
       121
        
       		Content:     step.Name(),

     

       108
       122
        
       		StepId:      idx,

     

       123
       123
       +
       		StepStatus:  status,

     

       109
       124
        
       		StepKind:    step.Kind(),

     

       110
       125
        
       		StepCommand: step.Command(),

     

       111
       126
        
       	}