commit 7f7d89ac561f40993247a710c8ce3b5892df4cc0 · veryroundbird.house/core

+98 -5

spindle/engine/engine.go

···

       1
       1
        
       package engine

     

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
       +
       	"bufio"

     

       4
       5
        
       	"context"

     

       5
       6
        
       	"fmt"

     

       6
       7
        
       	"io"

     
···

       32
       33
        
       	l      *slog.Logger

     

       33
       34
        
       	db     *db.DB

     

       34
       35
        
       	n      *notifier.Notifier

     

       36
       36
       +
       

     

       37
       37
       +
       	chanMu      sync.RWMutex

     

       38
       38
       +
       	stdoutChans map[string]chan string

     

       39
       39
       +
       	stderrChans map[string]chan string

     

       35
       40
        
       }

     

       36
       41
        
       

     

       37
       42
        
       func New(ctx context.Context, db *db.DB, n *notifier.Notifier) (*Engine, error) {

     
···

       42
       47
        
       

     

       43
       48
        
       	l := log.FromContext(ctx).With("component", "spindle")

     

       44
       49
        
       

     

       45
       45
       -
       	return &Engine{docker: dcli, l: l, db: db, n: n}, nil

     

       50
       50
       +
       	e := &Engine{

     

       51
       51
       +
       		docker: dcli,

     

       52
       52
       +
       		l:      l,

     

       53
       53
       +
       		db:     db,

     

       54
       54
       +
       		n:      n,

     

       55
       55
       +
       	}

     

       56
       56
       +
       

     

       57
       57
       +
       	e.stdoutChans = make(map[string]chan string, 100)

     

       58
       58
       +
       	e.stderrChans = make(map[string]chan string, 100)

     

       59
       59
       +
       

     

       60
       60
       +
       	return e, nil

     

       46
       61
        
       }

     

       47
       62
        
       

     

       48
       63
        
       // SetupPipeline sets up a new network for the pipeline, and possibly volumes etc.

     
···

       136
       151
        
       // ONLY marks pipeline as failed if container's exit code is non-zero.

     

       137
       152
        
       // All other errors are bubbled up.

     

       138
       153
        
       func (e *Engine) StartSteps(ctx context.Context, steps []*tangled.Pipeline_Step, id, image string) error {

     

       154
       154
       +
       	// set up logging channels

     

       155
       155
       +
       	e.chanMu.Lock()

     

       156
       156
       +
       	if _, exists := e.stdoutChans[id]; !exists {

     

       157
       157
       +
       		e.stdoutChans[id] = make(chan string, 100)

     

       158
       158
       +
       	}

     

       159
       159
       +
       	if _, exists := e.stderrChans[id]; !exists {

     

       160
       160
       +
       		e.stderrChans[id] = make(chan string, 100)

     

       161
       161
       +
       	}

     

       162
       162
       +
       	e.chanMu.Unlock()

     

       163
       163
       +
       

     

       164
       164
       +
       	// close channels after all steps are complete

     

       165
       165
       +
       	defer func() {

     

       166
       166
       +
       		close(e.stdoutChans[id])

     

       167
       167
       +
       		close(e.stderrChans[id])

     

       168
       168
       +
       	}()

     

       169
       169
       +
       

     

       139
       170
        
       	for _, step := range steps {

     

       140
       171
        
       		hostConfig := hostConfig(id)

     

       141
       172
        
       		resp, err := e.docker.ContainerCreate(ctx, &container.Config{

     
···

       166
       197
        
       		wg.Add(1)

     

       167
       198
        
       		go func() {

     

       168
       199
        
       			defer wg.Done()

     

       169
       169
       -
       			err := e.TailStep(ctx, resp.ID)

     

       200
       200
       +
       			err := e.TailStep(ctx, resp.ID, id)

     

       170
       201
        
       			if err != nil {

     

       171
       202
        
       				e.l.Error("failed to tail container", "container", resp.ID)

     

       172
       203
        
       				return

     
···

       211
       242
        
       	return info.State, nil

     

       212
       243
        
       }

     

       213
       244
        
       

     

       214
       214
       -
       func (e *Engine) TailStep(ctx context.Context, containerID string) error {

     

       245
       245
       +
       func (e *Engine) TailStep(ctx context.Context, containerID, pipelineID string) error {

     

       215
       246
        
       	logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{

     

       216
       247
        
       		Follow:     true,

     

       217
       248
        
       		ShowStdout: true,

     
···

       223
       254
        
       		return err

     

       224
       255
        
       	}

     

       225
       256
        
       

     

       257
       257
       +
       	// using StdCopy we demux logs and stream stdout and stderr to different

     

       258
       258
       +
       	// channels.

     

       259
       259
       +
       	//

     

       260
       260
       +
       	//    stdout w||r stdoutCh

     

       261
       261
       +
       	//    stderr w||r stderrCh

     

       262
       262
       +
       	//

     

       263
       263
       +
       

     

       264
       264
       +
       	rpipeOut, wpipeOut := io.Pipe()

     

       265
       265
       +
       	rpipeErr, wpipeErr := io.Pipe()

     

       266
       266
       +
       

     

       226
       267
        
       	go func() {

     

       227
       227
       -
       		_, _ = stdcopy.StdCopy(os.Stdout, os.Stdout, logs)

     

       228
       228
       -
       		_ = logs.Close()

     

       268
       268
       +
       		defer wpipeOut.Close()

     

       269
       269
       +
       		defer wpipeErr.Close()

     

       270
       270
       +
       		_, err := stdcopy.StdCopy(wpipeOut, wpipeErr, logs)

     

       271
       271
       +
       		if err != nil && err != io.EOF {

     

       272
       272
       +
       			e.l.Error("failed to copy logs", "error", err)

     

       273
       273
       +
       		}

     

       274
       274
       +
       	}()

     

       275
       275
       +
       

     

       276
       276
       +
       	// read from stdout and send to stdout pipe

     

       277
       277
       +
       	// NOTE: the stdoutCh channnel is closed further up in StartSteps

     

       278
       278
       +
       	// once all steps are done.

     

       279
       279
       +
       	go func() {

     

       280
       280
       +
       		e.chanMu.RLock()

     

       281
       281
       +
       		stdoutCh := e.stdoutChans[pipelineID]

     

       282
       282
       +
       		e.chanMu.RUnlock()

     

       283
       283
       +
       

     

       284
       284
       +
       		scanner := bufio.NewScanner(rpipeOut)

     

       285
       285
       +
       		for scanner.Scan() {

     

       286
       286
       +
       			stdoutCh <- scanner.Text()

     

       287
       287
       +
       		}

     

       288
       288
       +
       		if err := scanner.Err(); err != nil {

     

       289
       289
       +
       			e.l.Error("failed to scan stdout", "error", err)

     

       290
       290
       +
       		}

     

       291
       291
       +
       	}()

     

       292
       292
       +
       

     

       293
       293
       +
       	// read from stderr and send to stderr pipe

     

       294
       294
       +
       	// NOTE: the stderrCh channnel is closed further up in StartSteps

     

       295
       295
       +
       	// once all steps are done.

     

       296
       296
       +
       	go func() {

     

       297
       297
       +
       		e.chanMu.RLock()

     

       298
       298
       +
       		stderrCh := e.stderrChans[pipelineID]

     

       299
       299
       +
       		e.chanMu.RUnlock()

     

       300
       300
       +
       

     

       301
       301
       +
       		scanner := bufio.NewScanner(rpipeErr)

     

       302
       302
       +
       		for scanner.Scan() {

     

       303
       303
       +
       			stderrCh <- scanner.Text()

     

       304
       304
       +
       		}

     

       305
       305
       +
       		if err := scanner.Err(); err != nil {

     

       306
       306
       +
       			e.l.Error("failed to scan stderr", "error", err)

     

       307
       307
       +
       		}

     

       229
       308
        
       	}()

     

       309
       309
       +
       

     

       230
       310
        
       	return nil

     

       311
       311
       +
       }

     

       312
       312
       +
       

     

       313
       313
       +
       func (e *Engine) LogChannels(pipelineID string) (stdout <-chan string, stderr <-chan string, ok bool) {

     

       314
       314
       +
       	e.chanMu.RLock()

     

       315
       315
       +
       	defer e.chanMu.RUnlock()

     

       316
       316
       +
       

     

       317
       317
       +
       	stdoutCh, ok1 := e.stdoutChans[pipelineID]

     

       318
       318
       +
       	stderrCh, ok2 := e.stderrChans[pipelineID]

     

       319
       319
       +
       

     

       320
       320
       +
       	if !ok1 || !ok2 {

     

       321
       321
       +
       		return nil, nil, false

     

       322
       322
       +
       	}

     

       323
       323
       +
       	return stdoutCh, stderrCh, true

     

       231
       324
        
       }

     

       232
       325
        
       

     

       233
       326
        
       func workspaceVolume(id string) string {

+3 -2

spindle/server.go

···

       6
       6
        
       	"log/slog"

     

       7
       7
        
       	"net/http"

     

       8
       8
        
       

     

       9
       9
       +
       	"github.com/go-chi/chi/v5"

     

       9
       10
        
       	"golang.org/x/net/context"

     

       10
       11
        
       	"tangled.sh/tangled.sh/core/api/tangled"

     

       11
       12
        
       	"tangled.sh/tangled.sh/core/jetstream"

     
···

       52
       53
        
       	}

     

       53
       54
        
       

     

       54
       55
        
       	n := notifier.New()

     

       55
       55
       -
       

     

       56
       56
        
       	eng, err := engine.New(ctx, d, &n)

     

       57
       57
        
       	if err != nil {

     

       58
       58
        
       		return err

     
···

       89
       89
        
       }

     

       90
       90
        
       

     

       91
       91
        
       func (s *Spindle) Router() http.Handler {

     

       92
       92
       -
       	mux := &http.ServeMux{}

     

       92
       92
       +
       	mux := chi.NewRouter()

     

       93
       93
        
       

     

       94
       94
        
       	mux.HandleFunc("/events", s.Events)

     

       95
       95
       +
       	mux.HandleFunc("/logs/{pipelineID}", s.Logs)

     

       95
       96
        
       	return mux

     

       96
       97
        
       }

     

       97
       98

+100

spindle/stream.go

···

       1
       1
        
       package spindle

     

       2
       2
        
       

     

       3
       3
        
       import (

     

       4
       4
       +
       	"fmt"

     

       4
       5
        
       	"net/http"

     

       5
       6
        
       	"time"

     

       6
       7
        
       

     

       7
       8
        
       	"context"

     

       8
       9
        
       

     

       10
       10
       +
       	"github.com/go-chi/chi/v5"

     

       9
       11
        
       	"github.com/gorilla/websocket"

     

       10
       12
        
       )

     

       11
       13
        
       

     
···

       72
       74
        
       			}

     

       73
       75
        
       		}

     

       74
       76
        
       	}

     

       77
       77
       +
       }

     

       78
       78
       +
       

     

       79
       79
       +
       func (s *Spindle) Logs(w http.ResponseWriter, r *http.Request) {

     

       80
       80
       +
       	l := s.l.With("handler", "Logs")

     

       81
       81
       +
       

     

       82
       82
       +
       	pipelineID := chi.URLParam(r, "pipelineID")

     

       83
       83
       +
       	if pipelineID == "" {

     

       84
       84
       +
       		http.Error(w, "pipelineID required", http.StatusBadRequest)

     

       85
       85
       +
       		return

     

       86
       86
       +
       	}

     

       87
       87
       +
       	l = l.With("pipelineID", pipelineID)

     

       88
       88
       +
       

     

       89
       89
       +
       	conn, err := upgrader.Upgrade(w, r, nil)

     

       90
       90
       +
       	if err != nil {

     

       91
       91
       +
       		l.Error("websocket upgrade failed", "err", err)

     

       92
       92
       +
       		http.Error(w, "failed to upgrade", http.StatusInternalServerError)

     

       93
       93
       +
       		return

     

       94
       94
       +
       	}

     

       95
       95
       +
       	defer conn.Close()

     

       96
       96
       +
       	l.Info("upgraded http to wss")

     

       97
       97
       +
       

     

       98
       98
       +
       	ctx, cancel := context.WithCancel(r.Context())

     

       99
       99
       +
       	defer cancel()

     

       100
       100
       +
       

     

       101
       101
       +
       	go func() {

     

       102
       102
       +
       		for {

     

       103
       103
       +
       			if _, _, err := conn.NextReader(); err != nil {

     

       104
       104
       +
       				l.Info("client disconnected", "err", err)

     

       105
       105
       +
       				cancel()

     

       106
       106
       +
       				return

     

       107
       107
       +
       			}

     

       108
       108
       +
       		}

     

       109
       109
       +
       	}()

     

       110
       110
       +
       

     

       111
       111
       +
       	if err := s.streamLogs(ctx, conn, pipelineID); err != nil {

     

       112
       112
       +
       		l.Error("streamLogs failed", "err", err)

     

       113
       113
       +
       	}

     

       114
       114
       +
       	l.Info("logs connection closed")

     

       115
       115
       +
       }

     

       116
       116
       +
       

     

       117
       117
       +
       func (s *Spindle) streamLogs(ctx context.Context, conn *websocket.Conn, pipelineID string) error {

     

       118
       118
       +
       	l := s.l.With("pipelineID", pipelineID)

     

       119
       119
       +
       

     

       120
       120
       +
       	stdoutCh, stderrCh, ok := s.eng.LogChannels(pipelineID)

     

       121
       121
       +
       	if !ok {

     

       122
       122
       +
       		return fmt.Errorf("pipelineID %q not found", pipelineID)

     

       123
       123
       +
       	}

     

       124
       124
       +
       

     

       125
       125
       +
       	done := make(chan struct{})

     

       126
       126
       +
       

     

       127
       127
       +
       	go func() {

     

       128
       128
       +
       		for {

     

       129
       129
       +
       			select {

     

       130
       130
       +
       			case line, ok := <-stdoutCh:

     

       131
       131
       +
       				if !ok {

     

       132
       132
       +
       					done <- struct{}{}

     

       133
       133
       +
       					return

     

       134
       134
       +
       				}

     

       135
       135
       +
       				msg := map[string]string{"type": "stdout", "data": line}

     

       136
       136
       +
       				if err := conn.WriteJSON(msg); err != nil {

     

       137
       137
       +
       					l.Error("write stdout failed", "err", err)

     

       138
       138
       +
       					done <- struct{}{}

     

       139
       139
       +
       					return

     

       140
       140
       +
       				}

     

       141
       141
       +
       			case <-ctx.Done():

     

       142
       142
       +
       				done <- struct{}{}

     

       143
       143
       +
       				return

     

       144
       144
       +
       			}

     

       145
       145
       +
       		}

     

       146
       146
       +
       	}()

     

       147
       147
       +
       

     

       148
       148
       +
       	go func() {

     

       149
       149
       +
       		for {

     

       150
       150
       +
       			select {

     

       151
       151
       +
       			case line, ok := <-stderrCh:

     

       152
       152
       +
       				if !ok {

     

       153
       153
       +
       					done <- struct{}{}

     

       154
       154
       +
       					return

     

       155
       155
       +
       				}

     

       156
       156
       +
       				msg := map[string]string{"type": "stderr", "data": line}

     

       157
       157
       +
       				if err := conn.WriteJSON(msg); err != nil {

     

       158
       158
       +
       					l.Error("write stderr failed", "err", err)

     

       159
       159
       +
       					done <- struct{}{}

     

       160
       160
       +
       					return

     

       161
       161
       +
       				}

     

       162
       162
       +
       			case <-ctx.Done():

     

       163
       163
       +
       				done <- struct{}{}

     

       164
       164
       +
       				return

     

       165
       165
       +
       			}

     

       166
       166
       +
       		}

     

       167
       167
       +
       	}()

     

       168
       168
       +
       

     

       169
       169
       +
       	select {

     

       170
       170
       +
       	case <-done:

     

       171
       171
       +
       	case <-ctx.Done():

     

       172
       172
       +
       	}

     

       173
       173
       +
       

     

       174
       174
       +
       	return nil

     

       75
       175
        
       }

     

       76
       176
        
       

     

       77
       177
        
       func (s *Spindle) streamPipelines(conn *websocket.Conn, cursor *string) error {