forked from tangled.org/core
this repo has no description

appview,spindle: collapse system introduced steps

Signed-off-by: oppiliappan <me@oppi.li>

oppi.li 57f25f89 10b3b561

verified
Changed files
+190 -94
appview
pages
templates
pipelines
spindle
+20
appview/pages/pages.go
···
return p.executeRepo("repo/pipelines/pipelines", w, params)
}
+
type LogBlockParams struct {
+
Id int
+
Name string
+
Command string
+
Collapsed bool
+
}
+
+
func (p *Pages) LogBlock(w io.Writer, params LogBlockParams) error {
+
return p.executePlain("repo/pipelines/fragments/logBlock", w, params)
+
}
+
+
type LogLineParams struct {
+
Id int
+
Content string
+
}
+
+
func (p *Pages) LogLine(w io.Writer, params LogLineParams) error {
+
return p.executePlain("repo/pipelines/fragments/logLine", w, params)
+
}
+
type WorkflowParams struct {
LoggedInUser *oauth.User
RepoInfo repoinfo.RepoInfo
+16
appview/pages/templates/repo/pipelines/fragments/logBlock.html
···
+
{{ define "repo/pipelines/fragments/logBlock" }}
+
<div id="lines" hx-swap-oob="beforeend">
+
<details id="step-{{ .Id }}" {{if not .Collapsed}}open{{end}} class="group bg-gray-100 px-2 dark:bg-gray-900">
+
<summary class="sticky top-0 py-1 list-none cursor-pointer py-2 bg-gray-100 dark:bg-gray-900 hover:text-gray-500 hover:dark:text-gray-400">
+
<div class="group-open:hidden flex items-center gap-1">
+
{{ i "chevron-right" "w-4 h-4" }} {{ .Name }}
+
</div>
+
<div class="hidden group-open:flex items-center gap-1">
+
{{ i "chevron-down" "w-4 h-4" }} {{ .Name }}
+
</div>
+
</summary>
+
<div class="text-blue-600 dark:text-blue-300 font-mono">{{ .Command }}</div>
+
<div id="step-body-{{ .Id }}" class="font-mono"></div>
+
</details>
+
</div>
+
{{ end }}
+6
appview/pages/templates/repo/pipelines/fragments/logLine.html
···
+
{{ define "repo/pipelines/fragments/logLine" }}
+
<div id="step-body-{{ .Id }}" hx-swap-oob="beforeend">
+
<p>{{ .Content }}</p>
+
</div>
+
{{ end }}
+
+5 -13
appview/pages/templates/repo/pipelines/pipelines.html
···
{{ define "repoContent" }}
<div class="flex justify-between items-center gap-4">
-
<div class="flex gap-4">
-
</div>
-
-
</div>
-
<div class="error" id="issues"></div>
-
{{ end }}
-
-
{{ define "repoAfter" }}
-
<section
-
class="w-full flex flex-col gap-2 mt-2"
-
>
+
<div class="w-full flex flex-col gap-2">
{{ range .Pipelines }}
{{ block "pipeline" (list $ .) }} {{ end }}
{{ else }}
···
No pipelines run for this repository.
</p>
{{ end }}
-
</section>
+
</div>
+
</div>
{{ end }}
+
{{ define "pipeline" }}
{{ $root := index . 0 }}
{{ $p := index . 1 }}
-
<div class="py-4 px-6 bg-white dark:bg-gray-800 dark:text-white">
+
<div class="py-2 bg-white dark:bg-gray-800 dark:text-white">
{{ block "pipelineHeader" $ }} {{ end }}
</div>
{{ end }}
+3 -5
appview/pages/templates/repo/pipelines/workflow.html
···
{{ $active := .Workflow }}
{{ with .Pipeline }}
{{ $id := .Id }}
-
<div class="grid grid-cols-1 rounded border border-gray-200 dark:border-gray-700 divide-y divide-gray-200 dark:divide-gray-700">
+
<div class="sticky top-2 grid grid-cols-1 rounded border border-gray-200 dark:border-gray-700 divide-y divide-gray-200 dark:divide-gray-700">
{{ range $name, $all := .Statuses }}
<a href="/{{ $.RepoInfo.FullName }}/pipelines/{{ $id }}/workflow/{{ $name }}" class="no-underline hover:no-underline hover:bg-gray-100/25 hover:dark:bg-gray-700/25">
<div
···
{{ define "logs" }}
<div id="log-stream"
-
class="p-2 bg-gray-100 dark:bg-gray-900 font-mono text-sm min-h-96 max-h-screen overflow-auto flex flex-col-reverse [overflow-anchor:auto_!important]"
+
class="text-sm"
hx-ext="ws"
ws-connect="/{{ $.RepoInfo.FullName }}/pipelines/{{ .Pipeline.Id }}/workflow/{{ .Workflow }}/logs">
-
<div id="lines">
-
<!-- Each log line should be rendered with class="item" like below -->
-
<!-- <div class="item">[INFO] Log line here</div> -->
+
<div id="lines" class="flex flex-col gap-2">
</div>
</div>
{{ end }}
+72 -58
appview/pipelines/pipelines.go
···
package pipelines
import (
+
"bytes"
"context"
"encoding/json"
-
"fmt"
-
"html"
"log/slog"
"net/http"
"strings"
···
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
-
go func() {
-
for {
-
if _, _, err := clientConn.NextReader(); err != nil {
-
l.Error("failed to read", "err", err)
-
cancel()
-
return
-
}
-
}
-
}()
user := p.oauth.GetUser(r)
f, err := p.repoResolver.Resolve(r)
···
defer spindleConn.Close()
// create a channel for incoming messages
-
msgChan := make(chan []byte, 10)
-
errChan := make(chan error, 1)
-
+
evChan := make(chan logEvent, 100)
// start a goroutine to read from spindle
-
go func() {
-
defer close(msgChan)
-
defer close(errChan)
-
-
for {
-
_, msg, err := spindleConn.ReadMessage()
-
if err != nil {
-
if websocket.IsCloseError(err,
-
websocket.CloseNormalClosure,
-
websocket.CloseGoingAway,
-
websocket.CloseAbnormalClosure) {
-
errChan <- nil // signal graceful end
-
} else {
-
errChan <- err
-
}
-
return
-
}
-
msgChan <- msg
-
}
-
}()
+
go readLogs(spindleConn, evChan)
stepIdx := 0
+
var fragment bytes.Buffer
for {
select {
case <-ctx.Done():
l.Info("client disconnected")
return
-
case err := <-errChan:
-
if err != nil {
-
l.Error("error reading from spindle", "err", err)
+
+
case ev, ok := <-evChan:
+
if !ok {
+
continue
}
-
if err == nil {
-
l.Info("log tail complete")
+
if ev.err != nil && ev.isCloseError() {
+
l.Debug("graceful shutdown, tail complete", "err", err)
+
return
+
}
+
if ev.err != nil {
+
l.Error("error reading from spindle", "err", err)
+
return
}
-
return
-
case msg := <-msgChan:
var logLine spindlemodel.LogLine
-
if err = json.Unmarshal(msg, &logLine); err != nil {
+
if err = json.Unmarshal(ev.msg, &logLine); err != nil {
l.Error("failed to parse logline", "err", err)
continue
}
-
var fragment []byte
+
fragment.Reset()
+
switch logLine.Kind {
case spindlemodel.LogKindControl:
// control messages create a new step block
stepIdx++
-
fragment = fmt.Appendf(nil, `
-
<div id="lines" hx-swap-oob="beforeend">
-
<details id="step-%d" open>
-
<summary>%s</summary>
-
<div id="step-body-%d"></div>
-
</details>
-
</div>
-
`, stepIdx, logLine.Content, stepIdx)
+
collapsed := false
+
if logLine.StepKind == spindlemodel.StepKindSystem {
+
collapsed = true
+
}
+
err = p.pages.LogBlock(&fragment, pages.LogBlockParams{
+
Id: stepIdx,
+
Name: logLine.Content,
+
Command: logLine.StepCommand,
+
Collapsed: collapsed,
+
})
case spindlemodel.LogKindData:
// data messages simply insert new log lines into current step
-
escaped := html.EscapeString(logLine.Content)
-
fragment = fmt.Appendf(nil, `
-
<div id="step-body-%d" hx-swap-oob="beforeend">
-
<p>%s</p>
-
</div>
-
`, stepIdx, escaped)
+
err = p.pages.LogLine(&fragment, pages.LogLineParams{
+
Id: stepIdx,
+
Content: logLine.Content,
+
})
+
}
+
if err != nil {
+
l.Error("failed to render log line", "err", err)
+
return
}
-
if err = clientConn.WriteMessage(websocket.TextMessage, fragment); err != nil {
+
if err = clientConn.WriteMessage(websocket.TextMessage, fragment.Bytes()); err != nil {
l.Error("error writing to client", "err", err)
return
}
+
case <-time.After(30 * time.Second):
l.Debug("sent keepalive")
if err = clientConn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second)); err != nil {
l.Error("failed to write control", "err", err)
+
return
}
}
}
}
+
+
// either a message or an error
+
type logEvent struct {
+
msg []byte
+
err error
+
}
+
+
func (ev *logEvent) isCloseError() bool {
+
return websocket.IsCloseError(
+
ev.err,
+
websocket.CloseNormalClosure,
+
websocket.CloseGoingAway,
+
websocket.CloseAbnormalClosure,
+
)
+
}
+
+
// read logs from spindle and pass through to chan
+
func readLogs(conn *websocket.Conn, ch chan logEvent) {
+
defer close(ch)
+
+
for {
+
if conn == nil {
+
return
+
}
+
+
_, msg, err := conn.ReadMessage()
+
if err != nil {
+
ch <- logEvent{err: err}
+
return
+
}
+
ch <- logEvent{msg: msg}
+
}
+
}
+2 -2
spindle/engine/engine.go
···
}
defer wfLogger.Close()
-
ctl := wfLogger.ControlWriter()
-
ctl.Write([]byte(step.Command))
+
ctl := wfLogger.ControlWriter(stepIdx, step)
+
ctl.Write([]byte(step.Name))
logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{
Follow: true,
+27 -12
spindle/engine/logger.go
···
func (l *WorkflowLogger) DataWriter(stream string) io.Writer {
// TODO: emit stream
-
return &jsonWriter{logger: l, kind: models.LogKindData}
+
return &dataWriter{
+
logger: l,
+
stream: stream,
+
}
}
-
func (l *WorkflowLogger) ControlWriter() io.Writer {
-
return &jsonWriter{logger: l, kind: models.LogKindControl}
+
func (l *WorkflowLogger) ControlWriter(idx int, step models.Step) io.Writer {
+
return &controlWriter{
+
logger: l,
+
idx: idx,
+
step: step,
+
}
}
-
type jsonWriter struct {
+
type dataWriter struct {
logger *WorkflowLogger
-
kind models.LogKind
+
stream string
}
-
func (w *jsonWriter) Write(p []byte) (int, error) {
+
func (w *dataWriter) Write(p []byte) (int, error) {
line := strings.TrimRight(string(p), "\r\n")
+
entry := models.NewDataLogLine(line, w.stream)
+
if err := w.logger.encoder.Encode(entry); err != nil {
+
return 0, err
+
}
+
return len(p), nil
+
}
-
entry := models.LogLine{
-
Kind: w.kind,
-
Content: line,
-
}
+
type controlWriter struct {
+
logger *WorkflowLogger
+
idx int
+
step models.Step
+
}
+
func (w *controlWriter) Write(_ []byte) (int, error) {
+
entry := models.NewControlLogLine(w.idx, w.step)
if err := w.logger.encoder.Encode(entry); err != nil {
return 0, err
}
-
-
return len(p), nil
+
return len(w.step.Name), nil
}
+21
spindle/models/models.go
···
Stream string `json:"stream,omitempty"`
// fields if kind is "control"
+
StepId int `json:"step_id,omitempty"`
+
StepKind StepKind `json:"step_kind,omitempty"`
+
StepCommand string `json:"step_command,omitempty"`
+
}
+
+
func NewDataLogLine(content, stream string) LogLine {
+
return LogLine{
+
Kind: LogKindData,
+
Content: content,
+
Stream: stream,
+
}
+
}
+
+
func NewControlLogLine(idx int, step Step) LogLine {
+
return LogLine{
+
Kind: LogKindControl,
+
Content: step.Name,
+
StepId: idx,
+
StepKind: step.Kind,
+
StepCommand: step.Command,
+
}
}
+15 -1
spindle/models/pipeline.go
···
Command string
Name string
Environment map[string]string
+
Kind StepKind
}
+
+
type StepKind int
+
+
const (
+
// steps injected by the CI runner
+
StepKindSystem StepKind = iota
+
// steps defined by the user in the original pipeline
+
StepKindUser
+
)
type Workflow struct {
Steps []Step
···
sstep.Environment = stepEnvToMap(tstep.Environment)
sstep.Command = tstep.Command
sstep.Name = tstep.Name
+
sstep.Kind = StepKindUser
swf.Steps = append(swf.Steps, sstep)
}
swf.Name = twf.Name
···
setup.addStep(nixConfStep())
setup.addStep(cloneStep(*twf, *pl.TriggerMetadata.Repo, cfg.Server.Dev))
setup.addStep(checkoutStep(*twf, *pl.TriggerMetadata))
-
setup.addStep(dependencyStep(*twf))
+
// this step could be empty
+
if s := dependencyStep(*twf); s != nil {
+
setup.addStep(*s)
+
}
// append setup steps in order to the start of workflow steps
swf.Steps = append(*setup, swf.Steps...)
+3 -3
spindle/models/setup_steps.go
···
// For dependencies using a custom registry (i.e. not nixpkgs), it collects
// all packages and adds a single 'nix profile install' step to the
// beginning of the workflow's step list.
-
func dependencyStep(twf tangled.Pipeline_Workflow) Step {
+
func dependencyStep(twf tangled.Pipeline_Workflow) *Step {
var customPackages []string
for _, d := range twf.Dependencies {
···
"NIX_SHOW_DOWNLOAD_PROGRESS": "0",
},
}
-
return installStep
+
return &installStep
}
-
return Step{}
+
return nil
}