From 544c474e1c180a574ec769dbefa7bfdcaef665ef Mon Sep 17 00:00:00 2001 From: Evan Jarrett Date: Mon, 24 Nov 2025 00:20:39 -0600 Subject: [PATCH] spindle: add environment variables to pipeline workflows Introduces framework-provided TANGLED_* environment variables (repo info, ref/SHA, trigger-specific data) that are injected into workflow steps. Signed-off-by: Evan Jarrett --- spindle/engines/nixery/engine.go | 9 +- spindle/models/clone.go | 13 +- spindle/models/pipeline.go | 7 +- spindle/models/pipeline_env.go | 77 ++++++++ spindle/models/pipeline_env_test.go | 260 ++++++++++++++++++++++++++++ spindle/server.go | 11 ++ 6 files changed, 362 insertions(+), 15 deletions(-) create mode 100644 spindle/models/pipeline_env.go create mode 100644 spindle/models/pipeline_env_test.go diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go index 86a49d47..d28e8377 100644 --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -73,7 +73,6 @@ func (ss *setupSteps) addStep(step models.Step) { type addlFields struct { image string container string - env map[string]string } func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { @@ -103,7 +102,7 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin swf.Steps = append(swf.Steps, sstep) } swf.Name = twf.Name - addl.env = dwf.Environment + swf.Environment = dwf.Environment addl.image = workflowImage(dwf.Dependencies, e.cfg.NixeryPipelines.Nixery) setup := &setupSteps{} @@ -288,7 +287,7 @@ func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *m func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger *models.WorkflowLogger) error { addl := w.Data.(addlFields) - workflowEnvs := ConstructEnvs(addl.env) + workflowEnvs := ConstructEnvs(w.Environment) // TODO(winter): should SetupWorkflow also have secret access? // IMO yes, but probably worth thinking on. for _, s := range secrets { @@ -310,7 +309,7 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W envs.AddEnv("HOME", homeDir) mkExecResp, err := e.docker.ContainerExecCreate(ctx, addl.container, container.ExecOptions{ - Cmd: []string{"bash", "-c", step.command}, + Cmd: []string{"bash", "-c", step.Command()}, AttachStdout: true, AttachStderr: true, Env: envs, @@ -333,7 +332,7 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W // Docker doesn't provide an API to kill an exec run // (sure, we could grab the PID and kill it ourselves, // but that's wasted effort) - e.l.Warn("step timed out", "step", step.Name) + e.l.Warn("step timed out", "step", step.Name()) <-tailDone diff --git a/spindle/models/clone.go b/spindle/models/clone.go index c4b99826..2fe8753c 100644 --- a/spindle/models/clone.go +++ b/spindle/models/clone.go @@ -55,7 +55,7 @@ func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMe } } - repoURL := buildRepoURL(tr, dev) + repoURL := BuildRepoURL(tr.Repo, dev) var cloneOpts tangled.Pipeline_CloneOpts if twf.Clone != nil { @@ -101,20 +101,19 @@ func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { } } -// buildRepoURL constructs the repository URL from trigger metadata -func buildRepoURL(tr tangled.Pipeline_TriggerMetadata, devMode bool) string { - if tr.Repo == nil { +// BuildRepoURL constructs the repository URL from repo metadata. +func BuildRepoURL(repo *tangled.Pipeline_TriggerRepo, devMode bool) string { + if repo == nil { return "" } - // Determine protocol scheme := "https://" if devMode { scheme = "http://" } // Get host from knot - host := tr.Repo.Knot + host := repo.Knot // In dev mode, replace localhost with host.docker.internal for Docker networking if devMode && strings.Contains(host, "localhost") { @@ -122,7 +121,7 @@ func buildRepoURL(tr tangled.Pipeline_TriggerMetadata, devMode bool) string { } // Build URL: {scheme}{knot}/{did}/{repo} - return fmt.Sprintf("%s%s/%s/%s", scheme, host, tr.Repo.Did, tr.Repo.Repo) + return fmt.Sprintf("%s%s/%s/%s", scheme, host, repo.Did, repo.Repo) } // buildFetchArgs constructs the arguments for git fetch based on clone options diff --git a/spindle/models/pipeline.go b/spindle/models/pipeline.go index 3919d8b2..8f35d622 100644 --- a/spindle/models/pipeline.go +++ b/spindle/models/pipeline.go @@ -22,7 +22,8 @@ const ( ) type Workflow struct { - Steps []Step - Name string - Data any + Steps []Step + Name string + Data any + Environment map[string]string } diff --git a/spindle/models/pipeline_env.go b/spindle/models/pipeline_env.go new file mode 100644 index 00000000..9aca16b4 --- /dev/null +++ b/spindle/models/pipeline_env.go @@ -0,0 +1,77 @@ +package models + +import ( + "strings" + + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/api/tangled" + "tangled.org/core/workflow" +) + +// PipelineEnvVars extracts environment variables from pipeline trigger metadata. +// These are framework-provided variables that are injected into workflow steps. +func PipelineEnvVars(tr *tangled.Pipeline_TriggerMetadata, pipelineId PipelineId, devMode bool) map[string]string { + if tr == nil { + return nil + } + + env := make(map[string]string) + + // Standard CI environment variable + env["CI"] = "true" + + env["TANGLED_PIPELINE_ID"] = pipelineId.Rkey + + // Repo info + if tr.Repo != nil { + env["TANGLED_REPO_KNOT"] = tr.Repo.Knot + env["TANGLED_REPO_DID"] = tr.Repo.Did + env["TANGLED_REPO_NAME"] = tr.Repo.Repo + env["TANGLED_REPO_DEFAULT_BRANCH"] = tr.Repo.DefaultBranch + env["TANGLED_REPO_URL"] = BuildRepoURL(tr.Repo, devMode) + } + + switch workflow.TriggerKind(tr.Kind) { + case workflow.TriggerKindPush: + if tr.Push != nil { + refName := plumbing.ReferenceName(tr.Push.Ref) + refType := "branch" + if refName.IsTag() { + refType = "tag" + } + + env["TANGLED_REF"] = tr.Push.Ref + env["TANGLED_REF_NAME"] = refName.Short() + env["TANGLED_REF_TYPE"] = refType + env["TANGLED_SHA"] = tr.Push.NewSha + env["TANGLED_COMMIT_SHA"] = tr.Push.NewSha + } + + case workflow.TriggerKindPullRequest: + if tr.PullRequest != nil { + // For PRs, the "ref" is the source branch + env["TANGLED_REF"] = "refs/heads/" + tr.PullRequest.SourceBranch + env["TANGLED_REF_NAME"] = tr.PullRequest.SourceBranch + env["TANGLED_REF_TYPE"] = "branch" + env["TANGLED_SHA"] = tr.PullRequest.SourceSha + env["TANGLED_COMMIT_SHA"] = tr.PullRequest.SourceSha + + // PR-specific variables + env["TANGLED_PR_SOURCE_BRANCH"] = tr.PullRequest.SourceBranch + env["TANGLED_PR_TARGET_BRANCH"] = tr.PullRequest.TargetBranch + env["TANGLED_PR_SOURCE_SHA"] = tr.PullRequest.SourceSha + env["TANGLED_PR_ACTION"] = tr.PullRequest.Action + } + + case workflow.TriggerKindManual: + // Manual triggers may not have ref/sha info + // Include any manual inputs if present + if tr.Manual != nil { + for _, pair := range tr.Manual.Inputs { + env["TANGLED_INPUT_"+strings.ToUpper(pair.Key)] = pair.Value + } + } + } + + return env +} diff --git a/spindle/models/pipeline_env_test.go b/spindle/models/pipeline_env_test.go new file mode 100644 index 00000000..3560c881 --- /dev/null +++ b/spindle/models/pipeline_env_test.go @@ -0,0 +1,260 @@ +package models + +import ( + "testing" + + "tangled.org/core/api/tangled" + "tangled.org/core/workflow" +) + +func TestPipelineEnvVars_PushBranch(t *testing.T) { + tr := &tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123def456", + OldSha: "000000000000", + Ref: "refs/heads/main", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + DefaultBranch: "main", + }, + } + id := PipelineId{ + Knot: "example.com", + Rkey: "123123", + } + env := PipelineEnvVars(tr, id, false) + + // Check standard CI variable + if env["CI"] != "true" { + t.Errorf("Expected CI='true', got '%s'", env["CI"]) + } + + // Check ref variables + if env["TANGLED_REF"] != "refs/heads/main" { + t.Errorf("Expected TANGLED_REF='refs/heads/main', got '%s'", env["TANGLED_REF"]) + } + if env["TANGLED_REF_NAME"] != "main" { + t.Errorf("Expected TANGLED_REF_NAME='main', got '%s'", env["TANGLED_REF_NAME"]) + } + if env["TANGLED_REF_TYPE"] != "branch" { + t.Errorf("Expected TANGLED_REF_TYPE='branch', got '%s'", env["TANGLED_REF_TYPE"]) + } + + // Check SHA variables + if env["TANGLED_SHA"] != "abc123def456" { + t.Errorf("Expected TANGLED_SHA='abc123def456', got '%s'", env["TANGLED_SHA"]) + } + if env["TANGLED_COMMIT_SHA"] != "abc123def456" { + t.Errorf("Expected TANGLED_COMMIT_SHA='abc123def456', got '%s'", env["TANGLED_COMMIT_SHA"]) + } + + // Check repo variables + if env["TANGLED_REPO_KNOT"] != "example.com" { + t.Errorf("Expected TANGLED_REPO_KNOT='example.com', got '%s'", env["TANGLED_REPO_KNOT"]) + } + if env["TANGLED_REPO_DID"] != "did:plc:user123" { + t.Errorf("Expected TANGLED_REPO_DID='did:plc:user123', got '%s'", env["TANGLED_REPO_DID"]) + } + if env["TANGLED_REPO_NAME"] != "my-repo" { + t.Errorf("Expected TANGLED_REPO_NAME='my-repo', got '%s'", env["TANGLED_REPO_NAME"]) + } + if env["TANGLED_REPO_DEFAULT_BRANCH"] != "main" { + t.Errorf("Expected TANGLED_REPO_DEFAULT_BRANCH='main', got '%s'", env["TANGLED_REPO_DEFAULT_BRANCH"]) + } + if env["TANGLED_REPO_URL"] != "https://example.com/did:plc:user123/my-repo" { + t.Errorf("Expected TANGLED_REPO_URL='https://example.com/did:plc:user123/my-repo', got '%s'", env["TANGLED_REPO_URL"]) + } +} + +func TestPipelineEnvVars_PushTag(t *testing.T) { + tr := &tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123def456", + OldSha: "000000000000", + Ref: "refs/tags/v1.2.3", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + id := PipelineId{ + Knot: "example.com", + Rkey: "123123", + } + env := PipelineEnvVars(tr, id, false) + + if env["TANGLED_REF"] != "refs/tags/v1.2.3" { + t.Errorf("Expected TANGLED_REF='refs/tags/v1.2.3', got '%s'", env["TANGLED_REF"]) + } + if env["TANGLED_REF_NAME"] != "v1.2.3" { + t.Errorf("Expected TANGLED_REF_NAME='v1.2.3', got '%s'", env["TANGLED_REF_NAME"]) + } + if env["TANGLED_REF_TYPE"] != "tag" { + t.Errorf("Expected TANGLED_REF_TYPE='tag', got '%s'", env["TANGLED_REF_TYPE"]) + } +} + +func TestPipelineEnvVars_PullRequest(t *testing.T) { + tr := &tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPullRequest), + PullRequest: &tangled.Pipeline_PullRequestTriggerData{ + SourceBranch: "feature-branch", + TargetBranch: "main", + SourceSha: "pr-sha-789", + Action: "opened", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + id := PipelineId{ + Knot: "example.com", + Rkey: "123123", + } + env := PipelineEnvVars(tr, id, false) + + // Check ref variables for PR + if env["TANGLED_REF"] != "refs/heads/feature-branch" { + t.Errorf("Expected TANGLED_REF='refs/heads/feature-branch', got '%s'", env["TANGLED_REF"]) + } + if env["TANGLED_REF_NAME"] != "feature-branch" { + t.Errorf("Expected TANGLED_REF_NAME='feature-branch', got '%s'", env["TANGLED_REF_NAME"]) + } + if env["TANGLED_REF_TYPE"] != "branch" { + t.Errorf("Expected TANGLED_REF_TYPE='branch', got '%s'", env["TANGLED_REF_TYPE"]) + } + + // Check SHA variables + if env["TANGLED_SHA"] != "pr-sha-789" { + t.Errorf("Expected TANGLED_SHA='pr-sha-789', got '%s'", env["TANGLED_SHA"]) + } + if env["TANGLED_COMMIT_SHA"] != "pr-sha-789" { + t.Errorf("Expected TANGLED_COMMIT_SHA='pr-sha-789', got '%s'", env["TANGLED_COMMIT_SHA"]) + } + + // Check PR-specific variables + if env["TANGLED_PR_SOURCE_BRANCH"] != "feature-branch" { + t.Errorf("Expected TANGLED_PR_SOURCE_BRANCH='feature-branch', got '%s'", env["TANGLED_PR_SOURCE_BRANCH"]) + } + if env["TANGLED_PR_TARGET_BRANCH"] != "main" { + t.Errorf("Expected TANGLED_PR_TARGET_BRANCH='main', got '%s'", env["TANGLED_PR_TARGET_BRANCH"]) + } + if env["TANGLED_PR_SOURCE_SHA"] != "pr-sha-789" { + t.Errorf("Expected TANGLED_PR_SOURCE_SHA='pr-sha-789', got '%s'", env["TANGLED_PR_SOURCE_SHA"]) + } + if env["TANGLED_PR_ACTION"] != "opened" { + t.Errorf("Expected TANGLED_PR_ACTION='opened', got '%s'", env["TANGLED_PR_ACTION"]) + } +} + +func TestPipelineEnvVars_ManualWithInputs(t *testing.T) { + tr := &tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindManual), + Manual: &tangled.Pipeline_ManualTriggerData{ + Inputs: []*tangled.Pipeline_Pair{ + {Key: "version", Value: "1.0.0"}, + {Key: "environment", Value: "production"}, + }, + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + id := PipelineId{ + Knot: "example.com", + Rkey: "123123", + } + env := PipelineEnvVars(tr, id, false) + + // Check manual input variables + if env["TANGLED_INPUT_VERSION"] != "1.0.0" { + t.Errorf("Expected TANGLED_INPUT_VERSION='1.0.0', got '%s'", env["TANGLED_INPUT_VERSION"]) + } + if env["TANGLED_INPUT_ENVIRONMENT"] != "production" { + t.Errorf("Expected TANGLED_INPUT_ENVIRONMENT='production', got '%s'", env["TANGLED_INPUT_ENVIRONMENT"]) + } + + // Manual triggers shouldn't have ref/sha variables + if _, ok := env["TANGLED_REF"]; ok { + t.Error("Manual trigger should not have TANGLED_REF") + } + if _, ok := env["TANGLED_SHA"]; ok { + t.Error("Manual trigger should not have TANGLED_SHA") + } +} + +func TestPipelineEnvVars_DevMode(t *testing.T) { + tr := &tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "abc123", + Ref: "refs/heads/main", + }, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "localhost:3000", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + id := PipelineId{ + Knot: "example.com", + Rkey: "123123", + } + env := PipelineEnvVars(tr, id, true) + + // Dev mode should use http:// and replace localhost with host.docker.internal + expectedURL := "http://host.docker.internal:3000/did:plc:user123/my-repo" + if env["TANGLED_REPO_URL"] != expectedURL { + t.Errorf("Expected TANGLED_REPO_URL='%s', got '%s'", expectedURL, env["TANGLED_REPO_URL"]) + } +} + +func TestPipelineEnvVars_NilTrigger(t *testing.T) { + id := PipelineId{ + Knot: "example.com", + Rkey: "123123", + } + env := PipelineEnvVars(nil, id, false) + + if env != nil { + t.Error("Expected nil env for nil trigger") + } +} + +func TestPipelineEnvVars_NilPushData(t *testing.T) { + tr := &tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: nil, + Repo: &tangled.Pipeline_TriggerRepo{ + Knot: "example.com", + Did: "did:plc:user123", + Repo: "my-repo", + }, + } + id := PipelineId{ + Knot: "example.com", + Rkey: "123123", + } + env := PipelineEnvVars(tr, id, false) + + // Should still have repo variables + if env["TANGLED_REPO_KNOT"] != "example.com" { + t.Errorf("Expected TANGLED_REPO_KNOT='example.com', got '%s'", env["TANGLED_REPO_KNOT"]) + } + + // Should not have ref/sha variables + if _, ok := env["TANGLED_REF"]; ok { + t.Error("Should not have TANGLED_REF when push data is nil") + } +} diff --git a/spindle/server.go b/spindle/server.go index 0805f7d4..6ce9464d 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "log/slog" + "maps" "net/http" "github.com/go-chi/chi/v5" @@ -311,6 +312,9 @@ func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, workflows := make(map[models.Engine][]models.Workflow) + // Build pipeline environment variables once for all workflows + pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId, s.cfg.Server.Dev) + for _, w := range tpl.Workflows { if w != nil { if _, ok := s.engs[w.Engine]; !ok { @@ -336,6 +340,13 @@ func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, return err } + // inject TANGLED_* env vars after InitWorkflow + // This prevents user-defined env vars from overriding them + if ewf.Environment == nil { + ewf.Environment = make(map[string]string) + } + maps.Copy(ewf.Environment, pipelineEnv) + workflows[eng] = append(workflows[eng], *ewf) err = s.db.StatusPending(models.WorkflowId{ -- 2.43.0