From 6213b77baa4ca22217164dd359047f9f710e5b3e Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Mon, 16 Jun 2025 10:06:05 +0300 Subject: [PATCH] workflow,lexicons: allow setting per-step env vars Change-Id: qvusxmswyuxqrkzztyxynkmuttqpptkr Signed-off-by: Anirudh Oppiliappan --- api/tangled/cbor_gen.go | 221 ++++++++++++++++++++++++++++++++- api/tangled/tangledpipeline.go | 12 +- cmd/gen.go | 1 + lexicons/pipeline.json | 85 ++++--------- workflow/compile.go | 7 ++ workflow/def.go | 5 +- workflow/def_test.go | 9 ++ 7 files changed, 275 insertions(+), 65 deletions(-) diff --git a/api/tangled/cbor_gen.go b/api/tangled/cbor_gen.go index aafde3d..fef37ff 100644 --- a/api/tangled/cbor_gen.go +++ b/api/tangled/cbor_gen.go @@ -3169,7 +3169,7 @@ func (t *Pipeline_PushTriggerData) UnmarshalCBOR(r io.Reader) (err error) { return nil } -func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { +func (t *Pipeline_Step_Environment_Elem) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -3181,6 +3181,145 @@ func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { return err } + // t.Key (string) (string) + if len("key") > 1000000 { + return xerrors.Errorf("Value in field \"key\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("key"))); err != nil { + return err + } + if _, err := cw.WriteString(string("key")); err != nil { + return err + } + + if len(t.Key) > 1000000 { + return xerrors.Errorf("Value in field t.Key was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Key))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Key)); err != nil { + return err + } + + // t.Value (string) (string) + if len("value") > 1000000 { + return xerrors.Errorf("Value in field \"value\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("value"))); err != nil { + return err + } + if _, err := cw.WriteString(string("value")); err != nil { + return err + } + + if len(t.Value) > 1000000 { + return xerrors.Errorf("Value in field t.Value was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Value)); err != nil { + return err + } + return nil +} + +func (t *Pipeline_Step_Environment_Elem) UnmarshalCBOR(r io.Reader) (err error) { + *t = Pipeline_Step_Environment_Elem{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Pipeline_Step_Environment_Elem: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Key (string) (string) + case "key": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Key = string(sval) + } + // t.Value (string) (string) + case "value": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Value = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 3 + + if t.Environment == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + // t.Name (string) (string) if len("name") > 1000000 { return xerrors.Errorf("Value in field \"name\" was too long") @@ -3226,6 +3365,35 @@ func (t *Pipeline_Step) MarshalCBOR(w io.Writer) error { if _, err := cw.WriteString(string(t.Command)); err != nil { return err } + + // t.Environment ([]*tangled.Pipeline_Step_Environment_Elem) (slice) + if t.Environment != nil { + + if len("environment") > 1000000 { + return xerrors.Errorf("Value in field \"environment\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("environment"))); err != nil { + return err + } + if _, err := cw.WriteString(string("environment")); err != nil { + return err + } + + if len(t.Environment) > 8192 { + return xerrors.Errorf("Slice value in field t.Environment was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Environment))); err != nil { + return err + } + for _, v := range t.Environment { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + } return nil } @@ -3254,7 +3422,7 @@ func (t *Pipeline_Step) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 7) + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -3292,6 +3460,55 @@ func (t *Pipeline_Step) UnmarshalCBOR(r io.Reader) (err error) { t.Command = string(sval) } + // t.Environment ([]*tangled.Pipeline_Step_Environment_Elem) (slice) + case "environment": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Environment: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Environment = make([]*Pipeline_Step_Environment_Elem, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Environment[i] = new(Pipeline_Step_Environment_Elem) + if err := t.Environment[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Environment[i] pointer: %w", err) + } + } + + } + + } + } default: // Field doesn't exist on this type, so ignore it diff --git a/api/tangled/tangledpipeline.go b/api/tangled/tangledpipeline.go index 11c8b1b..208795e 100644 --- a/api/tangled/tangledpipeline.go +++ b/api/tangled/tangledpipeline.go @@ -61,8 +61,14 @@ type Pipeline_PushTriggerData struct { // Pipeline_Step is a "step" in the sh.tangled.pipeline schema. type Pipeline_Step struct { - Command string `json:"command" cborgen:"command"` - Name string `json:"name" cborgen:"name"` + Command string `json:"command" cborgen:"command"` + Environment []*Pipeline_Step_Environment_Elem `json:"environment,omitempty" cborgen:"environment,omitempty"` + Name string `json:"name" cborgen:"name"` +} + +type Pipeline_Step_Environment_Elem struct { + Key string `json:"key" cborgen:"key"` + Value string `json:"value" cborgen:"value"` } // Pipeline_TriggerMetadata is a "triggerMetadata" in the sh.tangled.pipeline schema. @@ -85,7 +91,7 @@ type Pipeline_TriggerRepo struct { // Pipeline_Workflow is a "workflow" in the sh.tangled.pipeline schema. type Pipeline_Workflow struct { Clone *Pipeline_CloneOpts `json:"clone" cborgen:"clone"` - Dependencies []Pipeline_Dependencies_Elem `json:"dependencies" cborgen:"dependencies"` + Dependencies []Pipeline_Dependencies_Elem `json:"dependencies" cborgen:"dependencies"` Environment []*Pipeline_Workflow_Environment_Elem `json:"environment" cborgen:"environment"` Name string `json:"name" cborgen:"name"` Steps []*Pipeline_Step `json:"steps" cborgen:"steps"` diff --git a/cmd/gen.go b/cmd/gen.go index aeeb3bd..06be449 100644 --- a/cmd/gen.go +++ b/cmd/gen.go @@ -29,6 +29,7 @@ func main() { tangled.Pipeline_ManualTriggerData{}, tangled.Pipeline_PullRequestTriggerData{}, tangled.Pipeline_PushTriggerData{}, + tangled.Pipeline_Step_Environment_Elem{}, tangled.Pipeline_Step{}, tangled.Pipeline_TriggerMetadata{}, tangled.Pipeline_TriggerRepo{}, diff --git a/lexicons/pipeline.json b/lexicons/pipeline.json index 63b0945..5092dd8 100644 --- a/lexicons/pipeline.json +++ b/lexicons/pipeline.json @@ -9,10 +9,7 @@ "key": "tid", "record": { "type": "object", - "required": [ - "triggerMetadata", - "workflows" - ], + "required": ["triggerMetadata", "workflows"], "properties": { "triggerMetadata": { "type": "ref", @@ -30,18 +27,11 @@ }, "triggerMetadata": { "type": "object", - "required": [ - "kind", - "repo" - ], + "required": ["kind", "repo"], "properties": { "kind": { "type": "string", - "enum": [ - "push", - "pull_request", - "manual" - ] + "enum": ["push", "pull_request", "manual"] }, "repo": { "type": "ref", @@ -63,12 +53,7 @@ }, "triggerRepo": { "type": "object", - "required": [ - "knot", - "did", - "repo", - "defaultBranch" - ], + "required": ["knot", "did", "repo", "defaultBranch"], "properties": { "knot": { "type": "string" @@ -87,11 +72,7 @@ }, "pushTriggerData": { "type": "object", - "required": [ - "ref", - "newSha", - "oldSha" - ], + "required": ["ref", "newSha", "oldSha"], "properties": { "ref": { "type": "string" @@ -110,12 +91,7 @@ }, "pullRequestTriggerData": { "type": "object", - "required": [ - "sourceBranch", - "targetBranch", - "sourceSha", - "action" - ], + "required": ["sourceBranch", "targetBranch", "sourceSha", "action"], "properties": { "sourceBranch": { "type": "string" @@ -140,10 +116,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "key", - "value" - ], + "required": ["key", "value"], "properties": { "key": { "type": "string" @@ -158,13 +131,7 @@ }, "workflow": { "type": "object", - "required": [ - "name", - "dependencies", - "steps", - "environment", - "clone" - ], + "required": ["name", "dependencies", "steps", "environment", "clone"], "properties": { "name": { "type": "string" @@ -184,10 +151,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "key", - "value" - ], + "required": ["key", "value"], "properties": { "key": { "type": "string" @@ -208,10 +172,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "registry", - "packages" - ], + "required": ["registry", "packages"], "properties": { "registry": { "type": "string" @@ -227,11 +188,7 @@ }, "cloneOpts": { "type": "object", - "required": [ - "skip", - "depth", - "submodules" - ], + "required": ["skip", "depth", "submodules"], "properties": { "skip": { "type": "boolean" @@ -246,16 +203,28 @@ }, "step": { "type": "object", - "required": [ - "name", - "command" - ], + "required": ["name", "command"], "properties": { "name": { "type": "string" }, "command": { "type": "string" + }, + "environment": { + "type": "array", + "items": { + "type": "object", + "required": ["key", "value"], + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } } } } diff --git a/workflow/compile.go b/workflow/compile.go index df53784..53f2fe1 100644 --- a/workflow/compile.go +++ b/workflow/compile.go @@ -97,6 +97,13 @@ func (compiler *Compiler) compileWorkflow(w Workflow) tangled.Pipeline_Workflow Command: s.Command, Name: s.Name, } + for k, v := range s.Environment { + e := &tangled.Pipeline_Step_Environment_Elem{ + Key: k, + Value: v, + } + step.Environment = append(step.Environment, e) + } cw.Steps = append(cw.Steps, &step) } for k, v := range w.Environment { diff --git a/workflow/def.go b/workflow/def.go index 4004765..b56160f 100644 --- a/workflow/def.go +++ b/workflow/def.go @@ -45,8 +45,9 @@ type ( } Step struct { - Name string `yaml:"name"` - Command string `yaml:"command"` + Name string `yaml:"name"` + Command string `yaml:"command"` + Environment map[string]string `yaml:"environment"` } StringList []string diff --git a/workflow/def_test.go b/workflow/def_test.go index 01661f7..d29ab67 100644 --- a/workflow/def_test.go +++ b/workflow/def_test.go @@ -105,6 +105,13 @@ clone: environment: HOME: /home/foo bar/baz CGO_ENABLED: 1 + +steps: + - name: Something + command: echo "hello" + environment: + FOO: bar + BAZ: qux ` wf, err := FromFile("test.yml", []byte(yamlData)) @@ -113,4 +120,6 @@ environment: assert.Len(t, wf.Environment, 2) assert.Equal(t, "/home/foo bar/baz", wf.Environment["HOME"]) assert.Equal(t, "1", wf.Environment["CGO_ENABLED"]) + assert.Equal(t, "bar", wf.Steps[0].Environment["FOO"]) + assert.Equal(t, "qux", wf.Steps[0].Environment["BAZ"]) } -- 2.43.0