forked from tangled.org/core
Monorepo for Tangled — https://tangled.org

spindle: engine: consume from knotstream and execute

The engine package currently uses the docker client setup the pipeline
and execute steps. The flow is like so:

- setup pipeline network for all steps to join to
- create a volume for the nix store (to persist packages across steps)
- create a volume for the workspace directory
- build a nixery.dev URL with packages we want in the container
- execute each step command in a new container using the same image

It's pretty unfinished still. Things to be done:

- support for other registries; currently only works with nixpkgs
- custom nixery URL
- ... a lot more that I'm forgetting now

Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.sh>

anirudh.fi d95215cc baeaadbf

verified
Changed files
+422 -17
spindle
+21 -3
go.mod
···
github.com/casbin/casbin/v2 v2.103.0
github.com/cyphar/filepath-securejoin v0.4.1
github.com/dgraph-io/ristretto v0.2.0
+
github.com/docker/docker v28.2.2+incompatible
github.com/dustin/go-humanize v1.0.1
github.com/gliderlabs/ssh v0.3.8
github.com/go-chi/chi/v5 v5.2.0
···
github.com/mattn/go-sqlite3 v1.14.24
github.com/microcosm-cc/bluemonday v1.0.27
github.com/posthog/posthog-go v1.5.5
+
github.com/redis/go-redis/v9 v9.3.0
github.com/resend/resend-go/v2 v2.15.0
github.com/sethvargo/go-envconfig v1.1.0
github.com/stretchr/testify v1.10.0
···
github.com/whyrusleeping/cbor-gen v0.3.1
github.com/yuin/goldmark v1.4.13
golang.org/x/crypto v0.38.0
-
golang.org/x/net v0.39.0
+
golang.org/x/net v0.40.0
+
golang.org/x/sync v0.14.0
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
gopkg.in/yaml.v3 v3.0.1
tangled.sh/icyphox.sh/atproto-oauth v0.0.0-20250526154904-3906c5336421
···
github.com/casbin/govaluate v1.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudflare/circl v1.6.0 // indirect
+
github.com/containerd/errdefs v1.0.0 // indirect
+
github.com/containerd/errdefs/pkg v0.3.0 // indirect
+
github.com/containerd/log v0.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.5 // indirect
+
github.com/docker/go-connections v0.5.0 // indirect
+
github.com/docker/go-units v0.5.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-enry/go-oniguruma v1.2.1 // indirect
···
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
+
github.com/moby/docker-image-spec v1.3.1 // indirect
+
github.com/moby/sys/atomicwriter v0.1.0 // indirect
+
github.com/moby/term v0.5.2 // indirect
+
github.com/morikuni/aec v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
···
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+
github.com/opencontainers/go-digest v1.0.0 // indirect
+
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pjbgf/sha1cd v0.3.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
···
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.63.0 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
-
github.com/redis/go-redis/v9 v9.3.0 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
···
go.opentelemetry.io/otel v1.36.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
+
go.opentelemetry.io/proto/otlp v1.6.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
-
golang.org/x/sync v0.13.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/time v0.8.0 // indirect
+
google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect
+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect
+
google.golang.org/grpc v1.72.1 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
+
gotest.tools/v3 v3.5.2 // indirect
lukechampine.com/blake3 v1.4.1 // indirect
)
+54 -4
go.sum
···
dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
+
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Blank-Xu/sql-adapter v1.1.1 h1:+g7QXU9sl/qT6Po97teMpf3GjAO0X9aFaqgSePXvYko=
github.com/Blank-Xu/sql-adapter v1.1.1/go.mod h1:o2g8EZhZ3TudnYEGDkoU+3jCTCgDgx1o/Ig5ajKkaLY=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
···
github.com/casbin/govaluate v1.2.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc=
github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
+
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
···
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk=
github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
+
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
···
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+
github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw=
+
github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
···
github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=
+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
···
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
+
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
+
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
+
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
+
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
+
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
···
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
+
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/oppiliappan/chroma/v2 v2.19.0 h1:PN7/pb+6JRKCva30NPTtRJMlrOyzgpPpIroNzy4ekHU=
···
github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog=
github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
···
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk=
+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME=
+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU=
+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4=
go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
···
go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
+
go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI=
+
go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
···
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
-
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
-
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
+
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
+
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
···
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
+
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
···
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+
google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 h1:Kog3KlB4xevJlAcbbbzPfRG0+X9fdoGM+UBRKVz6Wr0=
+
google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237/go.mod h1:ezi0AVyMKDWy5xAncvjLWH7UcLBB5n7y2fQ8MzjJcto=
+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34=
+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+
google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA=
+
google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
···
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg=
lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo=
+263
spindle/engine/engine.go
···
+
package engine
+
+
import (
+
"context"
+
"fmt"
+
"io"
+
"log/slog"
+
"os"
+
"path"
+
"sync"
+
+
"github.com/docker/docker/api/types/container"
+
"github.com/docker/docker/api/types/image"
+
"github.com/docker/docker/api/types/mount"
+
"github.com/docker/docker/api/types/network"
+
"github.com/docker/docker/api/types/volume"
+
"github.com/docker/docker/client"
+
"github.com/docker/docker/pkg/stdcopy"
+
"golang.org/x/sync/errgroup"
+
"tangled.sh/tangled.sh/core/api/tangled"
+
"tangled.sh/tangled.sh/core/log"
+
"tangled.sh/tangled.sh/core/spindle/db"
+
)
+
+
const (
+
workspaceDir = "/tangled/workspace"
+
)
+
+
type Engine struct {
+
docker client.APIClient
+
l *slog.Logger
+
db *db.DB
+
}
+
+
func New(ctx context.Context, db *db.DB) (*Engine, error) {
+
dcli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+
if err != nil {
+
return nil, err
+
}
+
+
l := log.FromContext(ctx).With("component", "spindle")
+
+
return &Engine{docker: dcli, l: l, db: db}, nil
+
}
+
+
// SetupPipeline sets up a new network for the pipeline, and possibly volumes etc.
+
// in the future. In here also goes other setup steps.
+
func (e *Engine) SetupPipeline(ctx context.Context, pipeline *tangled.Pipeline, id string) error {
+
e.l.Info("setting up pipeline", "pipeline", id)
+
+
_, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{
+
Name: workspaceVolume(id),
+
Driver: "local",
+
})
+
if err != nil {
+
return err
+
}
+
+
_, err = e.docker.VolumeCreate(ctx, volume.CreateOptions{
+
Name: nixVolume(id),
+
Driver: "local",
+
})
+
if err != nil {
+
return err
+
}
+
+
_, err = e.docker.NetworkCreate(ctx, pipelineName(id), network.CreateOptions{
+
Driver: "bridge",
+
})
+
if err != nil {
+
return err
+
}
+
+
err = e.db.CreatePipeline(id)
+
return err
+
}
+
+
func (e *Engine) StartWorkflows(ctx context.Context, pipeline *tangled.Pipeline, id string) error {
+
e.l.Info("starting all workflows in parallel", "pipeline", id)
+
+
err := e.db.MarkPipelineRunning(id)
+
if err != nil {
+
return err
+
}
+
+
g := errgroup.Group{}
+
for _, w := range pipeline.Workflows {
+
g.Go(func() error {
+
// TODO: actual checks for image/registry etc.
+
var deps string
+
for _, d := range w.Dependencies {
+
if d.Registry == "nixpkgs" {
+
deps = path.Join(d.Packages...)
+
}
+
}
+
+
// load defaults from somewhere else
+
deps = path.Join(deps, "bash", "git", "coreutils", "nix")
+
+
cimg := path.Join("nixery.dev", deps)
+
reader, err := e.docker.ImagePull(ctx, cimg, image.PullOptions{})
+
if err != nil {
+
e.l.Error("pipeline failed!", "id", id, "error", err.Error())
+
err := e.db.MarkPipelineFailed(id, -1, err.Error())
+
if err != nil {
+
return err
+
}
+
return fmt.Errorf("pulling image: %w", err)
+
}
+
defer reader.Close()
+
io.Copy(os.Stdout, reader)
+
+
err = e.StartSteps(ctx, w.Steps, id, cimg)
+
if err != nil {
+
e.l.Error("pipeline failed!", "id", id, "error", err.Error())
+
return e.db.MarkPipelineFailed(id, -1, err.Error())
+
}
+
+
return nil
+
})
+
}
+
+
err = g.Wait()
+
if err != nil {
+
e.l.Error("pipeline failed!", "id", id, "error", err.Error())
+
return e.db.MarkPipelineFailed(id, -1, err.Error())
+
}
+
+
e.l.Info("pipeline success!", "id", id)
+
return e.db.MarkPipelineSuccess(id)
+
}
+
+
// StartSteps starts all steps sequentially with the same base image.
+
// ONLY marks pipeline as failed if container's exit code is non-zero.
+
// All other errors are bubbled up.
+
func (e *Engine) StartSteps(ctx context.Context, steps []*tangled.Pipeline_Step, id, image string) error {
+
for _, step := range steps {
+
hostConfig := hostConfig(id)
+
resp, err := e.docker.ContainerCreate(ctx, &container.Config{
+
Image: image,
+
Cmd: []string{"bash", "-c", step.Command},
+
WorkingDir: workspaceDir,
+
Tty: false,
+
Hostname: "spindle",
+
Env: []string{"HOME=" + workspaceDir},
+
}, hostConfig, nil, nil, "")
+
if err != nil {
+
return fmt.Errorf("creating container: %w", err)
+
}
+
+
err = e.docker.NetworkConnect(ctx, pipelineName(id), resp.ID, nil)
+
if err != nil {
+
return fmt.Errorf("connecting network: %w", err)
+
}
+
+
err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{})
+
if err != nil {
+
return err
+
}
+
e.l.Info("started container", "name", resp.ID, "step", step.Name)
+
+
wg := sync.WaitGroup{}
+
+
wg.Add(1)
+
go func() {
+
defer wg.Done()
+
err := e.TailStep(ctx, resp.ID)
+
if err != nil {
+
e.l.Error("failed to tail container", "container", resp.ID)
+
return
+
}
+
}()
+
+
// wait until all logs are piped
+
wg.Wait()
+
+
state, err := e.WaitStep(ctx, resp.ID)
+
if err != nil {
+
return err
+
}
+
+
if state.ExitCode != 0 {
+
e.l.Error("pipeline failed!", "id", id, "error", state.Error, "exit_code", state.ExitCode)
+
return e.db.MarkPipelineFailed(id, state.ExitCode, state.Error)
+
}
+
}
+
+
return nil
+
+
}
+
+
func (e *Engine) WaitStep(ctx context.Context, containerID string) (*container.State, error) {
+
wait, errCh := e.docker.ContainerWait(ctx, containerID, container.WaitConditionNotRunning)
+
select {
+
case err := <-errCh:
+
if err != nil {
+
return nil, err
+
}
+
case <-wait:
+
}
+
+
e.l.Info("waited for container", "name", containerID)
+
+
info, err := e.docker.ContainerInspect(ctx, containerID)
+
if err != nil {
+
return nil, err
+
}
+
+
return info.State, nil
+
}
+
+
func (e *Engine) TailStep(ctx context.Context, containerID string) error {
+
logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{
+
Follow: true,
+
ShowStdout: true,
+
ShowStderr: true,
+
Details: false,
+
Timestamps: false,
+
})
+
if err != nil {
+
return err
+
}
+
+
go func() {
+
_, _ = stdcopy.StdCopy(os.Stdout, os.Stdout, logs)
+
_ = logs.Close()
+
}()
+
return nil
+
}
+
+
func workspaceVolume(id string) string {
+
return "workspace-" + id
+
}
+
+
func nixVolume(id string) string {
+
return "nix-" + id
+
}
+
+
func pipelineName(id string) string {
+
return "pipeline-" + id
+
}
+
+
func hostConfig(id string) *container.HostConfig {
+
hostConfig := &container.HostConfig{
+
Mounts: []mount.Mount{
+
{
+
Type: mount.TypeVolume,
+
Source: workspaceVolume(id),
+
Target: workspaceDir,
+
},
+
{
+
Type: mount.TypeVolume,
+
Source: nixVolume(id),
+
Target: "/nix",
+
},
+
},
+
ReadonlyRootfs: true,
+
CapDrop: []string{"ALL"},
+
SecurityOpt: []string{"no-new-privileges"},
+
}
+
+
return hostConfig
+
}
+54
spindle/exec.go
···
+
package spindle
+
+
import (
+
"context"
+
"encoding/json"
+
"fmt"
+
+
"tangled.sh/tangled.sh/core/api/tangled"
+
)
+
+
func (s *Spindle) exec(ctx context.Context, src string, msg []byte) error {
+
pipeline := tangled.Pipeline{}
+
data := map[string]any{}
+
err := json.Unmarshal(msg, &data)
+
if err != nil {
+
fmt.Println("error unmarshalling", err)
+
return err
+
}
+
+
if data["nsid"] == tangled.PipelineNSID {
+
event, ok := data["event"]
+
if !ok {
+
s.l.Error("no event in message")
+
return nil
+
}
+
+
rawEvent, err := json.Marshal(event)
+
if err != nil {
+
return err
+
}
+
+
err = json.Unmarshal(rawEvent, &pipeline)
+
if err != nil {
+
return err
+
}
+
+
rkey, ok := data["rkey"].(string)
+
if !ok {
+
s.l.Error("no rkey in message")
+
return nil
+
}
+
+
err = s.eng.SetupPipeline(ctx, &pipeline, rkey)
+
if err != nil {
+
return err
+
}
+
err = s.eng.StartWorkflows(ctx, &pipeline, rkey)
+
if err != nil {
+
return err
+
}
+
}
+
+
return nil
+
}
+30 -10
spindle/server.go
···
"golang.org/x/net/context"
"tangled.sh/tangled.sh/core/api/tangled"
"tangled.sh/tangled.sh/core/jetstream"
+
"tangled.sh/tangled.sh/core/knotclient"
"tangled.sh/tangled.sh/core/knotserver/notifier"
"tangled.sh/tangled.sh/core/log"
"tangled.sh/tangled.sh/core/rbac"
"tangled.sh/tangled.sh/core/spindle/config"
"tangled.sh/tangled.sh/core/spindle/db"
+
"tangled.sh/tangled.sh/core/spindle/engine"
)
type Spindle struct {
-
jc *jetstream.JetstreamClient
-
db *db.DB
-
e *rbac.Enforcer
-
l *slog.Logger
-
n *notifier.Notifier
+
jc *jetstream.JetstreamClient
+
db *db.DB
+
e *rbac.Enforcer
+
l *slog.Logger
+
n *notifier.Notifier
+
eng *engine.Engine
}
func Run(ctx context.Context) error {
···
n := notifier.New()
+
eng, err := engine.New(ctx, d)
+
if err != nil {
+
return err
+
}
+
spindle := Spindle{
-
jc: jc,
-
e: e,
-
db: d,
-
l: logger,
-
n: &n,
+
jc: jc,
+
e: e,
+
db: d,
+
l: logger,
+
n: &n,
+
eng: eng,
}
+
+
go func() {
+
logger.Info("starting event consumer")
+
ec := knotclient.NewEventConsumer(knotclient.ConsumerConfig{
+
Sources: []string{"ws://localhost:5555/events"},
+
Logger: logger,
+
ProcessFunc: spindle.exec,
+
})
+
+
ec.Start(ctx)
+
}()
logger.Info("starting spindle server", "address", cfg.Server.ListenAddr)
logger.Error("server error", "error", http.ListenAndServe(cfg.Server.ListenAddr, spindle.Router()))