pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7
8with lib;
9
10let
11 top = config.services.kubernetes;
12 cfg = top.pki;
13
14 csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (
15 builtins.toJSON {
16 key = {
17 algo = "rsa";
18 size = 2048;
19 };
20 names = singleton cfg.caSpec;
21 }
22 );
23
24 csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (
25 builtins.toJSON {
26 key = {
27 algo = "rsa";
28 size = 2048;
29 };
30 CN = top.masterAddress;
31 hosts = [ top.masterAddress ] ++ cfg.cfsslAPIExtraSANs;
32 }
33 );
34
35 cfsslAPITokenBaseName = "apitoken.secret";
36 cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
37 certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
38 cfsslAPITokenLength = 32;
39
40 clusterAdminKubeconfig =
41 with cfg.certs.clusterAdmin;
42 top.lib.mkKubeConfig "cluster-admin" {
43 server = top.apiserverAddress;
44 certFile = cert;
45 keyFile = key;
46 };
47
48 remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
49in
50{
51 ###### interface
52 options.services.kubernetes.pki = with lib.types; {
53
54 enable = mkEnableOption "easyCert issuer service";
55
56 certs = mkOption {
57 description = "List of certificate specs to feed to cert generator.";
58 default = { };
59 type = attrs;
60 };
61
62 genCfsslCACert = mkOption {
63 description = ''
64 Whether to automatically generate cfssl CA certificate and key,
65 if they don't exist.
66 '';
67 default = true;
68 type = bool;
69 };
70
71 genCfsslAPICerts = mkOption {
72 description = ''
73 Whether to automatically generate cfssl API webserver TLS cert and key,
74 if they don't exist.
75 '';
76 default = true;
77 type = bool;
78 };
79
80 cfsslAPIExtraSANs = mkOption {
81 description = ''
82 Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
83 '';
84 default = [ ];
85 example = [ "subdomain.example.com" ];
86 type = listOf str;
87 };
88
89 genCfsslAPIToken = mkOption {
90 description = ''
91 Whether to automatically generate cfssl API-token secret,
92 if they doesn't exist.
93 '';
94 default = true;
95 type = bool;
96 };
97
98 pkiTrustOnBootstrap = mkOption {
99 description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
100 default = true;
101 type = bool;
102 };
103
104 caCertPathPrefix = mkOption {
105 description = ''
106 Path-prefrix for the CA-certificate to be used for cfssl signing.
107 Suffixes ".pem" and "-key.pem" will be automatically appended for
108 the public and private keys respectively.
109 '';
110 default = "${config.services.cfssl.dataDir}/ca";
111 defaultText = literalExpression ''"''${config.services.cfssl.dataDir}/ca"'';
112 type = str;
113 };
114
115 caSpec = mkOption {
116 description = "Certificate specification for the auto-generated CAcert.";
117 default = {
118 CN = "kubernetes-cluster-ca";
119 O = "NixOS";
120 OU = "services.kubernetes.pki.caSpec";
121 L = "auto-generated";
122 };
123 type = attrs;
124 };
125
126 etcClusterAdminKubeconfig = mkOption {
127 description = ''
128 Symlink a kubeconfig with cluster-admin privileges to environment path
129 (/etc/\<path\>).
130 '';
131 default = null;
132 type = nullOr str;
133 };
134
135 };
136
137 ###### implementation
138 config = mkIf cfg.enable (
139 let
140 cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
141 cfsslCert = "${cfsslCertPathPrefix}.pem";
142 cfsslKey = "${cfsslCertPathPrefix}-key.pem";
143 in
144 {
145
146 services.cfssl = mkIf (top.apiserver.enable) {
147 enable = true;
148 address = "0.0.0.0";
149 tlsCert = cfsslCert;
150 tlsKey = cfsslKey;
151 configFile = toString (
152 pkgs.writeText "cfssl-config.json" (
153 builtins.toJSON {
154 signing = {
155 profiles = {
156 default = {
157 usages = [ "digital signature" ];
158 auth_key = "default";
159 expiry = "720h";
160 };
161 };
162 };
163 auth_keys = {
164 default = {
165 type = "standard";
166 key = "file:${cfsslAPITokenPath}";
167 };
168 };
169 }
170 )
171 );
172 };
173
174 systemd.services.cfssl.preStart =
175 with pkgs;
176 with config.services.cfssl;
177 mkIf (top.apiserver.enable) (
178 concatStringsSep "\n" [
179 "set -e"
180 (optionalString cfg.genCfsslCACert ''
181 if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
182 ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
183 ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
184 fi
185 '')
186 (optionalString cfg.genCfsslAPICerts ''
187 if [ ! -f "${dataDir}/cfssl.pem" ]; then
188 ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
189 ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
190 fi
191 '')
192 (optionalString cfg.genCfsslAPIToken ''
193 if [ ! -f "${cfsslAPITokenPath}" ]; then
194 install -o cfssl -m 400 <(head -c ${
195 toString (cfsslAPITokenLength / 2)
196 } /dev/urandom | od -An -t x | tr -d ' ') "${cfsslAPITokenPath}"
197 fi
198 '')
199 ]
200 );
201
202 systemd.services.kube-certmgr-bootstrap = {
203 description = "Kubernetes certmgr bootstrapper";
204 wantedBy = [ "certmgr.service" ];
205 after = [ "cfssl.target" ];
206 script = concatStringsSep "\n" [
207 ''
208 set -e
209
210 # If there's a cfssl (cert issuer) running locally, then don't rely on user to
211 # manually paste it in place. Just symlink.
212 # otherwise, create the target file, ready for users to insert the token
213
214 mkdir -p "$(dirname "${certmgrAPITokenPath}")"
215 if [ -f "${cfsslAPITokenPath}" ]; then
216 ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
217 elif [ ! -f "${certmgrAPITokenPath}" ]; then
218 # Don't remove the token if it already exists
219 install -m 600 /dev/null "${certmgrAPITokenPath}"
220 fi
221 ''
222 (optionalString (cfg.pkiTrustOnBootstrap) ''
223 if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
224 ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
225 ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
226 fi
227 '')
228 ];
229 serviceConfig = {
230 RestartSec = "10s";
231 Restart = "on-failure";
232 };
233 };
234
235 services.certmgr = {
236 enable = true;
237 package = pkgs.certmgr;
238 svcManager = "command";
239 specs =
240 let
241 mkSpec = _: cert: {
242 inherit (cert) action;
243 authority = {
244 inherit remote;
245 root_ca = cert.caCert;
246 profile = "default";
247 auth_key_file = certmgrAPITokenPath;
248 };
249 certificate = {
250 path = cert.cert;
251 };
252 private_key = cert.privateKeyOptions;
253 request = {
254 hosts = [ cert.CN ] ++ cert.hosts;
255 inherit (cert) CN;
256 key = {
257 algo = "rsa";
258 size = 2048;
259 };
260 names = [ cert.fields ];
261 };
262 };
263 in
264 mapAttrs mkSpec cfg.certs;
265 };
266
267 #TODO: Get rid of kube-addon-manager in the future for the following reasons
268 # - it is basically just a shell script wrapped around kubectl
269 # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
270 # - it is designed to be used with k8s system components only
271 # - it would be better with a more Nix-oriented way of managing addons
272 systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [
273 {
274 environment.KUBECONFIG =
275 with cfg.certs.addonManager;
276 top.lib.mkKubeConfig "addon-manager" {
277 server = top.apiserverAddress;
278 certFile = cert;
279 keyFile = key;
280 };
281 }
282
283 (optionalAttrs (top.addonManager.bootstrapAddons != { }) {
284 serviceConfig.PermissionsStartOnly = true;
285 preStart =
286 with pkgs;
287 let
288 files = mapAttrsToList (
289 n: v: writeText "${n}.json" (builtins.toJSON v)
290 ) top.addonManager.bootstrapAddons;
291 in
292 ''
293 export KUBECONFIG=${clusterAdminKubeconfig}
294 ${top.package}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
295 '';
296 })
297 ]);
298
299 environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (
300 cfg.etcClusterAdminKubeconfig != null
301 ) clusterAdminKubeconfig;
302
303 environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
304 (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
305 set -e
306 exec 1>&2
307
308 if [ $# -gt 0 ]; then
309 echo "Usage: $(basename $0)"
310 echo ""
311 echo "No args. Apitoken must be provided on stdin."
312 echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
313 exit 1
314 fi
315
316 if [ $(id -u) != 0 ]; then
317 echo "Run as root please."
318 exit 1
319 fi
320
321 read -r token
322 if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
323 echo "Token must be of length ${toString cfsslAPITokenLength}."
324 exit 1
325 fi
326
327 install -m 0600 <(echo $token) ${certmgrAPITokenPath}
328
329 echo "Restarting certmgr..." >&1
330 systemctl restart certmgr
331
332 echo "Waiting for certs to appear..." >&1
333
334 ${optionalString top.kubelet.enable ''
335 while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
336 echo "Restarting kubelet..." >&1
337 systemctl restart kubelet
338 ''}
339
340 ${optionalString top.proxy.enable ''
341 while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
342 echo "Restarting kube-proxy..." >&1
343 systemctl restart kube-proxy
344 ''}
345
346 ${optionalString top.flannel.enable ''
347 while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
348 echo "Restarting flannel..." >&1
349 systemctl restart flannel
350 ''}
351
352 echo "Node joined successfully"
353 '')
354 ];
355
356 # isolate etcd on loopback at the master node
357 # easyCerts doesn't support multimaster clusters anyway atm.
358 services.etcd = with cfg.certs.etcd; {
359 listenClientUrls = [ "https://127.0.0.1:2379" ];
360 listenPeerUrls = [ "https://127.0.0.1:2380" ];
361 advertiseClientUrls = [ "https://etcd.local:2379" ];
362 initialCluster = [ "${top.masterAddress}=https://etcd.local:2380" ];
363 initialAdvertisePeerUrls = [ "https://etcd.local:2380" ];
364 certFile = mkDefault cert;
365 keyFile = mkDefault key;
366 trustedCaFile = mkDefault caCert;
367 };
368 networking.extraHosts = mkIf (config.services.etcd.enable) ''
369 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
370 '';
371
372 services.flannel = with cfg.certs.flannelClient; {
373 kubeconfig = top.lib.mkKubeConfig "flannel" {
374 server = top.apiserverAddress;
375 certFile = cert;
376 keyFile = key;
377 };
378 };
379
380 services.kubernetes = {
381
382 apiserver = mkIf top.apiserver.enable (
383 with cfg.certs.apiServer;
384 {
385 etcd = with cfg.certs.apiserverEtcdClient; {
386 servers = [ "https://etcd.local:2379" ];
387 certFile = mkDefault cert;
388 keyFile = mkDefault key;
389 caFile = mkDefault caCert;
390 };
391 clientCaFile = mkDefault caCert;
392 tlsCertFile = mkDefault cert;
393 tlsKeyFile = mkDefault key;
394 serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
395 serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key;
396 kubeletClientCaFile = mkDefault caCert;
397 kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
398 kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
399 proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
400 proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
401 }
402 );
403 controllerManager = mkIf top.controllerManager.enable {
404 serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
405 rootCaFile = cfg.certs.controllerManagerClient.caCert;
406 kubeconfig = with cfg.certs.controllerManagerClient; {
407 certFile = mkDefault cert;
408 keyFile = mkDefault key;
409 };
410 };
411 scheduler = mkIf top.scheduler.enable {
412 kubeconfig = with cfg.certs.schedulerClient; {
413 certFile = mkDefault cert;
414 keyFile = mkDefault key;
415 };
416 };
417 kubelet = mkIf top.kubelet.enable {
418 clientCaFile = mkDefault cfg.certs.kubelet.caCert;
419 tlsCertFile = mkDefault cfg.certs.kubelet.cert;
420 tlsKeyFile = mkDefault cfg.certs.kubelet.key;
421 kubeconfig = with cfg.certs.kubeletClient; {
422 certFile = mkDefault cert;
423 keyFile = mkDefault key;
424 };
425 };
426 proxy = mkIf top.proxy.enable {
427 kubeconfig = with cfg.certs.kubeProxyClient; {
428 certFile = mkDefault cert;
429 keyFile = mkDefault key;
430 };
431 };
432 };
433 }
434 );
435
436 meta.buildDocsInSandbox = false;
437}