pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 top = config.services.kubernetes;
7 cfg = top.pki;
8
9 csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
10 key = {
11 algo = "rsa";
12 size = 2048;
13 };
14 names = singleton cfg.caSpec;
15 });
16
17 csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON {
18 key = {
19 algo = "rsa";
20 size = 2048;
21 };
22 CN = top.masterAddress;
23 hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs;
24 });
25
26 cfsslAPITokenBaseName = "apitoken.secret";
27 cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
28 certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
29 cfsslAPITokenLength = 32;
30
31 clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
32 top.lib.mkKubeConfig "cluster-admin" {
33 server = top.apiserverAddress;
34 certFile = cert;
35 keyFile = key;
36 };
37
38 remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
39in
40{
41 ###### interface
42 options.services.kubernetes.pki = with lib.types; {
43
44 enable = mkEnableOption (lib.mdDoc "easyCert issuer service");
45
46 certs = mkOption {
47 description = lib.mdDoc "List of certificate specs to feed to cert generator.";
48 default = {};
49 type = attrs;
50 };
51
52 genCfsslCACert = mkOption {
53 description = lib.mdDoc ''
54 Whether to automatically generate cfssl CA certificate and key,
55 if they don't exist.
56 '';
57 default = true;
58 type = bool;
59 };
60
61 genCfsslAPICerts = mkOption {
62 description = lib.mdDoc ''
63 Whether to automatically generate cfssl API webserver TLS cert and key,
64 if they don't exist.
65 '';
66 default = true;
67 type = bool;
68 };
69
70 cfsslAPIExtraSANs = mkOption {
71 description = lib.mdDoc ''
72 Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
73 '';
74 default = [];
75 example = [ "subdomain.example.com" ];
76 type = listOf str;
77 };
78
79 genCfsslAPIToken = mkOption {
80 description = lib.mdDoc ''
81 Whether to automatically generate cfssl API-token secret,
82 if they doesn't exist.
83 '';
84 default = true;
85 type = bool;
86 };
87
88 pkiTrustOnBootstrap = mkOption {
89 description = lib.mdDoc "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
90 default = true;
91 type = bool;
92 };
93
94 caCertPathPrefix = mkOption {
95 description = lib.mdDoc ''
96 Path-prefrix for the CA-certificate to be used for cfssl signing.
97 Suffixes ".pem" and "-key.pem" will be automatically appended for
98 the public and private keys respectively.
99 '';
100 default = "${config.services.cfssl.dataDir}/ca";
101 defaultText = literalExpression ''"''${config.services.cfssl.dataDir}/ca"'';
102 type = str;
103 };
104
105 caSpec = mkOption {
106 description = lib.mdDoc "Certificate specification for the auto-generated CAcert.";
107 default = {
108 CN = "kubernetes-cluster-ca";
109 O = "NixOS";
110 OU = "services.kubernetes.pki.caSpec";
111 L = "auto-generated";
112 };
113 type = attrs;
114 };
115
116 etcClusterAdminKubeconfig = mkOption {
117 description = lib.mdDoc ''
118 Symlink a kubeconfig with cluster-admin privileges to environment path
119 (/etc/\<path\>).
120 '';
121 default = null;
122 type = nullOr str;
123 };
124
125 };
126
127 ###### implementation
128 config = mkIf cfg.enable
129 (let
130 cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
131 cfsslCert = "${cfsslCertPathPrefix}.pem";
132 cfsslKey = "${cfsslCertPathPrefix}-key.pem";
133 in
134 {
135
136 services.cfssl = mkIf (top.apiserver.enable) {
137 enable = true;
138 address = "0.0.0.0";
139 tlsCert = cfsslCert;
140 tlsKey = cfsslKey;
141 configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON {
142 signing = {
143 profiles = {
144 default = {
145 usages = ["digital signature"];
146 auth_key = "default";
147 expiry = "720h";
148 };
149 };
150 };
151 auth_keys = {
152 default = {
153 type = "standard";
154 key = "file:${cfsslAPITokenPath}";
155 };
156 };
157 }));
158 };
159
160 systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable)
161 (concatStringsSep "\n" [
162 "set -e"
163 (optionalString cfg.genCfsslCACert ''
164 if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
165 ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
166 ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
167 fi
168 '')
169 (optionalString cfg.genCfsslAPICerts ''
170 if [ ! -f "${dataDir}/cfssl.pem" ]; then
171 ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
172 ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
173 fi
174 '')
175 (optionalString cfg.genCfsslAPIToken ''
176 if [ ! -f "${cfsslAPITokenPath}" ]; then
177 head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}"
178 fi
179 chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
180 '')]);
181
182 systemd.services.kube-certmgr-bootstrap = {
183 description = "Kubernetes certmgr bootstrapper";
184 wantedBy = [ "certmgr.service" ];
185 after = [ "cfssl.target" ];
186 script = concatStringsSep "\n" [''
187 set -e
188
189 # If there's a cfssl (cert issuer) running locally, then don't rely on user to
190 # manually paste it in place. Just symlink.
191 # otherwise, create the target file, ready for users to insert the token
192
193 mkdir -p "$(dirname "${certmgrAPITokenPath}")"
194 if [ -f "${cfsslAPITokenPath}" ]; then
195 ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
196 else
197 touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}"
198 fi
199 ''
200 (optionalString (cfg.pkiTrustOnBootstrap) ''
201 if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
202 ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
203 ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
204 fi
205 '')
206 ];
207 serviceConfig = {
208 RestartSec = "10s";
209 Restart = "on-failure";
210 };
211 };
212
213 services.certmgr = {
214 enable = true;
215 package = pkgs.certmgr-selfsigned;
216 svcManager = "command";
217 specs =
218 let
219 mkSpec = _: cert: {
220 inherit (cert) action;
221 authority = {
222 inherit remote;
223 file.path = cert.caCert;
224 root_ca = cert.caCert;
225 profile = "default";
226 auth_key_file = certmgrAPITokenPath;
227 };
228 certificate = {
229 path = cert.cert;
230 };
231 private_key = cert.privateKeyOptions;
232 request = {
233 hosts = [cert.CN] ++ cert.hosts;
234 inherit (cert) CN;
235 key = {
236 algo = "rsa";
237 size = 2048;
238 };
239 names = [ cert.fields ];
240 };
241 };
242 in
243 mapAttrs mkSpec cfg.certs;
244 };
245
246 #TODO: Get rid of kube-addon-manager in the future for the following reasons
247 # - it is basically just a shell script wrapped around kubectl
248 # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
249 # - it is designed to be used with k8s system components only
250 # - it would be better with a more Nix-oriented way of managing addons
251 systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
252 environment.KUBECONFIG = with cfg.certs.addonManager;
253 top.lib.mkKubeConfig "addon-manager" {
254 server = top.apiserverAddress;
255 certFile = cert;
256 keyFile = key;
257 };
258 }
259
260 (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
261 serviceConfig.PermissionsStartOnly = true;
262 preStart = with pkgs;
263 let
264 files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
265 top.addonManager.bootstrapAddons;
266 in
267 ''
268 export KUBECONFIG=${clusterAdminKubeconfig}
269 ${top.package}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
270 '';
271 })]);
272
273 environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (cfg.etcClusterAdminKubeconfig != null)
274 clusterAdminKubeconfig;
275
276 environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
277 (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
278 set -e
279 exec 1>&2
280
281 if [ $# -gt 0 ]; then
282 echo "Usage: $(basename $0)"
283 echo ""
284 echo "No args. Apitoken must be provided on stdin."
285 echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
286 exit 1
287 fi
288
289 if [ $(id -u) != 0 ]; then
290 echo "Run as root please."
291 exit 1
292 fi
293
294 read -r token
295 if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
296 echo "Token must be of length ${toString cfsslAPITokenLength}."
297 exit 1
298 fi
299
300 echo $token > ${certmgrAPITokenPath}
301 chmod 600 ${certmgrAPITokenPath}
302
303 echo "Restarting certmgr..." >&1
304 systemctl restart certmgr
305
306 echo "Waiting for certs to appear..." >&1
307
308 ${optionalString top.kubelet.enable ''
309 while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
310 echo "Restarting kubelet..." >&1
311 systemctl restart kubelet
312 ''}
313
314 ${optionalString top.proxy.enable ''
315 while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
316 echo "Restarting kube-proxy..." >&1
317 systemctl restart kube-proxy
318 ''}
319
320 ${optionalString top.flannel.enable ''
321 while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
322 echo "Restarting flannel..." >&1
323 systemctl restart flannel
324 ''}
325
326 echo "Node joined successfully"
327 '')];
328
329 # isolate etcd on loopback at the master node
330 # easyCerts doesn't support multimaster clusters anyway atm.
331 services.etcd = with cfg.certs.etcd; {
332 listenClientUrls = ["https://127.0.0.1:2379"];
333 listenPeerUrls = ["https://127.0.0.1:2380"];
334 advertiseClientUrls = ["https://etcd.local:2379"];
335 initialCluster = ["${top.masterAddress}=https://etcd.local:2380"];
336 initialAdvertisePeerUrls = ["https://etcd.local:2380"];
337 certFile = mkDefault cert;
338 keyFile = mkDefault key;
339 trustedCaFile = mkDefault caCert;
340 };
341 networking.extraHosts = mkIf (config.services.etcd.enable) ''
342 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
343 '';
344
345 services.flannel = with cfg.certs.flannelClient; {
346 kubeconfig = top.lib.mkKubeConfig "flannel" {
347 server = top.apiserverAddress;
348 certFile = cert;
349 keyFile = key;
350 };
351 };
352
353 services.kubernetes = {
354
355 apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
356 etcd = with cfg.certs.apiserverEtcdClient; {
357 servers = ["https://etcd.local:2379"];
358 certFile = mkDefault cert;
359 keyFile = mkDefault key;
360 caFile = mkDefault caCert;
361 };
362 clientCaFile = mkDefault caCert;
363 tlsCertFile = mkDefault cert;
364 tlsKeyFile = mkDefault key;
365 serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
366 serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key;
367 kubeletClientCaFile = mkDefault caCert;
368 kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
369 kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
370 proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
371 proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
372 });
373 controllerManager = mkIf top.controllerManager.enable {
374 serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
375 rootCaFile = cfg.certs.controllerManagerClient.caCert;
376 kubeconfig = with cfg.certs.controllerManagerClient; {
377 certFile = mkDefault cert;
378 keyFile = mkDefault key;
379 };
380 };
381 scheduler = mkIf top.scheduler.enable {
382 kubeconfig = with cfg.certs.schedulerClient; {
383 certFile = mkDefault cert;
384 keyFile = mkDefault key;
385 };
386 };
387 kubelet = mkIf top.kubelet.enable {
388 clientCaFile = mkDefault cfg.certs.kubelet.caCert;
389 tlsCertFile = mkDefault cfg.certs.kubelet.cert;
390 tlsKeyFile = mkDefault cfg.certs.kubelet.key;
391 kubeconfig = with cfg.certs.kubeletClient; {
392 certFile = mkDefault cert;
393 keyFile = mkDefault key;
394 };
395 };
396 proxy = mkIf top.proxy.enable {
397 kubeconfig = with cfg.certs.kubeProxyClient; {
398 certFile = mkDefault cert;
399 keyFile = mkDefault key;
400 };
401 };
402 };
403 });
404
405 meta.buildDocsInSandbox = false;
406}