pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4
5let
6 top = config.services.kubernetes;
7 cfg = top.pki;
8
9 csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
10 key = {
11 algo = "rsa";
12 size = 2048;
13 };
14 names = singleton cfg.caSpec;
15 });
16
17 csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON {
18 key = {
19 algo = "rsa";
20 size = 2048;
21 };
22 CN = top.masterAddress;
23 hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs;
24 });
25
26 cfsslAPITokenBaseName = "apitoken.secret";
27 cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
28 certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
29 cfsslAPITokenLength = 32;
30
31 clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
32 top.lib.mkKubeConfig "cluster-admin" {
33 server = top.apiserverAddress;
34 certFile = cert;
35 keyFile = key;
36 };
37
38 remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
39in
40{
41 ###### interface
42 options.services.kubernetes.pki = with lib.types; {
43
44 enable = mkEnableOption "easyCert issuer service";
45
46 certs = mkOption {
47 description = "List of certificate specs to feed to cert generator.";
48 default = {};
49 type = attrs;
50 };
51
52 genCfsslCACert = mkOption {
53 description = ''
54 Whether to automatically generate cfssl CA certificate and key,
55 if they don't exist.
56 '';
57 default = true;
58 type = bool;
59 };
60
61 genCfsslAPICerts = mkOption {
62 description = ''
63 Whether to automatically generate cfssl API webserver TLS cert and key,
64 if they don't exist.
65 '';
66 default = true;
67 type = bool;
68 };
69
70 cfsslAPIExtraSANs = mkOption {
71 description = ''
72 Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
73 '';
74 default = [];
75 example = [ "subdomain.example.com" ];
76 type = listOf str;
77 };
78
79 genCfsslAPIToken = mkOption {
80 description = ''
81 Whether to automatically generate cfssl API-token secret,
82 if they doesn't exist.
83 '';
84 default = true;
85 type = bool;
86 };
87
88 pkiTrustOnBootstrap = mkOption {
89 description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
90 default = true;
91 type = bool;
92 };
93
94 caCertPathPrefix = mkOption {
95 description = ''
96 Path-prefrix for the CA-certificate to be used for cfssl signing.
97 Suffixes ".pem" and "-key.pem" will be automatically appended for
98 the public and private keys respectively.
99 '';
100 default = "${config.services.cfssl.dataDir}/ca";
101 defaultText = literalExpression ''"''${config.services.cfssl.dataDir}/ca"'';
102 type = str;
103 };
104
105 caSpec = mkOption {
106 description = "Certificate specification for the auto-generated CAcert.";
107 default = {
108 CN = "kubernetes-cluster-ca";
109 O = "NixOS";
110 OU = "services.kubernetes.pki.caSpec";
111 L = "auto-generated";
112 };
113 type = attrs;
114 };
115
116 etcClusterAdminKubeconfig = mkOption {
117 description = ''
118 Symlink a kubeconfig with cluster-admin privileges to environment path
119 (/etc/\<path\>).
120 '';
121 default = null;
122 type = nullOr str;
123 };
124
125 };
126
127 ###### implementation
128 config = mkIf cfg.enable
129 (let
130 cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
131 cfsslCert = "${cfsslCertPathPrefix}.pem";
132 cfsslKey = "${cfsslCertPathPrefix}-key.pem";
133 in
134 {
135
136 services.cfssl = mkIf (top.apiserver.enable) {
137 enable = true;
138 address = "0.0.0.0";
139 tlsCert = cfsslCert;
140 tlsKey = cfsslKey;
141 configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON {
142 signing = {
143 profiles = {
144 default = {
145 usages = ["digital signature"];
146 auth_key = "default";
147 expiry = "720h";
148 };
149 };
150 };
151 auth_keys = {
152 default = {
153 type = "standard";
154 key = "file:${cfsslAPITokenPath}";
155 };
156 };
157 }));
158 };
159
160 systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable)
161 (concatStringsSep "\n" [
162 "set -e"
163 (optionalString cfg.genCfsslCACert ''
164 if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
165 ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
166 ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
167 fi
168 '')
169 (optionalString cfg.genCfsslAPICerts ''
170 if [ ! -f "${dataDir}/cfssl.pem" ]; then
171 ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
172 ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
173 fi
174 '')
175 (optionalString cfg.genCfsslAPIToken ''
176 if [ ! -f "${cfsslAPITokenPath}" ]; then
177 install -o cfssl -m 400 <(head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ') "${cfsslAPITokenPath}"
178 fi
179 '')]);
180
181 systemd.services.kube-certmgr-bootstrap = {
182 description = "Kubernetes certmgr bootstrapper";
183 wantedBy = [ "certmgr.service" ];
184 after = [ "cfssl.target" ];
185 script = concatStringsSep "\n" [''
186 set -e
187
188 # If there's a cfssl (cert issuer) running locally, then don't rely on user to
189 # manually paste it in place. Just symlink.
190 # otherwise, create the target file, ready for users to insert the token
191
192 mkdir -p "$(dirname "${certmgrAPITokenPath}")"
193 if [ -f "${cfsslAPITokenPath}" ]; then
194 ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
195 elif [ ! -f "${certmgrAPITokenPath}" ]; then
196 # Don't remove the token if it already exists
197 install -m 600 /dev/null "${certmgrAPITokenPath}"
198 fi
199 ''
200 (optionalString (cfg.pkiTrustOnBootstrap) ''
201 if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
202 ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
203 ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
204 fi
205 '')
206 ];
207 serviceConfig = {
208 RestartSec = "10s";
209 Restart = "on-failure";
210 };
211 };
212
213 services.certmgr = {
214 enable = true;
215 package = pkgs.certmgr;
216 svcManager = "command";
217 specs =
218 let
219 mkSpec = _: cert: {
220 inherit (cert) action;
221 authority = {
222 inherit remote;
223 root_ca = cert.caCert;
224 profile = "default";
225 auth_key_file = certmgrAPITokenPath;
226 };
227 certificate = {
228 path = cert.cert;
229 };
230 private_key = cert.privateKeyOptions;
231 request = {
232 hosts = [cert.CN] ++ cert.hosts;
233 inherit (cert) CN;
234 key = {
235 algo = "rsa";
236 size = 2048;
237 };
238 names = [ cert.fields ];
239 };
240 };
241 in
242 mapAttrs mkSpec cfg.certs;
243 };
244
245 #TODO: Get rid of kube-addon-manager in the future for the following reasons
246 # - it is basically just a shell script wrapped around kubectl
247 # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
248 # - it is designed to be used with k8s system components only
249 # - it would be better with a more Nix-oriented way of managing addons
250 systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
251 environment.KUBECONFIG = with cfg.certs.addonManager;
252 top.lib.mkKubeConfig "addon-manager" {
253 server = top.apiserverAddress;
254 certFile = cert;
255 keyFile = key;
256 };
257 }
258
259 (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
260 serviceConfig.PermissionsStartOnly = true;
261 preStart = with pkgs;
262 let
263 files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
264 top.addonManager.bootstrapAddons;
265 in
266 ''
267 export KUBECONFIG=${clusterAdminKubeconfig}
268 ${top.package}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
269 '';
270 })]);
271
272 environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (cfg.etcClusterAdminKubeconfig != null)
273 clusterAdminKubeconfig;
274
275 environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
276 (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
277 set -e
278 exec 1>&2
279
280 if [ $# -gt 0 ]; then
281 echo "Usage: $(basename $0)"
282 echo ""
283 echo "No args. Apitoken must be provided on stdin."
284 echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
285 exit 1
286 fi
287
288 if [ $(id -u) != 0 ]; then
289 echo "Run as root please."
290 exit 1
291 fi
292
293 read -r token
294 if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
295 echo "Token must be of length ${toString cfsslAPITokenLength}."
296 exit 1
297 fi
298
299 install -m 0600 <(echo $token) ${certmgrAPITokenPath}
300
301 echo "Restarting certmgr..." >&1
302 systemctl restart certmgr
303
304 echo "Waiting for certs to appear..." >&1
305
306 ${optionalString top.kubelet.enable ''
307 while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
308 echo "Restarting kubelet..." >&1
309 systemctl restart kubelet
310 ''}
311
312 ${optionalString top.proxy.enable ''
313 while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
314 echo "Restarting kube-proxy..." >&1
315 systemctl restart kube-proxy
316 ''}
317
318 ${optionalString top.flannel.enable ''
319 while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
320 echo "Restarting flannel..." >&1
321 systemctl restart flannel
322 ''}
323
324 echo "Node joined successfully"
325 '')];
326
327 # isolate etcd on loopback at the master node
328 # easyCerts doesn't support multimaster clusters anyway atm.
329 services.etcd = with cfg.certs.etcd; {
330 listenClientUrls = ["https://127.0.0.1:2379"];
331 listenPeerUrls = ["https://127.0.0.1:2380"];
332 advertiseClientUrls = ["https://etcd.local:2379"];
333 initialCluster = ["${top.masterAddress}=https://etcd.local:2380"];
334 initialAdvertisePeerUrls = ["https://etcd.local:2380"];
335 certFile = mkDefault cert;
336 keyFile = mkDefault key;
337 trustedCaFile = mkDefault caCert;
338 };
339 networking.extraHosts = mkIf (config.services.etcd.enable) ''
340 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
341 '';
342
343 services.flannel = with cfg.certs.flannelClient; {
344 kubeconfig = top.lib.mkKubeConfig "flannel" {
345 server = top.apiserverAddress;
346 certFile = cert;
347 keyFile = key;
348 };
349 };
350
351 services.kubernetes = {
352
353 apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
354 etcd = with cfg.certs.apiserverEtcdClient; {
355 servers = ["https://etcd.local:2379"];
356 certFile = mkDefault cert;
357 keyFile = mkDefault key;
358 caFile = mkDefault caCert;
359 };
360 clientCaFile = mkDefault caCert;
361 tlsCertFile = mkDefault cert;
362 tlsKeyFile = mkDefault key;
363 serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
364 serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key;
365 kubeletClientCaFile = mkDefault caCert;
366 kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
367 kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
368 proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
369 proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
370 });
371 controllerManager = mkIf top.controllerManager.enable {
372 serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
373 rootCaFile = cfg.certs.controllerManagerClient.caCert;
374 kubeconfig = with cfg.certs.controllerManagerClient; {
375 certFile = mkDefault cert;
376 keyFile = mkDefault key;
377 };
378 };
379 scheduler = mkIf top.scheduler.enable {
380 kubeconfig = with cfg.certs.schedulerClient; {
381 certFile = mkDefault cert;
382 keyFile = mkDefault key;
383 };
384 };
385 kubelet = mkIf top.kubelet.enable {
386 clientCaFile = mkDefault cfg.certs.kubelet.caCert;
387 tlsCertFile = mkDefault cfg.certs.kubelet.cert;
388 tlsKeyFile = mkDefault cfg.certs.kubelet.key;
389 kubeconfig = with cfg.certs.kubeletClient; {
390 certFile = mkDefault cert;
391 keyFile = mkDefault key;
392 };
393 };
394 proxy = mkIf top.proxy.enable {
395 kubeconfig = with cfg.certs.kubeProxyClient; {
396 certFile = mkDefault cert;
397 keyFile = mkDefault key;
398 };
399 };
400 };
401 });
402
403 meta.buildDocsInSandbox = false;
404}