pyrox.dev
1{ config, lib, options, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.kubernetes;
7 opt = options.services.kubernetes;
8
9 defaultContainerdSettings = {
10 version = 2;
11 root = "/var/lib/containerd";
12 state = "/run/containerd";
13 oom_score = 0;
14
15 grpc = {
16 address = "/run/containerd/containerd.sock";
17 };
18
19 plugins."io.containerd.grpc.v1.cri" = {
20 sandbox_image = "pause:latest";
21
22 cni = {
23 bin_dir = "/opt/cni/bin";
24 max_conf_num = 0;
25 };
26
27 containerd.runtimes.runc = {
28 runtime_type = "io.containerd.runc.v2";
29 options.SystemdCgroup = true;
30 };
31 };
32 };
33
34 mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
35 apiVersion = "v1";
36 kind = "Config";
37 clusters = [{
38 name = "local";
39 cluster.certificate-authority = conf.caFile or cfg.caFile;
40 cluster.server = conf.server;
41 }];
42 users = [{
43 inherit name;
44 user = {
45 client-certificate = conf.certFile;
46 client-key = conf.keyFile;
47 };
48 }];
49 contexts = [{
50 context = {
51 cluster = "local";
52 user = name;
53 };
54 name = "local";
55 }];
56 current-context = "local";
57 });
58
59 caCert = secret "ca";
60
61 etcdEndpoints = ["https://${cfg.masterAddress}:2379"];
62
63 mkCert = { name, CN, hosts ? [], fields ? {}, action ? "",
64 privateKeyOwner ? "kubernetes" }: rec {
65 inherit name caCert CN hosts fields action;
66 cert = secret name;
67 key = secret "${name}-key";
68 privateKeyOptions = {
69 owner = privateKeyOwner;
70 group = "nogroup";
71 mode = "0600";
72 path = key;
73 };
74 };
75
76 secret = name: "${cfg.secretsPath}/${name}.pem";
77
78 mkKubeConfigOptions = prefix: {
79 server = mkOption {
80 description = lib.mdDoc "${prefix} kube-apiserver server address.";
81 type = types.str;
82 };
83
84 caFile = mkOption {
85 description = lib.mdDoc "${prefix} certificate authority file used to connect to kube-apiserver.";
86 type = types.nullOr types.path;
87 default = cfg.caFile;
88 defaultText = literalExpression "config.${opt.caFile}";
89 };
90
91 certFile = mkOption {
92 description = lib.mdDoc "${prefix} client certificate file used to connect to kube-apiserver.";
93 type = types.nullOr types.path;
94 default = null;
95 };
96
97 keyFile = mkOption {
98 description = lib.mdDoc "${prefix} client key file used to connect to kube-apiserver.";
99 type = types.nullOr types.path;
100 default = null;
101 };
102 };
103in {
104
105 imports = [
106 (mkRemovedOptionModule [ "services" "kubernetes" "addons" "dashboard" ] "Removed due to it being an outdated version")
107 (mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "")
108 ];
109
110 ###### interface
111
112 options.services.kubernetes = {
113 roles = mkOption {
114 description = lib.mdDoc ''
115 Kubernetes role that this machine should take.
116
117 Master role will enable etcd, apiserver, scheduler, controller manager
118 addon manager, flannel and proxy services.
119 Node role will enable flannel, docker, kubelet and proxy services.
120 '';
121 default = [];
122 type = types.listOf (types.enum ["master" "node"]);
123 };
124
125 package = mkOption {
126 description = lib.mdDoc "Kubernetes package to use.";
127 type = types.package;
128 default = pkgs.kubernetes;
129 defaultText = literalExpression "pkgs.kubernetes";
130 };
131
132 kubeconfig = mkKubeConfigOptions "Default kubeconfig";
133
134 apiserverAddress = mkOption {
135 description = lib.mdDoc ''
136 Clusterwide accessible address for the kubernetes apiserver,
137 including protocol and optional port.
138 '';
139 example = "https://kubernetes-apiserver.example.com:6443";
140 type = types.str;
141 };
142
143 caFile = mkOption {
144 description = lib.mdDoc "Default kubernetes certificate authority";
145 type = types.nullOr types.path;
146 default = null;
147 };
148
149 dataDir = mkOption {
150 description = lib.mdDoc "Kubernetes root directory for managing kubelet files.";
151 default = "/var/lib/kubernetes";
152 type = types.path;
153 };
154
155 easyCerts = mkOption {
156 description = lib.mdDoc "Automatically setup x509 certificates and keys for the entire cluster.";
157 default = false;
158 type = types.bool;
159 };
160
161 featureGates = mkOption {
162 description = lib.mdDoc "List set of feature gates.";
163 default = [];
164 type = types.listOf types.str;
165 };
166
167 masterAddress = mkOption {
168 description = lib.mdDoc "Clusterwide available network address or hostname for the kubernetes master server.";
169 example = "master.example.com";
170 type = types.str;
171 };
172
173 path = mkOption {
174 description = lib.mdDoc "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
175 type = types.listOf types.package;
176 default = [];
177 };
178
179 clusterCidr = mkOption {
180 description = lib.mdDoc "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
181 default = "10.1.0.0/16";
182 type = types.nullOr types.str;
183 };
184
185 lib = mkOption {
186 description = lib.mdDoc "Common functions for the kubernetes modules.";
187 default = {
188 inherit mkCert;
189 inherit mkKubeConfig;
190 inherit mkKubeConfigOptions;
191 };
192 type = types.attrs;
193 };
194
195 secretsPath = mkOption {
196 description = lib.mdDoc "Default location for kubernetes secrets. Not a store location.";
197 type = types.path;
198 default = cfg.dataDir + "/secrets";
199 defaultText = literalExpression ''
200 config.${opt.dataDir} + "/secrets"
201 '';
202 };
203 };
204
205 ###### implementation
206
207 config = mkMerge [
208
209 (mkIf cfg.easyCerts {
210 services.kubernetes.pki.enable = mkDefault true;
211 services.kubernetes.caFile = caCert;
212 })
213
214 (mkIf (elem "master" cfg.roles) {
215 services.kubernetes.apiserver.enable = mkDefault true;
216 services.kubernetes.scheduler.enable = mkDefault true;
217 services.kubernetes.controllerManager.enable = mkDefault true;
218 services.kubernetes.addonManager.enable = mkDefault true;
219 services.kubernetes.proxy.enable = mkDefault true;
220 services.etcd.enable = true; # Cannot mkDefault because of flannel default options
221 services.kubernetes.kubelet = {
222 enable = mkDefault true;
223 taints = mkIf (!(elem "node" cfg.roles)) {
224 master = {
225 key = "node-role.kubernetes.io/master";
226 value = "true";
227 effect = "NoSchedule";
228 };
229 };
230 };
231 })
232
233
234 (mkIf (all (el: el == "master") cfg.roles) {
235 # if this node is only a master make it unschedulable by default
236 services.kubernetes.kubelet.unschedulable = mkDefault true;
237 })
238
239 (mkIf (elem "node" cfg.roles) {
240 services.kubernetes.kubelet.enable = mkDefault true;
241 services.kubernetes.proxy.enable = mkDefault true;
242 })
243
244 # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel
245 (mkIf (cfg.roles != []) {
246 services.kubernetes.flannel.enable = mkDefault true;
247 services.flannel.etcd.endpoints = mkDefault etcdEndpoints;
248 services.kubernetes.easyCerts = mkDefault true;
249 })
250
251 (mkIf cfg.apiserver.enable {
252 services.kubernetes.pki.etcClusterAdminKubeconfig = mkDefault "kubernetes/cluster-admin.kubeconfig";
253 services.kubernetes.apiserver.etcd.servers = mkDefault etcdEndpoints;
254 })
255
256 (mkIf cfg.kubelet.enable {
257 virtualisation.containerd = {
258 enable = mkDefault true;
259 settings = mapAttrsRecursive (name: mkDefault) defaultContainerdSettings;
260 };
261 })
262
263 (mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) {
264 services.kubernetes.pki.certs = {
265 serviceAccount = mkCert {
266 name = "service-account";
267 CN = "system:service-account-signer";
268 action = ''
269 systemctl reload \
270 kube-apiserver.service \
271 kube-controller-manager.service
272 '';
273 };
274 };
275 })
276
277 (mkIf (
278 cfg.apiserver.enable ||
279 cfg.scheduler.enable ||
280 cfg.controllerManager.enable ||
281 cfg.kubelet.enable ||
282 cfg.proxy.enable ||
283 cfg.addonManager.enable
284 ) {
285 systemd.targets.kubernetes = {
286 description = "Kubernetes";
287 wantedBy = [ "multi-user.target" ];
288 };
289
290 systemd.tmpfiles.rules = [
291 "d /opt/cni/bin 0755 root root -"
292 "d /run/kubernetes 0755 kubernetes kubernetes -"
293 "d /var/lib/kubernetes 0755 kubernetes kubernetes -"
294 ];
295
296 users.users.kubernetes = {
297 uid = config.ids.uids.kubernetes;
298 description = "Kubernetes user";
299 group = "kubernetes";
300 home = cfg.dataDir;
301 createHome = true;
302 };
303 users.groups.kubernetes.gid = config.ids.gids.kubernetes;
304
305 # dns addon is enabled by default
306 services.kubernetes.addons.dns.enable = mkDefault true;
307
308 services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null
309 then cfg.apiserver.advertiseAddress
310 else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
311 })
312 ];
313
314 meta.buildDocsInSandbox = false;
315}