pyrox.dev
1{
2 config,
3 lib,
4 options,
5 pkgs,
6 ...
7}:
8let
9 cfg = config.services.kubernetes;
10 opt = options.services.kubernetes;
11
12 defaultContainerdSettings = {
13 version = 2;
14 root = "/var/lib/containerd";
15 state = "/run/containerd";
16 oom_score = 0;
17
18 grpc = {
19 address = "/run/containerd/containerd.sock";
20 };
21
22 plugins."io.containerd.grpc.v1.cri" = {
23 sandbox_image = "pause:latest";
24
25 cni = {
26 bin_dir = "/opt/cni/bin";
27 max_conf_num = 0;
28 };
29
30 containerd.runtimes.runc = {
31 runtime_type = "io.containerd.runc.v2";
32 options.SystemdCgroup = true;
33 };
34 };
35 };
36
37 mkKubeConfig =
38 name: conf:
39 pkgs.writeText "${name}-kubeconfig" (
40 builtins.toJSON {
41 apiVersion = "v1";
42 kind = "Config";
43 clusters = [
44 {
45 name = "local";
46 cluster.certificate-authority = conf.caFile or cfg.caFile;
47 cluster.server = conf.server;
48 }
49 ];
50 users = [
51 {
52 inherit name;
53 user = {
54 client-certificate = conf.certFile;
55 client-key = conf.keyFile;
56 };
57 }
58 ];
59 contexts = [
60 {
61 context = {
62 cluster = "local";
63 user = name;
64 };
65 name = "local";
66 }
67 ];
68 current-context = "local";
69 }
70 );
71
72 caCert = secret "ca";
73
74 etcdEndpoints = [ "https://${cfg.masterAddress}:2379" ];
75
76 mkCert =
77 {
78 name,
79 CN,
80 hosts ? [ ],
81 fields ? { },
82 action ? "",
83 privateKeyOwner ? "kubernetes",
84 privateKeyGroup ? "kubernetes",
85 }:
86 rec {
87 inherit
88 name
89 caCert
90 CN
91 hosts
92 fields
93 action
94 ;
95 cert = secret name;
96 key = secret "${name}-key";
97 privateKeyOptions = {
98 owner = privateKeyOwner;
99 group = privateKeyGroup;
100 mode = "0600";
101 path = key;
102 };
103 };
104
105 secret = name: "${cfg.secretsPath}/${name}.pem";
106
107 mkKubeConfigOptions = prefix: {
108 server = lib.mkOption {
109 description = "${prefix} kube-apiserver server address.";
110 type = lib.types.str;
111 };
112
113 caFile = lib.mkOption {
114 description = "${prefix} certificate authority file used to connect to kube-apiserver.";
115 type = lib.types.nullOr lib.types.path;
116 default = cfg.caFile;
117 defaultText = lib.literalExpression "config.${opt.caFile}";
118 };
119
120 certFile = lib.mkOption {
121 description = "${prefix} client certificate file used to connect to kube-apiserver.";
122 type = lib.types.nullOr lib.types.path;
123 default = null;
124 };
125
126 keyFile = lib.mkOption {
127 description = "${prefix} client key file used to connect to kube-apiserver.";
128 type = lib.types.nullOr lib.types.path;
129 default = null;
130 };
131 };
132in
133{
134
135 imports = [
136 (lib.mkRemovedOptionModule [
137 "services"
138 "kubernetes"
139 "addons"
140 "dashboard"
141 ] "Removed due to it being an outdated version")
142 (lib.mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "")
143 ];
144
145 ###### interface
146
147 options.services.kubernetes = {
148 roles = lib.mkOption {
149 description = ''
150 Kubernetes role that this machine should take.
151
152 Master role will enable etcd, apiserver, scheduler, controller manager
153 addon manager, flannel and proxy services.
154 Node role will enable flannel, docker, kubelet and proxy services.
155 '';
156 default = [ ];
157 type = lib.types.listOf (
158 lib.types.enum [
159 "master"
160 "node"
161 ]
162 );
163 };
164
165 package = lib.mkPackageOption pkgs "kubernetes" { };
166
167 kubeconfig = mkKubeConfigOptions "Default kubeconfig";
168
169 apiserverAddress = lib.mkOption {
170 description = ''
171 Clusterwide accessible address for the kubernetes apiserver,
172 including protocol and optional port.
173 '';
174 example = "https://kubernetes-apiserver.example.com:6443";
175 type = lib.types.str;
176 };
177
178 caFile = lib.mkOption {
179 description = "Default kubernetes certificate authority";
180 type = lib.types.nullOr lib.types.path;
181 default = null;
182 };
183
184 dataDir = lib.mkOption {
185 description = "Kubernetes root directory for managing kubelet files.";
186 default = "/var/lib/kubernetes";
187 type = lib.types.path;
188 };
189
190 easyCerts = lib.mkOption {
191 description = "Automatically setup x509 certificates and keys for the entire cluster.";
192 default = false;
193 type = lib.types.bool;
194 };
195
196 featureGates = lib.mkOption {
197 description = "List set of feature gates.";
198 default = { };
199 type = lib.types.attrsOf lib.types.bool;
200 };
201
202 masterAddress = lib.mkOption {
203 description = "Clusterwide available network address or hostname for the kubernetes master server.";
204 example = "master.example.com";
205 type = lib.types.str;
206 };
207
208 path = lib.mkOption {
209 description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
210 type = lib.types.listOf lib.types.package;
211 default = [ ];
212 };
213
214 clusterCidr = lib.mkOption {
215 description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
216 default = "10.1.0.0/16";
217 type = lib.types.nullOr lib.types.str;
218 };
219
220 lib = lib.mkOption {
221 description = "Common functions for the kubernetes modules.";
222 default = {
223 inherit mkCert;
224 inherit mkKubeConfig;
225 inherit mkKubeConfigOptions;
226 };
227 type = lib.types.attrs;
228 };
229
230 secretsPath = lib.mkOption {
231 description = "Default location for kubernetes secrets. Not a store location.";
232 type = lib.types.path;
233 default = cfg.dataDir + "/secrets";
234 defaultText = lib.literalExpression ''
235 config.${opt.dataDir} + "/secrets"
236 '';
237 };
238 };
239
240 ###### implementation
241
242 config = lib.mkMerge [
243
244 (lib.mkIf cfg.easyCerts {
245 services.kubernetes.pki.enable = lib.mkDefault true;
246 services.kubernetes.caFile = caCert;
247 })
248
249 (lib.mkIf (lib.elem "master" cfg.roles) {
250 services.kubernetes.apiserver.enable = lib.mkDefault true;
251 services.kubernetes.scheduler.enable = lib.mkDefault true;
252 services.kubernetes.controllerManager.enable = lib.mkDefault true;
253 services.kubernetes.addonManager.enable = lib.mkDefault true;
254 services.kubernetes.proxy.enable = lib.mkDefault true;
255 services.etcd.enable = true; # Cannot mkDefault because of flannel default options
256 services.kubernetes.kubelet = {
257 enable = lib.mkDefault true;
258 taints = lib.mkIf (!(lib.elem "node" cfg.roles)) {
259 master = {
260 key = "node-role.kubernetes.io/master";
261 value = "true";
262 effect = "NoSchedule";
263 };
264 };
265 };
266 })
267
268 (lib.mkIf (lib.all (el: el == "master") cfg.roles) {
269 # if this node is only a master make it unschedulable by default
270 services.kubernetes.kubelet.unschedulable = lib.mkDefault true;
271 })
272
273 (lib.mkIf (lib.elem "node" cfg.roles) {
274 services.kubernetes.kubelet.enable = lib.mkDefault true;
275 services.kubernetes.proxy.enable = lib.mkDefault true;
276 })
277
278 # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel
279 (lib.mkIf (cfg.roles != [ ]) {
280 services.kubernetes.flannel.enable = lib.mkDefault true;
281 services.flannel.etcd.endpoints = lib.mkDefault etcdEndpoints;
282 services.kubernetes.easyCerts = lib.mkDefault true;
283 })
284
285 (lib.mkIf cfg.apiserver.enable {
286 services.kubernetes.pki.etcClusterAdminKubeconfig = lib.mkDefault "kubernetes/cluster-admin.kubeconfig";
287 services.kubernetes.apiserver.etcd.servers = lib.mkDefault etcdEndpoints;
288 })
289
290 (lib.mkIf cfg.kubelet.enable {
291 virtualisation.containerd = {
292 enable = lib.mkDefault true;
293 settings = lib.mapAttrsRecursive (name: lib.mkDefault) defaultContainerdSettings;
294 };
295 })
296
297 (lib.mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) {
298 services.kubernetes.pki.certs = {
299 serviceAccount = mkCert {
300 name = "service-account";
301 CN = "system:service-account-signer";
302 action = ''
303 systemctl restart \
304 kube-apiserver.service \
305 kube-controller-manager.service
306 '';
307 };
308 };
309 })
310
311 (lib.mkIf
312 (
313 cfg.apiserver.enable
314 || cfg.scheduler.enable
315 || cfg.controllerManager.enable
316 || cfg.kubelet.enable
317 || cfg.proxy.enable
318 || cfg.addonManager.enable
319 )
320 {
321 systemd.targets.kubernetes = {
322 description = "Kubernetes";
323 wantedBy = [ "multi-user.target" ];
324 };
325
326 systemd.tmpfiles.rules = [
327 "d /opt/cni/bin 0755 root root -"
328 "d /run/kubernetes 0755 kubernetes kubernetes -"
329 "d ${cfg.dataDir} 0755 kubernetes kubernetes -"
330 ];
331
332 users.users.kubernetes = {
333 uid = config.ids.uids.kubernetes;
334 description = "Kubernetes user";
335 group = "kubernetes";
336 home = cfg.dataDir;
337 createHome = true;
338 homeMode = "755";
339 };
340 users.groups.kubernetes.gid = config.ids.gids.kubernetes;
341
342 # dns addon is enabled by default
343 services.kubernetes.addons.dns.enable = lib.mkDefault true;
344
345 services.kubernetes.apiserverAddress = lib.mkDefault (
346 "https://${
347 if cfg.apiserver.advertiseAddress != null then
348 cfg.apiserver.advertiseAddress
349 else
350 "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"
351 }"
352 );
353 }
354 )
355 ];
356
357 meta.buildDocsInSandbox = false;
358}