pyrox.dev
1{ config, lib, options, pkgs, ... }:
2
3with lib;
4
5let
6 cfg = config.services.kubernetes;
7 opt = options.services.kubernetes;
8
9 defaultContainerdSettings = {
10 version = 2;
11 root = "/var/lib/containerd";
12 state = "/run/containerd";
13 oom_score = 0;
14
15 grpc = {
16 address = "/run/containerd/containerd.sock";
17 };
18
19 plugins."io.containerd.grpc.v1.cri" = {
20 sandbox_image = "pause:latest";
21
22 cni = {
23 bin_dir = "/opt/cni/bin";
24 max_conf_num = 0;
25 };
26
27 containerd.runtimes.runc = {
28 runtime_type = "io.containerd.runc.v2";
29 options.SystemdCgroup = true;
30 };
31 };
32 };
33
34 mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
35 apiVersion = "v1";
36 kind = "Config";
37 clusters = [{
38 name = "local";
39 cluster.certificate-authority = conf.caFile or cfg.caFile;
40 cluster.server = conf.server;
41 }];
42 users = [{
43 inherit name;
44 user = {
45 client-certificate = conf.certFile;
46 client-key = conf.keyFile;
47 };
48 }];
49 contexts = [{
50 context = {
51 cluster = "local";
52 user = name;
53 };
54 name = "local";
55 }];
56 current-context = "local";
57 });
58
59 caCert = secret "ca";
60
61 etcdEndpoints = ["https://${cfg.masterAddress}:2379"];
62
63 mkCert = { name, CN, hosts ? [], fields ? {}, action ? "",
64 privateKeyOwner ? "kubernetes" }: rec {
65 inherit name caCert CN hosts fields action;
66 cert = secret name;
67 key = secret "${name}-key";
68 privateKeyOptions = {
69 owner = privateKeyOwner;
70 group = "nogroup";
71 mode = "0600";
72 path = key;
73 };
74 };
75
76 secret = name: "${cfg.secretsPath}/${name}.pem";
77
78 mkKubeConfigOptions = prefix: {
79 server = mkOption {
80 description = "${prefix} kube-apiserver server address.";
81 type = types.str;
82 };
83
84 caFile = mkOption {
85 description = "${prefix} certificate authority file used to connect to kube-apiserver.";
86 type = types.nullOr types.path;
87 default = cfg.caFile;
88 defaultText = literalExpression "config.${opt.caFile}";
89 };
90
91 certFile = mkOption {
92 description = "${prefix} client certificate file used to connect to kube-apiserver.";
93 type = types.nullOr types.path;
94 default = null;
95 };
96
97 keyFile = mkOption {
98 description = "${prefix} client key file used to connect to kube-apiserver.";
99 type = types.nullOr types.path;
100 default = null;
101 };
102 };
103in {
104
105 imports = [
106 (mkRemovedOptionModule [ "services" "kubernetes" "addons" "dashboard" ] "Removed due to it being an outdated version")
107 (mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "")
108 ];
109
110 ###### interface
111
112 options.services.kubernetes = {
113 roles = mkOption {
114 description = ''
115 Kubernetes role that this machine should take.
116
117 Master role will enable etcd, apiserver, scheduler, controller manager
118 addon manager, flannel and proxy services.
119 Node role will enable flannel, docker, kubelet and proxy services.
120 '';
121 default = [];
122 type = types.listOf (types.enum ["master" "node"]);
123 };
124
125 package = mkPackageOption pkgs "kubernetes" { };
126
127 kubeconfig = mkKubeConfigOptions "Default kubeconfig";
128
129 apiserverAddress = mkOption {
130 description = ''
131 Clusterwide accessible address for the kubernetes apiserver,
132 including protocol and optional port.
133 '';
134 example = "https://kubernetes-apiserver.example.com:6443";
135 type = types.str;
136 };
137
138 caFile = mkOption {
139 description = "Default kubernetes certificate authority";
140 type = types.nullOr types.path;
141 default = null;
142 };
143
144 dataDir = mkOption {
145 description = "Kubernetes root directory for managing kubelet files.";
146 default = "/var/lib/kubernetes";
147 type = types.path;
148 };
149
150 easyCerts = mkOption {
151 description = "Automatically setup x509 certificates and keys for the entire cluster.";
152 default = false;
153 type = types.bool;
154 };
155
156 featureGates = mkOption {
157 description = "List set of feature gates.";
158 default = [];
159 type = types.listOf types.str;
160 };
161
162 masterAddress = mkOption {
163 description = "Clusterwide available network address or hostname for the kubernetes master server.";
164 example = "master.example.com";
165 type = types.str;
166 };
167
168 path = mkOption {
169 description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
170 type = types.listOf types.package;
171 default = [];
172 };
173
174 clusterCidr = mkOption {
175 description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
176 default = "10.1.0.0/16";
177 type = types.nullOr types.str;
178 };
179
180 lib = mkOption {
181 description = "Common functions for the kubernetes modules.";
182 default = {
183 inherit mkCert;
184 inherit mkKubeConfig;
185 inherit mkKubeConfigOptions;
186 };
187 type = types.attrs;
188 };
189
190 secretsPath = mkOption {
191 description = "Default location for kubernetes secrets. Not a store location.";
192 type = types.path;
193 default = cfg.dataDir + "/secrets";
194 defaultText = literalExpression ''
195 config.${opt.dataDir} + "/secrets"
196 '';
197 };
198 };
199
200 ###### implementation
201
202 config = mkMerge [
203
204 (mkIf cfg.easyCerts {
205 services.kubernetes.pki.enable = mkDefault true;
206 services.kubernetes.caFile = caCert;
207 })
208
209 (mkIf (elem "master" cfg.roles) {
210 services.kubernetes.apiserver.enable = mkDefault true;
211 services.kubernetes.scheduler.enable = mkDefault true;
212 services.kubernetes.controllerManager.enable = mkDefault true;
213 services.kubernetes.addonManager.enable = mkDefault true;
214 services.kubernetes.proxy.enable = mkDefault true;
215 services.etcd.enable = true; # Cannot mkDefault because of flannel default options
216 services.kubernetes.kubelet = {
217 enable = mkDefault true;
218 taints = mkIf (!(elem "node" cfg.roles)) {
219 master = {
220 key = "node-role.kubernetes.io/master";
221 value = "true";
222 effect = "NoSchedule";
223 };
224 };
225 };
226 })
227
228
229 (mkIf (all (el: el == "master") cfg.roles) {
230 # if this node is only a master make it unschedulable by default
231 services.kubernetes.kubelet.unschedulable = mkDefault true;
232 })
233
234 (mkIf (elem "node" cfg.roles) {
235 services.kubernetes.kubelet.enable = mkDefault true;
236 services.kubernetes.proxy.enable = mkDefault true;
237 })
238
239 # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel
240 (mkIf (cfg.roles != []) {
241 services.kubernetes.flannel.enable = mkDefault true;
242 services.flannel.etcd.endpoints = mkDefault etcdEndpoints;
243 services.kubernetes.easyCerts = mkDefault true;
244 })
245
246 (mkIf cfg.apiserver.enable {
247 services.kubernetes.pki.etcClusterAdminKubeconfig = mkDefault "kubernetes/cluster-admin.kubeconfig";
248 services.kubernetes.apiserver.etcd.servers = mkDefault etcdEndpoints;
249 })
250
251 (mkIf cfg.kubelet.enable {
252 virtualisation.containerd = {
253 enable = mkDefault true;
254 settings = mapAttrsRecursive (name: mkDefault) defaultContainerdSettings;
255 };
256 })
257
258 (mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) {
259 services.kubernetes.pki.certs = {
260 serviceAccount = mkCert {
261 name = "service-account";
262 CN = "system:service-account-signer";
263 action = ''
264 systemctl reload \
265 kube-apiserver.service \
266 kube-controller-manager.service
267 '';
268 };
269 };
270 })
271
272 (mkIf (
273 cfg.apiserver.enable ||
274 cfg.scheduler.enable ||
275 cfg.controllerManager.enable ||
276 cfg.kubelet.enable ||
277 cfg.proxy.enable ||
278 cfg.addonManager.enable
279 ) {
280 systemd.targets.kubernetes = {
281 description = "Kubernetes";
282 wantedBy = [ "multi-user.target" ];
283 };
284
285 systemd.tmpfiles.rules = [
286 "d /opt/cni/bin 0755 root root -"
287 "d /run/kubernetes 0755 kubernetes kubernetes -"
288 "d ${cfg.dataDir} 0755 kubernetes kubernetes -"
289 ];
290
291 users.users.kubernetes = {
292 uid = config.ids.uids.kubernetes;
293 description = "Kubernetes user";
294 group = "kubernetes";
295 home = cfg.dataDir;
296 createHome = true;
297 homeMode = "755";
298 };
299 users.groups.kubernetes.gid = config.ids.gids.kubernetes;
300
301 # dns addon is enabled by default
302 services.kubernetes.addons.dns.enable = mkDefault true;
303
304 services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null
305 then cfg.apiserver.advertiseAddress
306 else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
307 })
308 ];
309
310 meta.buildDocsInSandbox = false;
311}