pyrox.dev
1{ config, options, pkgs, lib, ... }:
2
3with lib;
4
5let
6 version = "1.10.1";
7 cfg = config.services.kubernetes.addons.dns;
8 ports = {
9 dns = 10053;
10 health = 10054;
11 metrics = 10055;
12 };
13in {
14 options.services.kubernetes.addons.dns = {
15 enable = mkEnableOption (lib.mdDoc "kubernetes dns addon");
16
17 clusterIp = mkOption {
18 description = lib.mdDoc "Dns addon clusterIP";
19
20 # this default is also what kubernetes users
21 default = (
22 concatStringsSep "." (
23 take 3 (splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
24 ))
25 ) + ".254";
26 defaultText = literalMD ''
27 The `x.y.z.254` IP of
28 `config.${options.services.kubernetes.apiserver.serviceClusterIpRange}`.
29 '';
30 type = types.str;
31 };
32
33 clusterDomain = mkOption {
34 description = lib.mdDoc "Dns cluster domain";
35 default = "cluster.local";
36 type = types.str;
37 };
38
39 replicas = mkOption {
40 description = lib.mdDoc "Number of DNS pod replicas to deploy in the cluster.";
41 default = 2;
42 type = types.int;
43 };
44
45 reconcileMode = mkOption {
46 description = lib.mdDoc ''
47 Controls the addon manager reconciliation mode for the DNS addon.
48
49 Setting reconcile mode to EnsureExists makes it possible to tailor DNS behavior by editing the coredns ConfigMap.
50
51 See: <https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md>.
52 '';
53 default = "Reconcile";
54 type = types.enum [ "Reconcile" "EnsureExists" ];
55 };
56
57 coredns = mkOption {
58 description = lib.mdDoc "Docker image to seed for the CoreDNS container.";
59 type = types.attrs;
60 default = {
61 imageName = "coredns/coredns";
62 imageDigest = "sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e";
63 finalImageTag = version;
64 sha256 = "0wg696920smmal7552a2zdhfncndn5kfammfa8bk8l7dz9bhk0y1";
65 };
66 };
67
68 corefile = mkOption {
69 description = lib.mdDoc ''
70 Custom coredns corefile configuration.
71
72 See: <https://coredns.io/manual/toc/#configuration>.
73 '';
74 type = types.str;
75 default = ''
76 .:${toString ports.dns} {
77 errors
78 health :${toString ports.health}
79 kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
80 pods insecure
81 fallthrough in-addr.arpa ip6.arpa
82 }
83 prometheus :${toString ports.metrics}
84 forward . /etc/resolv.conf
85 cache 30
86 loop
87 reload
88 loadbalance
89 }'';
90 defaultText = literalExpression ''
91 '''
92 .:${toString ports.dns} {
93 errors
94 health :${toString ports.health}
95 kubernetes ''${config.services.kubernetes.addons.dns.clusterDomain} in-addr.arpa ip6.arpa {
96 pods insecure
97 fallthrough in-addr.arpa ip6.arpa
98 }
99 prometheus :${toString ports.metrics}
100 forward . /etc/resolv.conf
101 cache 30
102 loop
103 reload
104 loadbalance
105 }
106 '''
107 '';
108 };
109 };
110
111 config = mkIf cfg.enable {
112 services.kubernetes.kubelet.seedDockerImages =
113 singleton (pkgs.dockerTools.pullImage cfg.coredns);
114
115 services.kubernetes.addonManager.bootstrapAddons = {
116 coredns-cr = {
117 apiVersion = "rbac.authorization.k8s.io/v1";
118 kind = "ClusterRole";
119 metadata = {
120 labels = {
121 "addonmanager.kubernetes.io/mode" = "Reconcile";
122 k8s-app = "kube-dns";
123 "kubernetes.io/cluster-service" = "true";
124 "kubernetes.io/bootstrapping" = "rbac-defaults";
125 };
126 name = "system:coredns";
127 };
128 rules = [
129 {
130 apiGroups = [ "" ];
131 resources = [ "endpoints" "services" "pods" "namespaces" ];
132 verbs = [ "list" "watch" ];
133 }
134 {
135 apiGroups = [ "" ];
136 resources = [ "nodes" ];
137 verbs = [ "get" ];
138 }
139 {
140 apiGroups = [ "discovery.k8s.io" ];
141 resources = [ "endpointslices" ];
142 verbs = [ "list" "watch" ];
143 }
144 ];
145 };
146
147 coredns-crb = {
148 apiVersion = "rbac.authorization.k8s.io/v1";
149 kind = "ClusterRoleBinding";
150 metadata = {
151 annotations = {
152 "rbac.authorization.kubernetes.io/autoupdate" = "true";
153 };
154 labels = {
155 "addonmanager.kubernetes.io/mode" = "Reconcile";
156 k8s-app = "kube-dns";
157 "kubernetes.io/cluster-service" = "true";
158 "kubernetes.io/bootstrapping" = "rbac-defaults";
159 };
160 name = "system:coredns";
161 };
162 roleRef = {
163 apiGroup = "rbac.authorization.k8s.io";
164 kind = "ClusterRole";
165 name = "system:coredns";
166 };
167 subjects = [
168 {
169 kind = "ServiceAccount";
170 name = "coredns";
171 namespace = "kube-system";
172 }
173 ];
174 };
175 };
176
177 services.kubernetes.addonManager.addons = {
178 coredns-sa = {
179 apiVersion = "v1";
180 kind = "ServiceAccount";
181 metadata = {
182 labels = {
183 "addonmanager.kubernetes.io/mode" = "Reconcile";
184 k8s-app = "kube-dns";
185 "kubernetes.io/cluster-service" = "true";
186 };
187 name = "coredns";
188 namespace = "kube-system";
189 };
190 };
191
192 coredns-cm = {
193 apiVersion = "v1";
194 kind = "ConfigMap";
195 metadata = {
196 labels = {
197 "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
198 k8s-app = "kube-dns";
199 "kubernetes.io/cluster-service" = "true";
200 };
201 name = "coredns";
202 namespace = "kube-system";
203 };
204 data = {
205 Corefile = cfg.corefile;
206 };
207 };
208
209 coredns-deploy = {
210 apiVersion = "apps/v1";
211 kind = "Deployment";
212 metadata = {
213 labels = {
214 "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
215 k8s-app = "kube-dns";
216 "kubernetes.io/cluster-service" = "true";
217 "kubernetes.io/name" = "CoreDNS";
218 };
219 name = "coredns";
220 namespace = "kube-system";
221 };
222 spec = {
223 replicas = cfg.replicas;
224 selector = {
225 matchLabels = { k8s-app = "kube-dns"; };
226 };
227 strategy = {
228 rollingUpdate = { maxUnavailable = 1; };
229 type = "RollingUpdate";
230 };
231 template = {
232 metadata = {
233 labels = {
234 k8s-app = "kube-dns";
235 };
236 };
237 spec = {
238 containers = [
239 {
240 args = [ "-conf" "/etc/coredns/Corefile" ];
241 image = with cfg.coredns; "${imageName}:${finalImageTag}";
242 imagePullPolicy = "Never";
243 livenessProbe = {
244 failureThreshold = 5;
245 httpGet = {
246 path = "/health";
247 port = ports.health;
248 scheme = "HTTP";
249 };
250 initialDelaySeconds = 60;
251 successThreshold = 1;
252 timeoutSeconds = 5;
253 };
254 name = "coredns";
255 ports = [
256 {
257 containerPort = ports.dns;
258 name = "dns";
259 protocol = "UDP";
260 }
261 {
262 containerPort = ports.dns;
263 name = "dns-tcp";
264 protocol = "TCP";
265 }
266 {
267 containerPort = ports.metrics;
268 name = "metrics";
269 protocol = "TCP";
270 }
271 ];
272 resources = {
273 limits = {
274 memory = "170Mi";
275 };
276 requests = {
277 cpu = "100m";
278 memory = "70Mi";
279 };
280 };
281 securityContext = {
282 allowPrivilegeEscalation = false;
283 capabilities = {
284 drop = [ "all" ];
285 };
286 readOnlyRootFilesystem = true;
287 };
288 volumeMounts = [
289 {
290 mountPath = "/etc/coredns";
291 name = "config-volume";
292 readOnly = true;
293 }
294 ];
295 }
296 ];
297 dnsPolicy = "Default";
298 nodeSelector = {
299 "beta.kubernetes.io/os" = "linux";
300 };
301 serviceAccountName = "coredns";
302 tolerations = [
303 {
304 effect = "NoSchedule";
305 key = "node-role.kubernetes.io/master";
306 }
307 {
308 key = "CriticalAddonsOnly";
309 operator = "Exists";
310 }
311 ];
312 volumes = [
313 {
314 configMap = {
315 items = [
316 {
317 key = "Corefile";
318 path = "Corefile";
319 }
320 ];
321 name = "coredns";
322 };
323 name = "config-volume";
324 }
325 ];
326 };
327 };
328 };
329 };
330
331 coredns-svc = {
332 apiVersion = "v1";
333 kind = "Service";
334 metadata = {
335 annotations = {
336 "prometheus.io/port" = toString ports.metrics;
337 "prometheus.io/scrape" = "true";
338 };
339 labels = {
340 "addonmanager.kubernetes.io/mode" = "Reconcile";
341 k8s-app = "kube-dns";
342 "kubernetes.io/cluster-service" = "true";
343 "kubernetes.io/name" = "CoreDNS";
344 };
345 name = "kube-dns";
346 namespace = "kube-system";
347 };
348 spec = {
349 clusterIP = cfg.clusterIp;
350 ports = [
351 {
352 name = "dns";
353 port = 53;
354 targetPort = ports.dns;
355 protocol = "UDP";
356 }
357 {
358 name = "dns-tcp";
359 port = 53;
360 targetPort = ports.dns;
361 protocol = "TCP";
362 }
363 ];
364 selector = { k8s-app = "kube-dns"; };
365 };
366 };
367 };
368
369 services.kubernetes.kubelet.clusterDns = mkDefault cfg.clusterIp;
370 };
371
372 meta.buildDocsInSandbox = false;
373}