pyrox.dev
1{
2 config,
3 options,
4 pkgs,
5 lib,
6 ...
7}:
8let
9 version = "1.10.1";
10 cfg = config.services.kubernetes.addons.dns;
11 ports = {
12 dns = 10053;
13 health = 10054;
14 metrics = 10055;
15 };
16in
17{
18 options.services.kubernetes.addons.dns = {
19 enable = lib.mkEnableOption "kubernetes dns addon";
20
21 clusterIp = lib.mkOption {
22 description = "Dns addon clusterIP";
23
24 # this default is also what kubernetes users
25 default =
26 (lib.concatStringsSep "." (
27 lib.take 3 (lib.splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange)
28 ))
29 + ".254";
30 defaultText = lib.literalMD ''
31 The `x.y.z.254` IP of
32 `config.${options.services.kubernetes.apiserver.serviceClusterIpRange}`.
33 '';
34 type = lib.types.str;
35 };
36
37 clusterDomain = lib.mkOption {
38 description = "Dns cluster domain";
39 default = "cluster.local";
40 type = lib.types.str;
41 };
42
43 replicas = lib.mkOption {
44 description = "Number of DNS pod replicas to deploy in the cluster.";
45 default = 2;
46 type = lib.types.int;
47 };
48
49 reconcileMode = lib.mkOption {
50 description = ''
51 Controls the addon manager reconciliation mode for the DNS addon.
52
53 Setting reconcile mode to EnsureExists makes it possible to tailor DNS behavior by editing the coredns ConfigMap.
54
55 See: <https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md>.
56 '';
57 default = "Reconcile";
58 type = lib.types.enum [
59 "Reconcile"
60 "EnsureExists"
61 ];
62 };
63
64 coredns = lib.mkOption {
65 description = "Docker image to seed for the CoreDNS container.";
66 type = lib.types.attrs;
67 default = {
68 imageName = "coredns/coredns";
69 imageDigest = "sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e";
70 finalImageTag = version;
71 sha256 = "0wg696920smmal7552a2zdhfncndn5kfammfa8bk8l7dz9bhk0y1";
72 };
73 };
74
75 corefile = lib.mkOption {
76 description = ''
77 Custom coredns corefile configuration.
78
79 See: <https://coredns.io/manual/toc/#configuration>.
80 '';
81 type = lib.types.str;
82 default = ''
83 .:${toString ports.dns} {
84 errors
85 health :${toString ports.health}
86 kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
87 pods insecure
88 fallthrough in-addr.arpa ip6.arpa
89 }
90 prometheus :${toString ports.metrics}
91 forward . /etc/resolv.conf
92 cache 30
93 loop
94 reload
95 loadbalance
96 }'';
97 defaultText = lib.literalExpression ''
98 '''
99 .:${toString ports.dns} {
100 errors
101 health :${toString ports.health}
102 kubernetes ''${config.services.kubernetes.addons.dns.clusterDomain} in-addr.arpa ip6.arpa {
103 pods insecure
104 fallthrough in-addr.arpa ip6.arpa
105 }
106 prometheus :${toString ports.metrics}
107 forward . /etc/resolv.conf
108 cache 30
109 loop
110 reload
111 loadbalance
112 }
113 '''
114 '';
115 };
116 };
117
118 config = lib.mkIf cfg.enable {
119 services.kubernetes.kubelet.seedDockerImages = lib.singleton (
120 pkgs.dockerTools.pullImage cfg.coredns
121 );
122
123 services.kubernetes.addonManager.bootstrapAddons = {
124 coredns-cr = {
125 apiVersion = "rbac.authorization.k8s.io/v1";
126 kind = "ClusterRole";
127 metadata = {
128 labels = {
129 "addonmanager.kubernetes.io/mode" = "Reconcile";
130 k8s-app = "kube-dns";
131 "kubernetes.io/cluster-service" = "true";
132 "kubernetes.io/bootstrapping" = "rbac-defaults";
133 };
134 name = "system:coredns";
135 };
136 rules = [
137 {
138 apiGroups = [ "" ];
139 resources = [
140 "endpoints"
141 "services"
142 "pods"
143 "namespaces"
144 ];
145 verbs = [
146 "list"
147 "watch"
148 ];
149 }
150 {
151 apiGroups = [ "" ];
152 resources = [ "nodes" ];
153 verbs = [ "get" ];
154 }
155 {
156 apiGroups = [ "discovery.k8s.io" ];
157 resources = [ "endpointslices" ];
158 verbs = [
159 "list"
160 "watch"
161 ];
162 }
163 ];
164 };
165
166 coredns-crb = {
167 apiVersion = "rbac.authorization.k8s.io/v1";
168 kind = "ClusterRoleBinding";
169 metadata = {
170 annotations = {
171 "rbac.authorization.kubernetes.io/autoupdate" = "true";
172 };
173 labels = {
174 "addonmanager.kubernetes.io/mode" = "Reconcile";
175 k8s-app = "kube-dns";
176 "kubernetes.io/cluster-service" = "true";
177 "kubernetes.io/bootstrapping" = "rbac-defaults";
178 };
179 name = "system:coredns";
180 };
181 roleRef = {
182 apiGroup = "rbac.authorization.k8s.io";
183 kind = "ClusterRole";
184 name = "system:coredns";
185 };
186 subjects = [
187 {
188 kind = "ServiceAccount";
189 name = "coredns";
190 namespace = "kube-system";
191 }
192 ];
193 };
194 };
195
196 services.kubernetes.addonManager.addons = {
197 coredns-sa = {
198 apiVersion = "v1";
199 kind = "ServiceAccount";
200 metadata = {
201 labels = {
202 "addonmanager.kubernetes.io/mode" = "Reconcile";
203 k8s-app = "kube-dns";
204 "kubernetes.io/cluster-service" = "true";
205 };
206 name = "coredns";
207 namespace = "kube-system";
208 };
209 };
210
211 coredns-cm = {
212 apiVersion = "v1";
213 kind = "ConfigMap";
214 metadata = {
215 labels = {
216 "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
217 k8s-app = "kube-dns";
218 "kubernetes.io/cluster-service" = "true";
219 };
220 name = "coredns";
221 namespace = "kube-system";
222 };
223 data = {
224 Corefile = cfg.corefile;
225 };
226 };
227
228 coredns-deploy = {
229 apiVersion = "apps/v1";
230 kind = "Deployment";
231 metadata = {
232 labels = {
233 "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
234 k8s-app = "kube-dns";
235 "kubernetes.io/cluster-service" = "true";
236 "kubernetes.io/name" = "CoreDNS";
237 };
238 name = "coredns";
239 namespace = "kube-system";
240 };
241 spec = {
242 replicas = cfg.replicas;
243 selector = {
244 matchLabels = {
245 k8s-app = "kube-dns";
246 };
247 };
248 strategy = {
249 rollingUpdate = {
250 maxUnavailable = 1;
251 };
252 type = "RollingUpdate";
253 };
254 template = {
255 metadata = {
256 labels = {
257 k8s-app = "kube-dns";
258 };
259 };
260 spec = {
261 containers = [
262 {
263 args = [
264 "-conf"
265 "/etc/coredns/Corefile"
266 ];
267 image = with cfg.coredns; "${imageName}:${finalImageTag}";
268 imagePullPolicy = "Never";
269 livenessProbe = {
270 failureThreshold = 5;
271 httpGet = {
272 path = "/health";
273 port = ports.health;
274 scheme = "HTTP";
275 };
276 initialDelaySeconds = 60;
277 successThreshold = 1;
278 timeoutSeconds = 5;
279 };
280 name = "coredns";
281 ports = [
282 {
283 containerPort = ports.dns;
284 name = "dns";
285 protocol = "UDP";
286 }
287 {
288 containerPort = ports.dns;
289 name = "dns-tcp";
290 protocol = "TCP";
291 }
292 {
293 containerPort = ports.metrics;
294 name = "metrics";
295 protocol = "TCP";
296 }
297 ];
298 resources = {
299 limits = {
300 memory = "170Mi";
301 };
302 requests = {
303 cpu = "100m";
304 memory = "70Mi";
305 };
306 };
307 securityContext = {
308 allowPrivilegeEscalation = false;
309 capabilities = {
310 drop = [ "all" ];
311 };
312 readOnlyRootFilesystem = true;
313 };
314 volumeMounts = [
315 {
316 mountPath = "/etc/coredns";
317 name = "config-volume";
318 readOnly = true;
319 }
320 ];
321 }
322 ];
323 dnsPolicy = "Default";
324 nodeSelector = {
325 "beta.kubernetes.io/os" = "linux";
326 };
327 serviceAccountName = "coredns";
328 tolerations = [
329 {
330 effect = "NoSchedule";
331 key = "node-role.kubernetes.io/master";
332 }
333 {
334 key = "CriticalAddonsOnly";
335 operator = "Exists";
336 }
337 ];
338 volumes = [
339 {
340 configMap = {
341 items = [
342 {
343 key = "Corefile";
344 path = "Corefile";
345 }
346 ];
347 name = "coredns";
348 };
349 name = "config-volume";
350 }
351 ];
352 };
353 };
354 };
355 };
356
357 coredns-svc = {
358 apiVersion = "v1";
359 kind = "Service";
360 metadata = {
361 annotations = {
362 "prometheus.io/port" = toString ports.metrics;
363 "prometheus.io/scrape" = "true";
364 };
365 labels = {
366 "addonmanager.kubernetes.io/mode" = "Reconcile";
367 k8s-app = "kube-dns";
368 "kubernetes.io/cluster-service" = "true";
369 "kubernetes.io/name" = "CoreDNS";
370 };
371 name = "kube-dns";
372 namespace = "kube-system";
373 };
374 spec = {
375 clusterIP = cfg.clusterIp;
376 ports = [
377 {
378 name = "dns";
379 port = 53;
380 targetPort = ports.dns;
381 protocol = "UDP";
382 }
383 {
384 name = "dns-tcp";
385 port = 53;
386 targetPort = ports.dns;
387 protocol = "TCP";
388 }
389 ];
390 selector = {
391 k8s-app = "kube-dns";
392 };
393 };
394 };
395 };
396
397 services.kubernetes.kubelet.clusterDns = lib.mkDefault [ cfg.clusterIp ];
398 };
399
400 meta.buildDocsInSandbox = false;
401}