pyrox.dev
1{ config, lib, pkgs, utils, ... }:
2
3with utils;
4with lib;
5
6let
7
8 cfg = config.networking;
9 interfaces = attrValues cfg.interfaces;
10
11 slaves = concatMap (i: i.interfaces) (attrValues cfg.bonds)
12 ++ concatMap (i: i.interfaces) (attrValues cfg.bridges)
13 ++ concatMap (i: i.interfaces) (attrValues cfg.vswitches)
14 ++ concatMap (i: [i.interface]) (attrValues cfg.macvlans)
15 ++ concatMap (i: [i.interface]) (attrValues cfg.vlans);
16
17 # We must escape interfaces due to the systemd interpretation
18 subsystemDevice = interface:
19 "sys-subsystem-net-devices-${escapeSystemdPath interface}.device";
20
21 interfaceIps = i:
22 i.ipv4.addresses
23 ++ optionals cfg.enableIPv6 i.ipv6.addresses;
24
25 destroyBond = i: ''
26 while true; do
27 UPDATED=1
28 SLAVES=$(ip link | grep 'master ${i}' | awk -F: '{print $2}')
29 for I in $SLAVES; do
30 UPDATED=0
31 ip link set "$I" nomaster
32 done
33 [ "$UPDATED" -eq "1" ] && break
34 done
35 ip link set "${i}" down 2>/dev/null || true
36 ip link del "${i}" 2>/dev/null || true
37 '';
38
39 # warn that these attributes are deprecated (2017-2-2)
40 # Should be removed in the release after next
41 bondDeprecation = rec {
42 deprecated = [ "lacp_rate" "miimon" "mode" "xmit_hash_policy" ];
43 filterDeprecated = bond: (filterAttrs (attrName: attr:
44 elem attrName deprecated && attr != null) bond);
45 };
46
47 bondWarnings =
48 let oneBondWarnings = bondName: bond:
49 mapAttrsToList (bondText bondName) (bondDeprecation.filterDeprecated bond);
50 bondText = bondName: optName: _:
51 "${bondName}.${optName} is deprecated, use ${bondName}.driverOptions";
52 in {
53 warnings = flatten (mapAttrsToList oneBondWarnings cfg.bonds);
54 };
55
56 normalConfig = {
57
58 systemd.services =
59 let
60
61 deviceDependency = dev:
62 # Use systemd service if we manage device creation, else
63 # trust udev when not in a container
64 if (hasAttr dev (filterAttrs (k: v: v.virtual) cfg.interfaces)) ||
65 (hasAttr dev cfg.bridges) ||
66 (hasAttr dev cfg.bonds) ||
67 (hasAttr dev cfg.macvlans) ||
68 (hasAttr dev cfg.sits) ||
69 (hasAttr dev cfg.vlans) ||
70 (hasAttr dev cfg.vswitches)
71 then [ "${dev}-netdev.service" ]
72 else optional (dev != null && dev != "lo" && !config.boot.isContainer) (subsystemDevice dev);
73
74 hasDefaultGatewaySet = (cfg.defaultGateway != null && cfg.defaultGateway.address != "")
75 || (cfg.enableIPv6 && cfg.defaultGateway6 != null && cfg.defaultGateway6.address != "");
76
77 networkLocalCommands = {
78 after = [ "network-setup.service" ];
79 bindsTo = [ "network-setup.service" ];
80 };
81
82 networkSetup =
83 { description = "Networking Setup";
84
85 after = [ "network-pre.target" "systemd-udevd.service" "systemd-sysctl.service" ];
86 before = [ "network.target" "shutdown.target" ];
87 wants = [ "network.target" ];
88 partOf = map (i: "network-addresses-${i.name}.service") interfaces;
89 conflicts = [ "shutdown.target" ];
90 wantedBy = [ "multi-user.target" ] ++ optional hasDefaultGatewaySet "network-online.target";
91
92 unitConfig.ConditionCapability = "CAP_NET_ADMIN";
93
94 path = [ pkgs.iproute ];
95
96 serviceConfig = {
97 Type = "oneshot";
98 RemainAfterExit = true;
99 };
100
101 unitConfig.DefaultDependencies = false;
102
103 script =
104 ''
105 # Set the static DNS configuration, if given.
106 ${pkgs.openresolv}/sbin/resolvconf -m 1 -a static <<EOF
107 ${optionalString (cfg.nameservers != [] && cfg.domain != null) ''
108 domain ${cfg.domain}
109 ''}
110 ${optionalString (cfg.search != []) ("search " + concatStringsSep " " cfg.search)}
111 ${flip concatMapStrings cfg.nameservers (ns: ''
112 nameserver ${ns}
113 '')}
114 EOF
115
116 # Set the default gateway.
117 ${optionalString (cfg.defaultGateway != null && cfg.defaultGateway.address != "") ''
118 ${optionalString (cfg.defaultGateway.interface != null) ''
119 ip route replace ${cfg.defaultGateway.address} dev ${cfg.defaultGateway.interface} ${optionalString (cfg.defaultGateway.metric != null)
120 "metric ${toString cfg.defaultGateway.metric}"
121 } proto static
122 ''}
123 ip route replace default ${optionalString (cfg.defaultGateway.metric != null)
124 "metric ${toString cfg.defaultGateway.metric}"
125 } via "${cfg.defaultGateway.address}" ${
126 optionalString (cfg.defaultGatewayWindowSize != null)
127 "window ${toString cfg.defaultGatewayWindowSize}"} ${
128 optionalString (cfg.defaultGateway.interface != null)
129 "dev ${cfg.defaultGateway.interface}"} proto static
130 ''}
131 ${optionalString (cfg.defaultGateway6 != null && cfg.defaultGateway6.address != "") ''
132 ${optionalString (cfg.defaultGateway6.interface != null) ''
133 ip -6 route replace ${cfg.defaultGateway6.address} dev ${cfg.defaultGateway6.interface} ${optionalString (cfg.defaultGateway6.metric != null)
134 "metric ${toString cfg.defaultGateway6.metric}"
135 } proto static
136 ''}
137 ip -6 route replace default ${optionalString (cfg.defaultGateway6.metric != null)
138 "metric ${toString cfg.defaultGateway6.metric}"
139 } via "${cfg.defaultGateway6.address}" ${
140 optionalString (cfg.defaultGatewayWindowSize != null)
141 "window ${toString cfg.defaultGatewayWindowSize}"} ${
142 optionalString (cfg.defaultGateway6.interface != null)
143 "dev ${cfg.defaultGateway6.interface}"} proto static
144 ''}
145 '';
146 };
147
148 # For each interface <foo>, create a job ‘network-addresses-<foo>.service"
149 # that performs static address configuration. It has a "wants"
150 # dependency on ‘<foo>.service’, which is supposed to create
151 # the interface and need not exist (i.e. for hardware
152 # interfaces). It has a binds-to dependency on the actual
153 # network device, so it only gets started after the interface
154 # has appeared, and it's stopped when the interface
155 # disappears.
156 configureAddrs = i:
157 let
158 ips = interfaceIps i;
159 in
160 nameValuePair "network-addresses-${i.name}"
161 { description = "Address configuration of ${i.name}";
162 wantedBy = [
163 "network-setup.service"
164 "network-link-${i.name}.service"
165 "network.target"
166 ];
167 # order before network-setup because the routes that are configured
168 # there may need ip addresses configured
169 before = [ "network-setup.service" ];
170 bindsTo = deviceDependency i.name;
171 after = [ "network-pre.target" ] ++ (deviceDependency i.name);
172 serviceConfig.Type = "oneshot";
173 serviceConfig.RemainAfterExit = true;
174 # Restart rather than stop+start this unit to prevent the
175 # network from dying during switch-to-configuration.
176 stopIfChanged = false;
177 path = [ pkgs.iproute ];
178 script =
179 ''
180 state="/run/nixos/network/addresses/${i.name}"
181 mkdir -p $(dirname "$state")
182
183 ${flip concatMapStrings ips (ip:
184 let
185 cidr = "${ip.address}/${toString ip.prefixLength}";
186 in
187 ''
188 echo "${cidr}" >> $state
189 echo -n "adding address ${cidr}... "
190 if out=$(ip addr add "${cidr}" dev "${i.name}" 2>&1); then
191 echo "done"
192 elif ! echo "$out" | grep "File exists" >/dev/null 2>&1; then
193 echo "'ip addr add "${cidr}" dev "${i.name}"' failed: $out"
194 exit 1
195 fi
196 ''
197 )}
198
199 state="/run/nixos/network/routes/${i.name}"
200 mkdir -p $(dirname "$state")
201
202 ${flip concatMapStrings (i.ipv4.routes ++ i.ipv6.routes) (route:
203 let
204 cidr = "${route.address}/${toString route.prefixLength}";
205 via = optionalString (route.via != null) ''via "${route.via}"'';
206 options = concatStrings (mapAttrsToList (name: val: "${name} ${val} ") route.options);
207 in
208 ''
209 echo "${cidr}" >> $state
210 echo -n "adding route ${cidr}... "
211 if out=$(ip route add "${cidr}" ${options} ${via} dev "${i.name}" proto static 2>&1); then
212 echo "done"
213 elif ! echo "$out" | grep "File exists" >/dev/null 2>&1; then
214 echo "'ip route add "${cidr}" ${options} ${via} dev "${i.name}"' failed: $out"
215 exit 1
216 fi
217 ''
218 )}
219 '';
220 preStop = ''
221 state="/run/nixos/network/routes/${i.name}"
222 while read cidr; do
223 echo -n "deleting route $cidr... "
224 ip route del "$cidr" dev "${i.name}" >/dev/null 2>&1 && echo "done" || echo "failed"
225 done < "$state"
226 rm -f "$state"
227
228 state="/run/nixos/network/addresses/${i.name}"
229 while read cidr; do
230 echo -n "deleting address $cidr... "
231 ip addr del "$cidr" dev "${i.name}" >/dev/null 2>&1 && echo "done" || echo "failed"
232 done < "$state"
233 rm -f "$state"
234 '';
235 };
236
237 createTunDevice = i: nameValuePair "${i.name}-netdev"
238 { description = "Virtual Network Interface ${i.name}";
239 bindsTo = [ "dev-net-tun.device" ];
240 after = [ "dev-net-tun.device" "network-pre.target" ];
241 wantedBy = [ "network-setup.service" (subsystemDevice i.name) ];
242 partOf = [ "network-setup.service" ];
243 before = [ "network-setup.service" ];
244 path = [ pkgs.iproute ];
245 serviceConfig = {
246 Type = "oneshot";
247 RemainAfterExit = true;
248 };
249 script = ''
250 ip tuntap add dev "${i.name}" mode "${i.virtualType}" user "${i.virtualOwner}"
251 '';
252 postStop = ''
253 ip link del ${i.name} || true
254 '';
255 };
256
257 createBridgeDevice = n: v: nameValuePair "${n}-netdev"
258 (let
259 deps = concatLists (map deviceDependency v.interfaces);
260 in
261 { description = "Bridge Interface ${n}";
262 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
263 bindsTo = deps ++ optional v.rstp "mstpd.service";
264 partOf = [ "network-setup.service" ] ++ optional v.rstp "mstpd.service";
265 after = [ "network-pre.target" ] ++ deps ++ optional v.rstp "mstpd.service"
266 ++ concatMap (i: [ "network-addresses-${i}.service" "network-link-${i}.service" ]) v.interfaces;
267 before = [ "network-setup.service" ];
268 serviceConfig.Type = "oneshot";
269 serviceConfig.RemainAfterExit = true;
270 path = [ pkgs.iproute ];
271 script = ''
272 # Remove Dead Interfaces
273 echo "Removing old bridge ${n}..."
274 ip link show "${n}" >/dev/null 2>&1 && ip link del "${n}"
275
276 echo "Adding bridge ${n}..."
277 ip link add name "${n}" type bridge
278
279 # Enslave child interfaces
280 ${flip concatMapStrings v.interfaces (i: ''
281 ip link set "${i}" master "${n}"
282 ip link set "${i}" up
283 '')}
284 # Save list of enslaved interfaces
285 echo "${flip concatMapStrings v.interfaces (i: ''
286 ${i}
287 '')}" > /run/${n}.interfaces
288
289 ${optionalString config.virtualisation.libvirtd.enable ''
290 # Enslave dynamically added interfaces which may be lost on nixos-rebuild
291 for uri in qemu:///system lxc:///; do
292 for dom in $(${pkgs.libvirt}/bin/virsh -c $uri list --name); do
293 ${pkgs.libvirt}/bin/virsh -c $uri dumpxml "$dom" | \
294 ${pkgs.xmlstarlet}/bin/xmlstarlet sel -t -m "//domain/devices/interface[@type='bridge'][source/@bridge='${n}'][target/@dev]" -v "concat('ip link set ',target/@dev,' master ',source/@bridge,';')" | \
295 ${pkgs.bash}/bin/bash
296 done
297 done
298 ''}
299
300 # Enable stp on the interface
301 ${optionalString v.rstp ''
302 echo 2 >/sys/class/net/${n}/bridge/stp_state
303 ''}
304
305 ip link set "${n}" up
306 '';
307 postStop = ''
308 ip link set "${n}" down || true
309 ip link del "${n}" || true
310 rm -f /run/${n}.interfaces
311 '';
312 reload = ''
313 # Un-enslave child interfaces (old list of interfaces)
314 for interface in `cat /run/${n}.interfaces`; do
315 ip link set "$interface" nomaster up
316 done
317
318 # Enslave child interfaces (new list of interfaces)
319 ${flip concatMapStrings v.interfaces (i: ''
320 ip link set "${i}" master "${n}"
321 ip link set "${i}" up
322 '')}
323 # Save list of enslaved interfaces
324 echo "${flip concatMapStrings v.interfaces (i: ''
325 ${i}
326 '')}" > /run/${n}.interfaces
327
328 # (Un-)set stp on the bridge
329 echo ${if v.rstp then "2" else "0"} > /sys/class/net/${n}/bridge/stp_state
330 '';
331 reloadIfChanged = true;
332 });
333
334 createVswitchDevice = n: v: nameValuePair "${n}-netdev"
335 (let
336 deps = concatLists (map deviceDependency v.interfaces);
337 ofRules = pkgs.writeText "vswitch-${n}-openFlowRules" v.openFlowRules;
338 in
339 { description = "Open vSwitch Interface ${n}";
340 wantedBy = [ "network-setup.service" "vswitchd.service" ] ++ deps;
341 bindsTo = [ "vswitchd.service" (subsystemDevice n) ] ++ deps;
342 partOf = [ "network-setup.service" "vswitchd.service" ];
343 after = [ "network-pre.target" "vswitchd.service" ] ++ deps;
344 before = [ "network-setup.service" ];
345 serviceConfig.Type = "oneshot";
346 serviceConfig.RemainAfterExit = true;
347 path = [ pkgs.iproute config.virtualisation.vswitch.package ];
348 script = ''
349 echo "Removing old Open vSwitch ${n}..."
350 ovs-vsctl --if-exists del-br ${n}
351
352 echo "Adding Open vSwitch ${n}..."
353 ovs-vsctl -- add-br ${n} ${concatMapStrings (i: " -- add-port ${n} ${i}") v.interfaces} \
354 ${concatMapStrings (x: " -- set-controller ${n} " + x) v.controllers} \
355 ${concatMapStrings (x: " -- " + x) (splitString "\n" v.extraOvsctlCmds)}
356
357 echo "Adding OpenFlow rules for Open vSwitch ${n}..."
358 ovs-ofctl add-flows ${n} ${ofRules}
359 '';
360 postStop = ''
361 ip link set ${n} down || true
362 ovs-ofctl del-flows ${n} || true
363 ovs-vsctl --if-exists del-br ${n}
364 '';
365 });
366
367 createBondDevice = n: v: nameValuePair "${n}-netdev"
368 (let
369 deps = concatLists (map deviceDependency v.interfaces);
370 in
371 { description = "Bond Interface ${n}";
372 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
373 bindsTo = deps;
374 partOf = [ "network-setup.service" ];
375 after = [ "network-pre.target" ] ++ deps
376 ++ concatMap (i: [ "network-addresses-${i}.service" "network-link-${i}.service" ]) v.interfaces;
377 before = [ "network-setup.service" ];
378 serviceConfig.Type = "oneshot";
379 serviceConfig.RemainAfterExit = true;
380 path = [ pkgs.iproute pkgs.gawk ];
381 script = ''
382 echo "Destroying old bond ${n}..."
383 ${destroyBond n}
384
385 echo "Creating new bond ${n}..."
386 ip link add name "${n}" type bond \
387 ${let opts = (mapAttrs (const toString)
388 (bondDeprecation.filterDeprecated v))
389 // v.driverOptions;
390 in concatStringsSep "\n"
391 (mapAttrsToList (set: val: " ${set} ${val} \\") opts)}
392
393 # !!! There must be a better way to wait for the interface
394 while [ ! -d "/sys/class/net/${n}" ]; do sleep 0.1; done;
395
396 # Bring up the bond and enslave the specified interfaces
397 ip link set "${n}" up
398 ${flip concatMapStrings v.interfaces (i: ''
399 ip link set "${i}" down
400 ip link set "${i}" master "${n}"
401 '')}
402 '';
403 postStop = destroyBond n;
404 });
405
406 createMacvlanDevice = n: v: nameValuePair "${n}-netdev"
407 (let
408 deps = deviceDependency v.interface;
409 in
410 { description = "Vlan Interface ${n}";
411 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
412 bindsTo = deps;
413 partOf = [ "network-setup.service" ];
414 after = [ "network-pre.target" ] ++ deps;
415 before = [ "network-setup.service" ];
416 serviceConfig.Type = "oneshot";
417 serviceConfig.RemainAfterExit = true;
418 path = [ pkgs.iproute ];
419 script = ''
420 # Remove Dead Interfaces
421 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
422 ip link add link "${v.interface}" name "${n}" type macvlan \
423 ${optionalString (v.mode != null) "mode ${v.mode}"}
424 ip link set "${n}" up
425 '';
426 postStop = ''
427 ip link delete "${n}" || true
428 '';
429 });
430
431 createSitDevice = n: v: nameValuePair "${n}-netdev"
432 (let
433 deps = deviceDependency v.dev;
434 in
435 { description = "6-to-4 Tunnel Interface ${n}";
436 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
437 bindsTo = deps;
438 partOf = [ "network-setup.service" ];
439 after = [ "network-pre.target" ] ++ deps;
440 before = [ "network-setup.service" ];
441 serviceConfig.Type = "oneshot";
442 serviceConfig.RemainAfterExit = true;
443 path = [ pkgs.iproute ];
444 script = ''
445 # Remove Dead Interfaces
446 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
447 ip link add name "${n}" type sit \
448 ${optionalString (v.remote != null) "remote \"${v.remote}\""} \
449 ${optionalString (v.local != null) "local \"${v.local}\""} \
450 ${optionalString (v.ttl != null) "ttl ${toString v.ttl}"} \
451 ${optionalString (v.dev != null) "dev \"${v.dev}\""}
452 ip link set "${n}" up
453 '';
454 postStop = ''
455 ip link delete "${n}" || true
456 '';
457 });
458
459 createVlanDevice = n: v: nameValuePair "${n}-netdev"
460 (let
461 deps = deviceDependency v.interface;
462 in
463 { description = "Vlan Interface ${n}";
464 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
465 bindsTo = deps;
466 partOf = [ "network-setup.service" ];
467 after = [ "network-pre.target" ] ++ deps;
468 before = [ "network-setup.service" ];
469 serviceConfig.Type = "oneshot";
470 serviceConfig.RemainAfterExit = true;
471 path = [ pkgs.iproute ];
472 script = ''
473 # Remove Dead Interfaces
474 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
475 ip link add link "${v.interface}" name "${n}" type vlan id "${toString v.id}"
476
477 # We try to bring up the logical VLAN interface. If the master
478 # interface the logical interface is dependent upon is not up yet we will
479 # fail to immediately bring up the logical interface. The resulting logical
480 # interface will brought up later when the master interface is up.
481 ip link set "${n}" up || true
482 '';
483 postStop = ''
484 ip link delete "${n}" || true
485 '';
486 });
487
488 in listToAttrs (
489 map configureAddrs interfaces ++
490 map createTunDevice (filter (i: i.virtual) interfaces))
491 // mapAttrs' createBridgeDevice cfg.bridges
492 // mapAttrs' createVswitchDevice cfg.vswitches
493 // mapAttrs' createBondDevice cfg.bonds
494 // mapAttrs' createMacvlanDevice cfg.macvlans
495 // mapAttrs' createSitDevice cfg.sits
496 // mapAttrs' createVlanDevice cfg.vlans
497 // {
498 "network-setup" = networkSetup;
499 "network-local-commands" = networkLocalCommands;
500 };
501
502 services.udev.extraRules =
503 ''
504 KERNEL=="tun", TAG+="systemd"
505 '';
506
507
508 };
509
510in
511
512{
513 config = mkMerge [
514 bondWarnings
515 (mkIf (!cfg.useNetworkd) normalConfig)
516 { # Ensure slave interfaces are brought up
517 networking.interfaces = genAttrs slaves (i: {});
518 }
519 ];
520}