pyrox.dev
1{ config, lib, pkgs, utils, ... }:
2
3with utils;
4with lib;
5
6let
7
8 cfg = config.networking;
9 interfaces = attrValues cfg.interfaces;
10
11 slaves = concatMap (i: i.interfaces) (attrValues cfg.bonds)
12 ++ concatMap (i: i.interfaces) (attrValues cfg.bridges)
13 ++ concatMap (i: attrNames (filterAttrs (_: config: config.type != "internal") i.interfaces)) (attrValues cfg.vswitches)
14 ++ concatMap (i: [i.interface]) (attrValues cfg.macvlans)
15 ++ concatMap (i: [i.interface]) (attrValues cfg.vlans);
16
17 # We must escape interfaces due to the systemd interpretation
18 subsystemDevice = interface:
19 "sys-subsystem-net-devices-${escapeSystemdPath interface}.device";
20
21 interfaceIps = i:
22 i.ipv4.addresses
23 ++ optionals cfg.enableIPv6 i.ipv6.addresses;
24
25 destroyBond = i: ''
26 while true; do
27 UPDATED=1
28 SLAVES=$(ip link | grep 'master ${i}' | awk -F: '{print $2}')
29 for I in $SLAVES; do
30 UPDATED=0
31 ip link set "$I" nomaster
32 done
33 [ "$UPDATED" -eq "1" ] && break
34 done
35 ip link set "${i}" down 2>/dev/null || true
36 ip link del "${i}" 2>/dev/null || true
37 '';
38
39 # warn that these attributes are deprecated (2017-2-2)
40 # Should be removed in the release after next
41 bondDeprecation = rec {
42 deprecated = [ "lacp_rate" "miimon" "mode" "xmit_hash_policy" ];
43 filterDeprecated = bond: (filterAttrs (attrName: attr:
44 elem attrName deprecated && attr != null) bond);
45 };
46
47 bondWarnings =
48 let oneBondWarnings = bondName: bond:
49 mapAttrsToList (bondText bondName) (bondDeprecation.filterDeprecated bond);
50 bondText = bondName: optName: _:
51 "${bondName}.${optName} is deprecated, use ${bondName}.driverOptions";
52 in {
53 warnings = flatten (mapAttrsToList oneBondWarnings cfg.bonds);
54 };
55
56 normalConfig = {
57 systemd.network.links = let
58 createNetworkLink = i: nameValuePair "40-${i.name}" {
59 matchConfig.OriginalName = i.name;
60 linkConfig = optionalAttrs (i.macAddress != null) {
61 MACAddress = i.macAddress;
62 } // optionalAttrs (i.mtu != null) {
63 MTUBytes = toString i.mtu;
64 };
65 };
66 in listToAttrs (map createNetworkLink interfaces);
67 systemd.services =
68 let
69
70 deviceDependency = dev:
71 # Use systemd service if we manage device creation, else
72 # trust udev when not in a container
73 if (hasAttr dev (filterAttrs (k: v: v.virtual) cfg.interfaces)) ||
74 (hasAttr dev cfg.bridges) ||
75 (hasAttr dev cfg.bonds) ||
76 (hasAttr dev cfg.macvlans) ||
77 (hasAttr dev cfg.sits) ||
78 (hasAttr dev cfg.vlans) ||
79 (hasAttr dev cfg.vswitches)
80 then [ "${dev}-netdev.service" ]
81 else optional (dev != null && dev != "lo" && !config.boot.isContainer) (subsystemDevice dev);
82
83 hasDefaultGatewaySet = (cfg.defaultGateway != null && cfg.defaultGateway.address != "")
84 || (cfg.enableIPv6 && cfg.defaultGateway6 != null && cfg.defaultGateway6.address != "");
85
86 networkLocalCommands = {
87 after = [ "network-setup.service" ];
88 bindsTo = [ "network-setup.service" ];
89 };
90
91 networkSetup =
92 { description = "Networking Setup";
93
94 after = [ "network-pre.target" "systemd-udevd.service" "systemd-sysctl.service" ];
95 before = [ "network.target" "shutdown.target" ];
96 wants = [ "network.target" ];
97 # exclude bridges from the partOf relationship to fix container networking bug #47210
98 partOf = map (i: "network-addresses-${i.name}.service") (filter (i: !(hasAttr i.name cfg.bridges)) interfaces);
99 conflicts = [ "shutdown.target" ];
100 wantedBy = [ "multi-user.target" ] ++ optional hasDefaultGatewaySet "network-online.target";
101
102 unitConfig.ConditionCapability = "CAP_NET_ADMIN";
103
104 path = [ pkgs.iproute2 ];
105
106 serviceConfig = {
107 Type = "oneshot";
108 RemainAfterExit = true;
109 };
110
111 unitConfig.DefaultDependencies = false;
112
113 script =
114 ''
115 ${optionalString config.networking.resolvconf.enable ''
116 # Set the static DNS configuration, if given.
117 ${pkgs.openresolv}/sbin/resolvconf -m 1 -a static <<EOF
118 ${optionalString (cfg.nameservers != [] && cfg.domain != null) ''
119 domain ${cfg.domain}
120 ''}
121 ${optionalString (cfg.search != []) ("search " + concatStringsSep " " cfg.search)}
122 ${flip concatMapStrings cfg.nameservers (ns: ''
123 nameserver ${ns}
124 '')}
125 EOF
126 ''}
127
128 # Set the default gateway.
129 ${optionalString (cfg.defaultGateway != null && cfg.defaultGateway.address != "") ''
130 ${optionalString (cfg.defaultGateway.interface != null) ''
131 ip route replace ${cfg.defaultGateway.address} dev ${cfg.defaultGateway.interface} ${optionalString (cfg.defaultGateway.metric != null)
132 "metric ${toString cfg.defaultGateway.metric}"
133 } proto static
134 ''}
135 ip route replace default ${optionalString (cfg.defaultGateway.metric != null)
136 "metric ${toString cfg.defaultGateway.metric}"
137 } via "${cfg.defaultGateway.address}" ${
138 optionalString (cfg.defaultGatewayWindowSize != null)
139 "window ${toString cfg.defaultGatewayWindowSize}"} ${
140 optionalString (cfg.defaultGateway.interface != null)
141 "dev ${cfg.defaultGateway.interface}"} proto static
142 ''}
143 ${optionalString (cfg.defaultGateway6 != null && cfg.defaultGateway6.address != "") ''
144 ${optionalString (cfg.defaultGateway6.interface != null) ''
145 ip -6 route replace ${cfg.defaultGateway6.address} dev ${cfg.defaultGateway6.interface} ${optionalString (cfg.defaultGateway6.metric != null)
146 "metric ${toString cfg.defaultGateway6.metric}"
147 } proto static
148 ''}
149 ip -6 route replace default ${optionalString (cfg.defaultGateway6.metric != null)
150 "metric ${toString cfg.defaultGateway6.metric}"
151 } via "${cfg.defaultGateway6.address}" ${
152 optionalString (cfg.defaultGatewayWindowSize != null)
153 "window ${toString cfg.defaultGatewayWindowSize}"} ${
154 optionalString (cfg.defaultGateway6.interface != null)
155 "dev ${cfg.defaultGateway6.interface}"} proto static
156 ''}
157 '';
158 };
159
160 # For each interface <foo>, create a job ‘network-addresses-<foo>.service"
161 # that performs static address configuration. It has a "wants"
162 # dependency on ‘<foo>.service’, which is supposed to create
163 # the interface and need not exist (i.e. for hardware
164 # interfaces). It has a binds-to dependency on the actual
165 # network device, so it only gets started after the interface
166 # has appeared, and it's stopped when the interface
167 # disappears.
168 configureAddrs = i:
169 let
170 ips = interfaceIps i;
171 in
172 nameValuePair "network-addresses-${i.name}"
173 { description = "Address configuration of ${i.name}";
174 wantedBy = [
175 "network-setup.service"
176 "network.target"
177 ];
178 # order before network-setup because the routes that are configured
179 # there may need ip addresses configured
180 before = [ "network-setup.service" ];
181 bindsTo = deviceDependency i.name;
182 after = [ "network-pre.target" ] ++ (deviceDependency i.name);
183 serviceConfig.Type = "oneshot";
184 serviceConfig.RemainAfterExit = true;
185 # Restart rather than stop+start this unit to prevent the
186 # network from dying during switch-to-configuration.
187 stopIfChanged = false;
188 path = [ pkgs.iproute2 ];
189 script =
190 ''
191 state="/run/nixos/network/addresses/${i.name}"
192 mkdir -p $(dirname "$state")
193
194 ip link set "${i.name}" up
195
196 ${flip concatMapStrings ips (ip:
197 let
198 cidr = "${ip.address}/${toString ip.prefixLength}";
199 in
200 ''
201 echo "${cidr}" >> $state
202 echo -n "adding address ${cidr}... "
203 if out=$(ip addr add "${cidr}" dev "${i.name}" 2>&1); then
204 echo "done"
205 elif ! echo "$out" | grep "File exists" >/dev/null 2>&1; then
206 echo "'ip addr add "${cidr}" dev "${i.name}"' failed: $out"
207 exit 1
208 fi
209 ''
210 )}
211
212 state="/run/nixos/network/routes/${i.name}"
213 mkdir -p $(dirname "$state")
214
215 ${flip concatMapStrings (i.ipv4.routes ++ i.ipv6.routes) (route:
216 let
217 cidr = "${route.address}/${toString route.prefixLength}";
218 via = optionalString (route.via != null) ''via "${route.via}"'';
219 options = concatStrings (mapAttrsToList (name: val: "${name} ${val} ") route.options);
220 in
221 ''
222 echo "${cidr}" >> $state
223 echo -n "adding route ${cidr}... "
224 if out=$(ip route add "${cidr}" ${options} ${via} dev "${i.name}" proto static 2>&1); then
225 echo "done"
226 elif ! echo "$out" | grep "File exists" >/dev/null 2>&1; then
227 echo "'ip route add "${cidr}" ${options} ${via} dev "${i.name}"' failed: $out"
228 exit 1
229 fi
230 ''
231 )}
232 '';
233 preStop = ''
234 state="/run/nixos/network/routes/${i.name}"
235 if [ -e "$state" ]; then
236 while read cidr; do
237 echo -n "deleting route $cidr... "
238 ip route del "$cidr" dev "${i.name}" >/dev/null 2>&1 && echo "done" || echo "failed"
239 done < "$state"
240 rm -f "$state"
241 fi
242
243 state="/run/nixos/network/addresses/${i.name}"
244 if [ -e "$state" ]; then
245 while read cidr; do
246 echo -n "deleting address $cidr... "
247 ip addr del "$cidr" dev "${i.name}" >/dev/null 2>&1 && echo "done" || echo "failed"
248 done < "$state"
249 rm -f "$state"
250 fi
251 '';
252 };
253
254 createTunDevice = i: nameValuePair "${i.name}-netdev"
255 { description = "Virtual Network Interface ${i.name}";
256 bindsTo = optional (!config.boot.isContainer) "dev-net-tun.device";
257 after = optional (!config.boot.isContainer) "dev-net-tun.device" ++ [ "network-pre.target" ];
258 wantedBy = [ "network-setup.service" (subsystemDevice i.name) ];
259 partOf = [ "network-setup.service" ];
260 before = [ "network-setup.service" ];
261 path = [ pkgs.iproute2 ];
262 serviceConfig = {
263 Type = "oneshot";
264 RemainAfterExit = true;
265 };
266 script = ''
267 ip tuntap add dev "${i.name}" mode "${i.virtualType}" user "${i.virtualOwner}"
268 '';
269 postStop = ''
270 ip link del ${i.name} || true
271 '';
272 };
273
274 createBridgeDevice = n: v: nameValuePair "${n}-netdev"
275 (let
276 deps = concatLists (map deviceDependency v.interfaces);
277 in
278 { description = "Bridge Interface ${n}";
279 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
280 bindsTo = deps ++ optional v.rstp "mstpd.service";
281 partOf = [ "network-setup.service" ] ++ optional v.rstp "mstpd.service";
282 after = [ "network-pre.target" ] ++ deps ++ optional v.rstp "mstpd.service"
283 ++ map (i: "network-addresses-${i}.service") v.interfaces;
284 before = [ "network-setup.service" ];
285 serviceConfig.Type = "oneshot";
286 serviceConfig.RemainAfterExit = true;
287 path = [ pkgs.iproute2 ];
288 script = ''
289 # Remove Dead Interfaces
290 echo "Removing old bridge ${n}..."
291 ip link show "${n}" >/dev/null 2>&1 && ip link del "${n}"
292
293 echo "Adding bridge ${n}..."
294 ip link add name "${n}" type bridge
295
296 # Enslave child interfaces
297 ${flip concatMapStrings v.interfaces (i: ''
298 ip link set "${i}" master "${n}"
299 ip link set "${i}" up
300 '')}
301 # Save list of enslaved interfaces
302 echo "${flip concatMapStrings v.interfaces (i: ''
303 ${i}
304 '')}" > /run/${n}.interfaces
305
306 ${optionalString config.virtualisation.libvirtd.enable ''
307 # Enslave dynamically added interfaces which may be lost on nixos-rebuild
308 #
309 # if `libvirtd.service` is not running, do not use `virsh` which would try activate it via 'libvirtd.socket' and thus start it out-of-order.
310 # `libvirtd.service` will set up bridge interfaces when it will start normally.
311 #
312 if /run/current-system/systemd/bin/systemctl --quiet is-active 'libvirtd.service'; then
313 for uri in qemu:///system lxc:///; do
314 for dom in $(${pkgs.libvirt}/bin/virsh -c $uri list --name); do
315 ${pkgs.libvirt}/bin/virsh -c $uri dumpxml "$dom" | \
316 ${pkgs.xmlstarlet}/bin/xmlstarlet sel -t -m "//domain/devices/interface[@type='bridge'][source/@bridge='${n}'][target/@dev]" -v "concat('ip link set ',target/@dev,' master ',source/@bridge,';')" | \
317 ${pkgs.bash}/bin/bash
318 done
319 done
320 fi
321 ''}
322
323 # Enable stp on the interface
324 ${optionalString v.rstp ''
325 echo 2 >/sys/class/net/${n}/bridge/stp_state
326 ''}
327
328 ip link set "${n}" up
329 '';
330 postStop = ''
331 ip link set "${n}" down || true
332 ip link del "${n}" || true
333 rm -f /run/${n}.interfaces
334 '';
335 reload = ''
336 # Un-enslave child interfaces (old list of interfaces)
337 for interface in `cat /run/${n}.interfaces`; do
338 ip link set "$interface" nomaster up
339 done
340
341 # Enslave child interfaces (new list of interfaces)
342 ${flip concatMapStrings v.interfaces (i: ''
343 ip link set "${i}" master "${n}"
344 ip link set "${i}" up
345 '')}
346 # Save list of enslaved interfaces
347 echo "${flip concatMapStrings v.interfaces (i: ''
348 ${i}
349 '')}" > /run/${n}.interfaces
350
351 # (Un-)set stp on the bridge
352 echo ${if v.rstp then "2" else "0"} > /sys/class/net/${n}/bridge/stp_state
353 '';
354 reloadIfChanged = true;
355 });
356
357 createVswitchDevice = n: v: nameValuePair "${n}-netdev"
358 (let
359 deps = concatLists (map deviceDependency (attrNames (filterAttrs (_: config: config.type != "internal") v.interfaces)));
360 internalConfigs = map (i: "network-addresses-${i}.service") (attrNames (filterAttrs (_: config: config.type == "internal") v.interfaces));
361 ofRules = pkgs.writeText "vswitch-${n}-openFlowRules" v.openFlowRules;
362 in
363 { description = "Open vSwitch Interface ${n}";
364 wantedBy = [ "network-setup.service" (subsystemDevice n) ] ++ internalConfigs;
365 # before = [ "network-setup.service" ];
366 # should work without internalConfigs dependencies because address/link configuration depends
367 # on the device, which is created by ovs-vswitchd with type=internal, but it does not...
368 before = [ "network-setup.service" ] ++ internalConfigs;
369 partOf = [ "network-setup.service" ]; # shutdown the bridge when network is shutdown
370 bindsTo = [ "ovs-vswitchd.service" ]; # requires ovs-vswitchd to be alive at all times
371 after = [ "network-pre.target" "ovs-vswitchd.service" ] ++ deps; # start switch after physical interfaces and vswitch daemon
372 wants = deps; # if one or more interface fails, the switch should continue to run
373 serviceConfig.Type = "oneshot";
374 serviceConfig.RemainAfterExit = true;
375 path = [ pkgs.iproute2 config.virtualisation.vswitch.package ];
376 preStart = ''
377 echo "Resetting Open vSwitch ${n}..."
378 ovs-vsctl --if-exists del-br ${n} -- add-br ${n} \
379 -- set bridge ${n} protocols=${concatStringsSep "," v.supportedOpenFlowVersions}
380 '';
381 script = ''
382 echo "Configuring Open vSwitch ${n}..."
383 ovs-vsctl ${concatStrings (mapAttrsToList (name: config: " -- add-port ${n} ${name}" + optionalString (config.vlan != null) " tag=${toString config.vlan}") v.interfaces)} \
384 ${concatStrings (mapAttrsToList (name: config: optionalString (config.type != null) " -- set interface ${name} type=${config.type}") v.interfaces)} \
385 ${concatMapStrings (x: " -- set-controller ${n} " + x) v.controllers} \
386 ${concatMapStrings (x: " -- " + x) (splitString "\n" v.extraOvsctlCmds)}
387
388
389 echo "Adding OpenFlow rules for Open vSwitch ${n}..."
390 ovs-ofctl --protocols=${v.openFlowVersion} add-flows ${n} ${ofRules}
391 '';
392 postStop = ''
393 echo "Cleaning Open vSwitch ${n}"
394 echo "Shuting down internal ${n} interface"
395 ip link set ${n} down || true
396 echo "Deleting flows for ${n}"
397 ovs-ofctl --protocols=${v.openFlowVersion} del-flows ${n} || true
398 echo "Deleting Open vSwitch ${n}"
399 ovs-vsctl --if-exists del-br ${n} || true
400 '';
401 });
402
403 createBondDevice = n: v: nameValuePair "${n}-netdev"
404 (let
405 deps = concatLists (map deviceDependency v.interfaces);
406 in
407 { description = "Bond Interface ${n}";
408 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
409 bindsTo = deps;
410 partOf = [ "network-setup.service" ];
411 after = [ "network-pre.target" ] ++ deps
412 ++ map (i: "network-addresses-${i}.service") v.interfaces;
413 before = [ "network-setup.service" ];
414 serviceConfig.Type = "oneshot";
415 serviceConfig.RemainAfterExit = true;
416 path = [ pkgs.iproute2 pkgs.gawk ];
417 script = ''
418 echo "Destroying old bond ${n}..."
419 ${destroyBond n}
420
421 echo "Creating new bond ${n}..."
422 ip link add name "${n}" type bond \
423 ${let opts = (mapAttrs (const toString)
424 (bondDeprecation.filterDeprecated v))
425 // v.driverOptions;
426 in concatStringsSep "\n"
427 (mapAttrsToList (set: val: " ${set} ${val} \\") opts)}
428
429 # !!! There must be a better way to wait for the interface
430 while [ ! -d "/sys/class/net/${n}" ]; do sleep 0.1; done;
431
432 # Bring up the bond and enslave the specified interfaces
433 ip link set "${n}" up
434 ${flip concatMapStrings v.interfaces (i: ''
435 ip link set "${i}" down
436 ip link set "${i}" master "${n}"
437 '')}
438 '';
439 postStop = destroyBond n;
440 });
441
442 createMacvlanDevice = n: v: nameValuePair "${n}-netdev"
443 (let
444 deps = deviceDependency v.interface;
445 in
446 { description = "Vlan Interface ${n}";
447 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
448 bindsTo = deps;
449 partOf = [ "network-setup.service" ];
450 after = [ "network-pre.target" ] ++ deps;
451 before = [ "network-setup.service" ];
452 serviceConfig.Type = "oneshot";
453 serviceConfig.RemainAfterExit = true;
454 path = [ pkgs.iproute2 ];
455 script = ''
456 # Remove Dead Interfaces
457 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
458 ip link add link "${v.interface}" name "${n}" type macvlan \
459 ${optionalString (v.mode != null) "mode ${v.mode}"}
460 ip link set "${n}" up
461 '';
462 postStop = ''
463 ip link delete "${n}" || true
464 '';
465 });
466
467 createSitDevice = n: v: nameValuePair "${n}-netdev"
468 (let
469 deps = deviceDependency v.dev;
470 in
471 { description = "6-to-4 Tunnel Interface ${n}";
472 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
473 bindsTo = deps;
474 partOf = [ "network-setup.service" ];
475 after = [ "network-pre.target" ] ++ deps;
476 before = [ "network-setup.service" ];
477 serviceConfig.Type = "oneshot";
478 serviceConfig.RemainAfterExit = true;
479 path = [ pkgs.iproute2 ];
480 script = ''
481 # Remove Dead Interfaces
482 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
483 ip link add name "${n}" type sit \
484 ${optionalString (v.remote != null) "remote \"${v.remote}\""} \
485 ${optionalString (v.local != null) "local \"${v.local}\""} \
486 ${optionalString (v.ttl != null) "ttl ${toString v.ttl}"} \
487 ${optionalString (v.dev != null) "dev \"${v.dev}\""}
488 ip link set "${n}" up
489 '';
490 postStop = ''
491 ip link delete "${n}" || true
492 '';
493 });
494
495 createVlanDevice = n: v: nameValuePair "${n}-netdev"
496 (let
497 deps = deviceDependency v.interface;
498 in
499 { description = "Vlan Interface ${n}";
500 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
501 bindsTo = deps;
502 partOf = [ "network-setup.service" ];
503 after = [ "network-pre.target" ] ++ deps;
504 before = [ "network-setup.service" ];
505 serviceConfig.Type = "oneshot";
506 serviceConfig.RemainAfterExit = true;
507 path = [ pkgs.iproute2 ];
508 script = ''
509 # Remove Dead Interfaces
510 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
511 ip link add link "${v.interface}" name "${n}" type vlan id "${toString v.id}"
512
513 # We try to bring up the logical VLAN interface. If the master
514 # interface the logical interface is dependent upon is not up yet we will
515 # fail to immediately bring up the logical interface. The resulting logical
516 # interface will brought up later when the master interface is up.
517 ip link set "${n}" up || true
518 '';
519 postStop = ''
520 ip link delete "${n}" || true
521 '';
522 });
523
524 in listToAttrs (
525 map configureAddrs interfaces ++
526 map createTunDevice (filter (i: i.virtual) interfaces))
527 // mapAttrs' createBridgeDevice cfg.bridges
528 // mapAttrs' createVswitchDevice cfg.vswitches
529 // mapAttrs' createBondDevice cfg.bonds
530 // mapAttrs' createMacvlanDevice cfg.macvlans
531 // mapAttrs' createSitDevice cfg.sits
532 // mapAttrs' createVlanDevice cfg.vlans
533 // {
534 network-setup = networkSetup;
535 network-local-commands = networkLocalCommands;
536 };
537
538 services.udev.extraRules =
539 ''
540 KERNEL=="tun", TAG+="systemd"
541 '';
542
543
544 };
545
546in
547
548{
549 config = mkMerge [
550 bondWarnings
551 (mkIf (!cfg.useNetworkd) normalConfig)
552 { # Ensure slave interfaces are brought up
553 networking.interfaces = genAttrs slaves (i: {});
554 }
555 ];
556}