Merge pull request #302300 from Ma27/kernel-zstd

linux kernel: prefer zstd where possible

Changed files
+82 -43
nixos
doc
manual
release-notes
modules
services
hardware
pkgs
+4
nixos/doc/manual/release-notes/rl-2405.section.md
···
- `system.etc.overlay.enable` option was added. If enabled, `/etc` is
mounted via an overlayfs instead of being created by a custom perl script.
+
- For each supporting version of the Linux kernel firmware blobs and kernel modules
+
are compressed with zstd. For firmware blobs this means an increase of 4.4% in size, however
+
a significantly higher decompression speed.
+
- NixOS AMIs are now uploaded regularly to a new AWS Account.
Instructions on how to use them can be found on <https://nixos.github.io/amis>.
We are working on integration the data into the NixOS homepage.
+10 -4
nixos/modules/services/hardware/udev.nix
···
mv etc/udev/hwdb.bin $out
'';
-
compressFirmware = firmware: if (config.boot.kernelPackages.kernelAtLeast "5.3" && (firmware.compressFirmware or true)) then
-
pkgs.compressFirmwareXz firmware
-
else
-
id firmware;
+
compressFirmware = firmware:
+
let
+
inherit (config.boot.kernelPackages) kernelAtLeast;
+
in
+
if ! (firmware.compressFirmware or true) then
+
firmware
+
else
+
if kernelAtLeast "5.19" then pkgs.compressFirmwareZstd firmware
+
else if kernelAtLeast "5.3" then pkgs.compressFirmwareXz firmware
+
else firmware;
# Udev has a 512-character limit for ENV{PATH}, so create a symlink
# tree to work around this.
-29
pkgs/build-support/kernel/compress-firmware-xz.nix
···
-
{ runCommand, lib }:
-
-
firmware:
-
-
let
-
args = {
-
allowedRequisites = [];
-
} // lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; };
-
in
-
-
runCommand "${firmware.name}-xz" args ''
-
mkdir -p $out/lib
-
(cd ${firmware} && find lib/firmware -type d -print0) |
-
(cd $out && xargs -0 mkdir -v --)
-
(cd ${firmware} && find lib/firmware -type f -print0) |
-
(cd $out && xargs -0rtP "$NIX_BUILD_CORES" -n1 \
-
sh -c 'xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${firmware}/$1" > "$1.xz"' --)
-
(cd ${firmware} && find lib/firmware -type l) | while read link; do
-
target="$(readlink "${firmware}/$link")"
-
if [ -f "${firmware}/$link" ]; then
-
ln -vs -- "''${target/^${firmware}/$out}.xz" "$out/$link.xz"
-
else
-
ln -vs -- "''${target/^${firmware}/$out}" "$out/$link"
-
fi
-
done
-
-
echo "Checking for broken symlinks:"
-
find -L $out -type l -print -execdir false -- '{}' '+'
-
''
+43
pkgs/build-support/kernel/compress-firmware.nix
···
+
{ runCommand, lib, type ? "zstd", zstd }:
+
+
firmware:
+
+
let
+
compressor = {
+
xz = {
+
ext = "xz";
+
nativeBuildInputs = [ ];
+
cmd = file: target: ''xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${file}" > "${target}"'';
+
};
+
zstd = {
+
ext = "zst";
+
nativeBuildInputs = [ zstd ];
+
cmd = file: target: ''zstd -T1 -19 --long --check -f "${file}" -o "${target}"'';
+
};
+
}.${type} or (throw "Unsupported compressor type for firmware.");
+
+
args = {
+
allowedRequisites = [];
+
inherit (compressor) nativeBuildInputs;
+
} // lib.optionalAttrs (firmware ? meta) { inherit (firmware) meta; };
+
in
+
+
runCommand "${firmware.name}-${type}" args ''
+
mkdir -p $out/lib
+
(cd ${firmware} && find lib/firmware -type d -print0) |
+
(cd $out && xargs -0 mkdir -v --)
+
(cd ${firmware} && find lib/firmware -type f -print0) |
+
(cd $out && xargs -0rtP "$NIX_BUILD_CORES" -n1 \
+
sh -c '${compressor.cmd "${firmware}/$1" "$1.${compressor.ext}"}' --)
+
(cd ${firmware} && find lib/firmware -type l) | while read link; do
+
target="$(readlink "${firmware}/$link")"
+
if [ -f "${firmware}/$link" ]; then
+
ln -vs -- "''${target/^${firmware}/$out}.${compressor.ext}" "$out/$link.${compressor.ext}"
+
else
+
ln -vs -- "''${target/^${firmware}/$out}" "$out/$link"
+
fi
+
done
+
+
echo "Checking for broken symlinks:"
+
find -L $out -type l -print -execdir false -- '{}' '+'
+
''
+1 -1
pkgs/build-support/kernel/modules-closure.sh
···
# of its output.
modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:' | while read -r i; do
echo "firmware for $module: $i"
-
for name in "$i" "$i.xz" ""; do
+
for name in "$i" "$i.xz" "$i.zst" ""; do
[ -z "$name" ] && echo "WARNING: missing firmware $i for module $module"
if cp -v --parents --no-preserve=mode lib/firmware/$name "$out" 2>/dev/null; then
break
+9
pkgs/build-support/vm/default.nix
···
${pkgs.stdenv.cc.libc}/lib/libc.so.* \
${pkgs.stdenv.cc.libc}/lib/libm.so.* \
${pkgs.stdenv.cc.libc}/lib/libresolv.so.* \
+
${pkgs.stdenv.cc.libc}/lib/libpthread.so.* \
+
${pkgs.zstd.out}/lib/libzstd.so.* \
+
${pkgs.xz.out}/lib/liblzma.so.* \
$out/lib
# Copy BusyBox.
cp -pd ${pkgs.busybox}/bin/* $out/bin
+
cp -pd ${pkgs.kmod}/bin/* $out/bin
# Run patchelf to make the programs refer to the copied libraries.
for i in $out/bin/* $out/lib/*; do if ! test -L $i; then nuke-refs $i; fi; done
···
echo "patching $i..."
patchelf --set-interpreter $out/lib/ld-*.so.? --set-rpath $out/lib $i || true
fi
+
done
+
+
find $out/lib -type f \! -name 'ld*.so.?' | while read i; do
+
echo "patching $i..."
+
patchelf --set-rpath $out/lib $i
done
''; # */
+12 -8
pkgs/os-specific/linux/kernel/common-config.nix
···
};
zram = {
-
ZRAM = module;
-
ZRAM_WRITEBACK = option yes;
-
ZRAM_MULTI_COMP = whenAtLeast "6.2" yes;
-
ZSWAP = option yes;
-
ZPOOL = yes;
-
ZBUD = option yes;
+
ZRAM = module;
+
ZRAM_WRITEBACK = option yes;
+
ZRAM_MULTI_COMP = whenAtLeast "6.2" yes;
+
ZRAM_DEF_COMP_ZSTD = whenAtLeast "5.11" yes;
+
ZSWAP = option yes;
+
ZSWAP_COMPRESSOR_DEFAULT_ZSTD = whenAtLeast "5.7" yes;
+
ZPOOL = yes;
+
ZSMALLOC = option yes;
};
brcmfmac = {
···
THRUSTMASTER_FF = yes;
ZEROPLUS_FF = yes;
-
MODULE_COMPRESS = whenOlder "5.13" yes;
-
MODULE_COMPRESS_XZ = yes;
+
MODULE_COMPRESS = whenOlder "5.13" yes;
+
MODULE_COMPRESS_XZ = whenOlder "5.13" yes;
+
MODULE_COMPRESS_ZSTD = whenAtLeast "5.13" yes;
SYSVIPC = yes; # System-V IPC
···
FW_LOADER_USER_HELPER_FALLBACK = option no;
FW_LOADER_COMPRESS = whenAtLeast "5.3" yes;
+
FW_LOADER_COMPRESS_ZSTD = whenAtLeast "5.19" yes;
HOTPLUG_PCI_ACPI = yes; # PCI hotplug using ACPI
HOTPLUG_PCI_PCIE = yes; # PCI-Expresscard hotplug support
+3 -1
pkgs/top-level/all-packages.nix
···
makeBinaryWrapper = callPackage ../build-support/setup-hooks/make-binary-wrapper { };
-
compressFirmwareXz = callPackage ../build-support/kernel/compress-firmware-xz.nix { };
+
compressFirmwareXz = callPackage ../build-support/kernel/compress-firmware.nix { type = "xz"; };
+
+
compressFirmwareZstd = callPackage ../build-support/kernel/compress-firmware.nix { type = "zstd"; };
makeModulesClosure = { kernel, firmware, rootModules, allowMissing ? false }:
callPackage ../build-support/kernel/modules-closure.nix {