Merge pull request #75584 from Infinisil/settings-formats

Configuration file formats for JSON, INI, YAML and TOML

Changed files
+489 -2
lib
nixos
doc
pkgs
+4 -2
lib/generators.nix
···
else if isAttrs v then err "attrsets" v
# functions can’t be printed of course
else if isFunction v then err "functions" v
-
# let’s not talk about floats. There is no sensible `toString` for them.
-
else if isFloat v then err "floats" v
else err "this value is" (toString v);
···
else if isAttrs v then err "attrsets" v
# functions can’t be printed of course
else if isFunction v then err "functions" v
+
# Floats currently can't be converted to precise strings,
+
# condition warning on nix version once this isn't a problem anymore
+
# See https://github.com/NixOS/nix/pull/3480
+
else if isFloat v then libStr.floatToString v
else err "this value is" (toString v);
+16
lib/strings.nix
···
*/
fixedWidthNumber = width: n: fixedWidthString width "0" (toString n);
/* Check whether a value can be coerced to a string */
isCoercibleToString = x:
builtins.elem (builtins.typeOf x) [ "path" "string" "null" "int" "float" "bool" ] ||
···
*/
fixedWidthNumber = width: n: fixedWidthString width "0" (toString n);
+
/* Convert a float to a string, but emit a warning when precision is lost
+
during the conversion
+
+
Example:
+
floatToString 0.000001
+
=> "0.000001"
+
floatToString 0.0000001
+
=> trace: warning: Imprecise conversion from float to string 0.000000
+
"0.000000"
+
*/
+
floatToString = float: let
+
result = toString float;
+
precise = float == builtins.fromJSON result;
+
in if precise then result
+
else lib.warn "Imprecise conversion from float to string ${result}" result;
+
/* Check whether a value can be coerced to a string */
isCoercibleToString = x:
builtins.elem (builtins.typeOf x) [ "path" "string" "null" "int" "float" "bool" ] ||
+179
nixos/doc/manual/development/settings-options.xml
···
···
+
<section xmlns="http://docbook.org/ns/docbook"
+
xmlns:xlink="http://www.w3.org/1999/xlink"
+
xmlns:xi="http://www.w3.org/2001/XInclude"
+
version="5.0"
+
xml:id="sec-settings-options">
+
<title>Options for Program Settings</title>
+
+
<para>
+
Many programs have configuration files where program-specific settings can be declared. File formats can be separated into two categories:
+
<itemizedlist>
+
<listitem>
+
<para>
+
Nix-representable ones: These can trivially be mapped to a subset of Nix syntax. E.g. JSON is an example, since its values like <literal>{"foo":{"bar":10}}</literal> can be mapped directly to Nix: <literal>{ foo = { bar = 10; }; }</literal>. Other examples are INI, YAML and TOML. The following section explains the convention for these settings.
+
</para>
+
</listitem>
+
<listitem>
+
<para>
+
Non-nix-representable ones: These can't be trivially mapped to a subset of Nix syntax. Most generic programming languages are in this group, e.g. bash, since the statement <literal>if true; then echo hi; fi</literal> doesn't have a trivial representation in Nix.
+
</para>
+
<para>
+
Currently there are no fixed conventions for these, but it is common to have a <literal>configFile</literal> option for setting the configuration file path directly. The default value of <literal>configFile</literal> can be an auto-generated file, with convenient options for controlling the contents. For example an option of type <literal>attrsOf str</literal> can be used for representing environment variables which generates a section like <literal>export FOO="foo"</literal>. Often it can also be useful to also include an <literal>extraConfig</literal> option of type <literal>lines</literal> to allow arbitrary text after the autogenerated part of the file.
+
</para>
+
</listitem>
+
</itemizedlist>
+
</para>
+
<section xml:id="sec-settings-nix-representable">
+
<title>Nix-representable Formats (JSON, YAML, TOML, INI, ...)</title>
+
<para>
+
By convention, formats like this are handled with a generic <literal>settings</literal> option, representing the full program configuration as a Nix value. The type of this option should represent the format. The most common formats have a predefined type and string generator already declared under <literal>pkgs.formats</literal>:
+
<variablelist>
+
<varlistentry>
+
<term>
+
<varname>pkgs.formats.json</varname> { }
+
</term>
+
<listitem>
+
<para>
+
A function taking an empty attribute set (for future extensibility) and returning a set with JSON-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+
</para>
+
</listitem>
+
</varlistentry>
+
<varlistentry>
+
<term>
+
<varname>pkgs.formats.yaml</varname> { }
+
</term>
+
<listitem>
+
<para>
+
A function taking an empty attribute set (for future extensibility) and returning a set with YAML-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+
</para>
+
</listitem>
+
</varlistentry>
+
<varlistentry>
+
<term>
+
<varname>pkgs.formats.ini</varname> { <replaceable>listsAsDuplicateKeys</replaceable> ? false, ... }
+
</term>
+
<listitem>
+
<para>
+
A function taking an attribute set with values
+
<variablelist>
+
<varlistentry>
+
<term>
+
<varname>listsAsDuplicateKeys</varname>
+
</term>
+
<listitem>
+
<para>
+
A boolean for controlling whether list values can be used to represent duplicate INI keys
+
</para>
+
</listitem>
+
</varlistentry>
+
</variablelist>
+
It returns a set with INI-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+
</para>
+
</listitem>
+
</varlistentry>
+
<varlistentry>
+
<term>
+
<varname>pkgs.formats.toml</varname> { }
+
</term>
+
<listitem>
+
<para>
+
A function taking an empty attribute set (for future extensibility) and returning a set with TOML-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+
</para>
+
</listitem>
+
</varlistentry>
+
</variablelist>
+
+
</para>
+
<para xml:id="pkgs-formats-result">
+
These functions all return an attribute set with these values:
+
<variablelist>
+
<varlistentry>
+
<term>
+
<varname>type</varname>
+
</term>
+
<listitem>
+
<para>
+
A module system type representing a value of the format
+
</para>
+
</listitem>
+
</varlistentry>
+
<varlistentry>
+
<term>
+
<varname>generate</varname> <replaceable>filename</replaceable> <replaceable>jsonValue</replaceable>
+
</term>
+
<listitem>
+
<para>
+
A function that can render a value of the format to a file. Returns a file path.
+
<note>
+
<para>
+
This function puts the value contents in the Nix store. So this should be avoided for secrets.
+
</para>
+
</note>
+
</para>
+
</listitem>
+
</varlistentry>
+
</variablelist>
+
</para>
+
<example xml:id="ex-settings-nix-representable">
+
<title>Module with conventional <literal>settings</literal> option</title>
+
<para>
+
The following shows a module for an example program that uses a JSON configuration file. It demonstrates how above values can be used, along with some other related best practices. See the comments for explanations.
+
</para>
+
<programlisting>
+
{ options, config, lib, pkgs, ... }:
+
let
+
cfg = config.services.foo;
+
# Define the settings format used for this program
+
settingsFormat = pkgs.formats.json {};
+
in {
+
+
options.services.foo = {
+
enable = lib.mkEnableOption "foo service";
+
+
settings = lib.mkOption {
+
# Setting this type allows for correct merging behavior
+
type = settingsFormat.type;
+
default = {};
+
description = ''
+
Configuration for foo, see
+
&lt;link xlink:href="https://example.com/docs/foo"/&gt;
+
for supported values.
+
'';
+
};
+
};
+
+
config = lib.mkIf cfg.enable {
+
# We can assign some default settings here to make the service work by just
+
# enabling it. We use `mkDefault` for values that can be changed without
+
# problems
+
services.foo.settings = {
+
# Fails at runtime without any value set
+
log_level = lib.mkDefault "WARN";
+
+
# We assume systemd's `StateDirectory` is used, so we require this value,
+
# therefore no mkDefault
+
data_path = "/var/lib/foo";
+
+
# Since we use this to create a user we need to know the default value at
+
# eval time
+
user = lib.mkDefault "foo";
+
};
+
+
environment.etc."foo.json".source =
+
# The formats generator function takes a filename and the Nix value
+
# representing the format value and produces a filepath with that value
+
# rendered in the format
+
settingsFormat.generate "foo-config.json" cfg.settings;
+
+
# We know that the `user` attribute exists because we set a default value
+
# for it above, allowing us to use it without worries here
+
users.users.${cfg.settings.user} = {}
+
+
# ...
+
};
+
}
+
</programlisting>
+
</example>
+
</section>
+
+
</section>
+1
nixos/doc/manual/development/writing-modules.xml
···
<xi:include href="meta-attributes.xml" />
<xi:include href="importing-modules.xml" />
<xi:include href="replace-modules.xml" />
</chapter>
···
<xi:include href="meta-attributes.xml" />
<xi:include href="importing-modules.xml" />
<xi:include href="replace-modules.xml" />
+
<xi:include href="settings-options.xml" />
</chapter>
+11
pkgs/pkgs-lib/default.nix
···
···
+
# pkgs-lib is for functions and values that can't be in lib because
+
# they depend on some packages. This notably is *not* for supporting package
+
# building, instead pkgs/build-support is the place for that.
+
{ lib, pkgs }: {
+
# setting format types and generators. These do not fit in lib/types.nix,
+
# because they depend on pkgs for rendering some formats
+
formats = import ./formats.nix {
+
inherit lib pkgs;
+
};
+
}
+
+109
pkgs/pkgs-lib/formats.nix
···
···
+
{ lib, pkgs }:
+
rec {
+
+
/*
+
+
Every following entry represents a format for program configuration files
+
used for `settings`-style options (see https://github.com/NixOS/rfcs/pull/42).
+
Each entry should look as follows:
+
+
<format> = <parameters>: {
+
# ^^ Parameters for controlling the format
+
+
# The module system type most suitable for representing such a format
+
# The description needs to be overwritten for recursive types
+
type = ...;
+
+
# generate :: Name -> Value -> Path
+
# A function for generating a file with a value of such a type
+
generate = ...;
+
+
});
+
*/
+
+
+
json = {}: {
+
+
type = with lib.types; let
+
valueType = nullOr (oneOf [
+
bool
+
int
+
float
+
str
+
(attrsOf valueType)
+
(listOf valueType)
+
]) // {
+
description = "JSON value";
+
};
+
in valueType;
+
+
generate = name: value: pkgs.runCommandNoCC name {
+
nativeBuildInputs = [ pkgs.jq ];
+
value = builtins.toJSON value;
+
passAsFile = [ "value" ];
+
} ''
+
jq . "$valuePath"> $out
+
'';
+
+
};
+
+
# YAML has been a strict superset of JSON since 1.2
+
yaml = {}:
+
let jsonSet = json {};
+
in jsonSet // {
+
type = jsonSet.type // {
+
description = "YAML value";
+
};
+
};
+
+
ini = { listsAsDuplicateKeys ? false, ... }@args: {
+
+
type = with lib.types; let
+
+
singleIniAtom = nullOr (oneOf [
+
bool
+
int
+
float
+
str
+
]) // {
+
description = "INI atom (null, bool, int, float or string)";
+
};
+
+
iniAtom =
+
if listsAsDuplicateKeys then
+
coercedTo singleIniAtom lib.singleton (listOf singleIniAtom) // {
+
description = singleIniAtom.description + " or a list of them for duplicate keys";
+
}
+
else
+
singleIniAtom;
+
+
in attrsOf (attrsOf iniAtom);
+
+
generate = name: value: pkgs.writeText name (lib.generators.toINI args value);
+
+
};
+
+
toml = {}: json {} // {
+
type = with lib.types; let
+
valueType = oneOf [
+
bool
+
int
+
float
+
str
+
(attrsOf valueType)
+
(listOf valueType)
+
] // {
+
description = "TOML value";
+
};
+
in valueType;
+
+
generate = name: value: pkgs.runCommandNoCC name {
+
nativeBuildInputs = [ pkgs.remarshal ];
+
value = builtins.toJSON value;
+
passAsFile = [ "value" ];
+
} ''
+
json2toml "$valuePath" "$out"
+
'';
+
+
};
+
}
+7
pkgs/pkgs-lib/tests/default.nix
···
···
+
# Call nix-build on this file to run all tests in this directory
+
{ pkgs ? import ../../.. {} }:
+
let
+
formats = import ./formats.nix { inherit pkgs; };
+
in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [
+
{ name = "formats"; path = import ./formats.nix { inherit pkgs; }; }
+
]
+157
pkgs/pkgs-lib/tests/formats.nix
···
···
+
{ pkgs }:
+
let
+
inherit (pkgs) lib formats;
+
in
+
with lib;
+
let
+
+
evalFormat = format: args: def:
+
let
+
formatSet = format args;
+
config = formatSet.type.merge [] (imap1 (n: def: {
+
value = def;
+
file = "def${toString n}";
+
}) [ def ]);
+
in formatSet.generate "test-format-file" config;
+
+
runBuildTest = name: { drv, expected }: pkgs.runCommandNoCC name {} ''
+
if diff ${drv} ${builtins.toFile "expected" expected}; then
+
touch $out
+
else
+
echo "Got: $(cat ${drv})"
+
echo "Should be: ${expected}"
+
exit 1
+
fi
+
'';
+
+
runBuildTests = tests: pkgs.linkFarm "nixpkgs-pkgs-lib-format-tests" (mapAttrsToList (name: value: { inherit name; path = runBuildTest name value; }) (filterAttrs (name: value: value != null) tests));
+
+
in runBuildTests {
+
+
testJsonAtoms = {
+
drv = evalFormat formats.json {} {
+
null = null;
+
false = false;
+
true = true;
+
int = 10;
+
float = 3.141;
+
str = "foo";
+
attrs.foo = null;
+
list = [ null null ];
+
};
+
expected = ''
+
{
+
"attrs": {
+
"foo": null
+
},
+
"false": false,
+
"float": 3.141,
+
"int": 10,
+
"list": [
+
null,
+
null
+
],
+
"null": null,
+
"str": "foo",
+
"true": true
+
}
+
'';
+
};
+
+
testYamlAtoms = {
+
drv = evalFormat formats.yaml {} {
+
null = null;
+
false = false;
+
true = true;
+
float = 3.141;
+
str = "foo";
+
attrs.foo = null;
+
list = [ null null ];
+
};
+
expected = ''
+
{
+
"attrs": {
+
"foo": null
+
},
+
"false": false,
+
"float": 3.141,
+
"list": [
+
null,
+
null
+
],
+
"null": null,
+
"str": "foo",
+
"true": true
+
}
+
'';
+
};
+
+
testIniAtoms = {
+
drv = evalFormat formats.ini {} {
+
foo = {
+
bool = true;
+
int = 10;
+
float = 3.141;
+
str = "string";
+
};
+
};
+
expected = ''
+
[foo]
+
bool=true
+
float=3.141000
+
int=10
+
str=string
+
'';
+
};
+
+
testIniDuplicateKeys = {
+
drv = evalFormat formats.ini { listsAsDuplicateKeys = true; } {
+
foo = {
+
bar = [ null true "test" 1.2 10 ];
+
baz = false;
+
qux = "qux";
+
};
+
};
+
expected = ''
+
[foo]
+
bar=null
+
bar=true
+
bar=test
+
bar=1.200000
+
bar=10
+
baz=false
+
qux=qux
+
'';
+
};
+
+
testTomlAtoms = {
+
drv = evalFormat formats.toml {} {
+
false = false;
+
true = true;
+
int = 10;
+
float = 3.141;
+
str = "foo";
+
attrs.foo = "foo";
+
list = [ 1 2 ];
+
level1.level2.level3.level4 = "deep";
+
};
+
expected = ''
+
false = false
+
float = 3.141
+
int = 10
+
list = [1, 2]
+
str = "foo"
+
true = true
+
+
[attrs]
+
foo = "foo"
+
+
[level1]
+
+
[level1.level2]
+
+
[level1.level2.level3]
+
level4 = "deep"
+
'';
+
};
+
}
+3
pkgs/top-level/all-packages.nix
···
#package writers
writers = callPackage ../build-support/writers {};
### TOOLS
_0x0 = callPackage ../tools/misc/0x0 { };
···
#package writers
writers = callPackage ../build-support/writers {};
+
# lib functions depending on pkgs
+
inherit (import ../pkgs-lib { inherit lib pkgs; }) formats;
+
### TOOLS
_0x0 = callPackage ../tools/misc/0x0 { };
+2
pkgs/top-level/release.nix
···
manual = import ../../doc { inherit pkgs nixpkgs; };
lib-tests = import ../../lib/tests/release.nix { inherit pkgs; };
darwin-tested = if supportDarwin then pkgs.releaseTools.aggregate
{ name = "nixpkgs-darwin-${jobs.tarball.version}";
···
[ jobs.tarball
jobs.manual
jobs.lib-tests
jobs.stdenv.x86_64-linux
jobs.linux.x86_64-linux
jobs.pandoc.x86_64-linux
···
manual = import ../../doc { inherit pkgs nixpkgs; };
lib-tests = import ../../lib/tests/release.nix { inherit pkgs; };
+
pkgs-lib-tests = import ../pkgs-lib/tests { inherit pkgs; };
darwin-tested = if supportDarwin then pkgs.releaseTools.aggregate
{ name = "nixpkgs-darwin-${jobs.tarball.version}";
···
[ jobs.tarball
jobs.manual
jobs.lib-tests
+
jobs.pkgs-lib-tests
jobs.stdenv.x86_64-linux
jobs.linux.x86_64-linux
jobs.pandoc.x86_64-linux