Merge pull request #221318 from mweinelt/synapse-extras

nixos/matrix-synapse: Allow passing extras, discover extras from config

Changed files
+219 -45
nixos
doc
manual
release-notes
modules
services
matrix
tests
matrix
pkgs
+7
nixos/doc/manual/release-notes/rl-2311.section.md
···
- The `services.ananicy.extraRules` option now has the type of `listOf attrs` instead of `string`.
+
- The `matrix-synapse` package & module have undergone some significant internal changes, for most setups no intervention is needed, though:
+
- The option [`services.matrix-synapse.package`](#opt-services.matrix-synapse.package) is now read-only. For modifying the package, use an overlay which modifies `matrix-synapse-unwrapped` instead. More on that below.
+
- The `enableSystemd` & `enableRedis` arguments have been removed and `matrix-synapse` has been renamed to `matrix-synapse-unwrapped`. Also, several optional dependencies (such as `psycopg2` or `authlib`) have been removed.
+
- These optional dependencies are automatically added via a wrapper (`pkgs.matrix-synapse.override { extras = ["redis"]; }` for `hiredis` & `txredisapi` for instance) if the relevant config section is declared in `services.matrix-synapse.settings`. For instance, if `services.matrix-synapse.settings.redis.enabled` is set to `true`, `"redis"` will be automatically added to the `extras` list of `pkgs.matrix-synapse`.
+
- A list of all extras (and the extras enabled by default) can be found at the [option's reference for `services.matrix-synapse.extras`](#opt-services.matrix-synapse.extras).
+
- In some cases (e.g. for running synapse workers) it was necessary to re-use the `PYTHONPATH` of `matrix-synapse.service`'s environment to have all plugins available. This isn't necessary anymore, instead `config.services.matrix-synapse.package` can be used as it points to the wrapper with properly configured `extras` and also all plugins defined via [`services.matrix-synapse.plugins`](#opt-services.matrix-synapse.plugins) available. This is also the reason for why the option is read-only now, it's supposed to be set by the module only.
+
- `etcd` has been updated to 3.5, you will want to read the [3.3 to 3.4](https://etcd.io/docs/v3.5/upgrades/upgrade_3_4/) and [3.4 to 3.5](https://etcd.io/docs/v3.5/upgrades/upgrade_3_5/) upgrade guides
- `consul` has been updated to `1.16.0`. See the [release note](https://github.com/hashicorp/consul/releases/tag/v1.16.0) for more details. Once a new Consul version has started and upgraded its data directory, it generally cannot be downgraded to the previous version.
+75 -12
nixos/modules/services/matrix/synapse.nix
···
# remove null values from the final configuration
finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings;
configFile = format.generate "homeserver.yaml" finalSettings;
-
logConfigFile = format.generate "log_config.yaml" cfg.logConfig;
-
-
pluginsEnv = cfg.package.python.buildEnv.override {
-
extraLibs = cfg.plugins;
-
};
usePostgresql = cfg.settings.database.name == "psycopg2";
hasLocalPostgresDB = let args = cfg.settings.database.args; in
···
"${bindAddress}"
}:${builtins.toString listener.port}/"
'';
+
+
defaultExtras = [
+
"systemd"
+
"postgres"
+
"url-preview"
+
"user-search"
+
];
+
+
wantedExtras = cfg.extras
+
++ lib.optional (cfg.settings ? oidc_providers) "oidc"
+
++ lib.optional (cfg.settings ? jwt_config) "jwt"
+
++ lib.optional (cfg.settings ? saml2_config) "saml2"
+
++ lib.optional (cfg.settings ? opentracing) "opentracing"
+
++ lib.optional (cfg.settings ? redis) "redis"
+
++ lib.optional (cfg.settings ? sentry) "sentry"
+
++ lib.optional (cfg.settings ? user_directory) "user-search"
+
++ lib.optional (cfg.settings.url_preview_enabled) "url-preview"
+
++ lib.optional (cfg.settings.database.name == "psycopg2") "postgres";
+
+
wrapped = pkgs.matrix-synapse.override {
+
extras = wantedExtras;
+
inherit (cfg) plugins;
+
};
in {
imports = [
···
package = mkOption {
type = types.package;
-
default = pkgs.matrix-synapse;
-
defaultText = literalExpression "pkgs.matrix-synapse";
+
readOnly = true;
+
description = lib.mdDoc ''
+
Reference to the `matrix-synapse` wrapper with all extras
+
(e.g. for `oidc` or `saml2`) added to the `PYTHONPATH` of all executables.
+
+
This option is useful to reference the "final" `matrix-synapse` package that's
+
actually used by `matrix-synapse.service`. For instance, when using
+
workers, it's possible to run
+
`''${config.services.matrix-synapse.package}/bin/synapse_worker` and
+
no additional PYTHONPATH needs to be specified for extras or plugins configured
+
via `services.matrix-synapse`.
+
+
However, this means that this option is supposed to be only declared
+
by the `services.matrix-synapse` module itself and is thus read-only.
+
In order to modify `matrix-synapse` itself, use an overlay to override
+
`pkgs.matrix-synapse-unwrapped`.
+
'';
+
};
+
+
extras = mkOption {
+
type = types.listOf (types.enum (lib.attrNames pkgs.matrix-synapse-unwrapped.optional-dependencies));
+
default = defaultExtras;
+
example = literalExpression ''
+
[
+
"cache-memory" # Provide statistics about caching memory consumption
+
"jwt" # JSON Web Token authentication
+
"opentracing" # End-to-end tracing support using Jaeger
+
"oidc" # OpenID Connect authentication
+
"postgres" # PostgreSQL database backend
+
"redis" # Redis support for the replication stream between worker processes
+
"saml2" # SAML2 authentication
+
"sentry" # Error tracking and performance metrics
+
"systemd" # Provide the JournalHandler used in the default log_config
+
"url-preview" # Support for oEmbed URL previews
+
"user-search" # Support internationalized domain names in user-search
+
]
+
'';
description = lib.mdDoc ''
-
Overridable attribute of the matrix synapse server package to use.
+
Explicitly install extras provided by matrix-synapse. Most
+
will require some additional configuration.
+
+
Extras will automatically be enabled, when the relevant
+
configuration sections are present.
+
+
Please note that this option is additive: i.e. when adding a new item
+
to this list, the defaults are still kept. To override the defaults as well,
+
use `lib.mkForce`.
'';
};
···
default = {};
description = mdDoc ''
The primary synapse configuration. See the
-
[sample configuration](https://github.com/matrix-org/synapse/blob/v${cfg.package.version}/docs/sample_config.yaml)
+
[sample configuration](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_config.yaml)
for possible values.
Secrets should be passed in by using the `extraConfigFiles` option.
···
];
services.matrix-synapse.configFile = configFile;
+
services.matrix-synapse.package = wrapped;
+
+
# default them, so they are additive
+
services.matrix-synapse.extras = defaultExtras;
users.users.matrix-synapse = {
group = "matrix-synapse";
···
--keys-directory ${cfg.dataDir} \
--generate-keys
'';
-
environment = {
-
PYTHONPATH = makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ];
-
} // optionalAttrs (cfg.withJemalloc) {
+
environment = optionalAttrs (cfg.withJemalloc) {
LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
};
serviceConfig = {
+14 -1
nixos/tests/matrix/synapse.nix
···
nodes = {
# Since 0.33.0, matrix-synapse doesn't allow underscores in server names
-
serverpostgres = { pkgs, nodes, ... }: let
+
serverpostgres = { pkgs, nodes, config, ... }: let
mailserverIP = nodes.mailserver.config.networking.primaryIPAddress;
in
{
···
database = {
name = "psycopg2";
args.password = "synapse";
+
};
+
redis = {
+
enabled = true;
+
host = "localhost";
+
port = config.services.redis.servers.matrix-synapse.port;
};
tls_certificate_path = "${cert}";
tls_private_key_path = "${key}";
···
LC_COLLATE = "C"
LC_CTYPE = "C";
'';
+
};
+
+
services.redis.servers.matrix-synapse = {
+
enable = true;
+
port = 6380;
};
networking.extraHosts = ''
···
serverpostgres.wait_for_unit("matrix-synapse.service")
serverpostgres.wait_until_succeeds(
"curl --fail -L --cacert ${ca_pem} https://localhost:8448/"
+
)
+
serverpostgres.wait_until_succeeds(
+
"journalctl -u matrix-synapse.service | grep -q 'Connected to redis'"
)
serverpostgres.require_unit_state("postgresql.service")
serverpostgres.succeed("register_new_matrix_user -u ${testUser} -p ${testPassword} -a -k ${registrationSharedSecret} https://localhost:8448/")
+69 -21
pkgs/servers/matrix-synapse/default.nix
···
-
{ lib, stdenv, fetchFromGitHub, python3, openssl, cargo, rustPlatform, rustc
-
, enableSystemd ? lib.meta.availableOn stdenv.hostPlatform python3.pkgs.systemd
+
{ lib
+
, stdenv
+
, fetchFromGitHub
+
, python3
+
, openssl
+
, cargo
+
, rustPlatform
+
, rustc
, nixosTests
-
, enableRedis ? true
, callPackage
}:
···
plugins = python3.pkgs.callPackage ./plugins { };
tools = callPackage ./tools { };
in
-
with python3.pkgs;
-
buildPythonApplication rec {
+
python3.pkgs.buildPythonApplication rec {
pname = "matrix-synapse";
version = "1.89.0";
format = "pyproject";
···
sed -i '/^setuptools_rust =/d' pyproject.toml
'';
-
nativeBuildInputs = [
+
nativeBuildInputs = with python3.pkgs; [
poetry-core
rustPlatform.cargoSetupHook
setuptools-rust
···
rustc
];
-
buildInputs = [ openssl ];
+
buildInputs = [
+
openssl
+
];
-
propagatedBuildInputs = [
-
authlib
+
propagatedBuildInputs = with python3.pkgs; [
+
attrs
bcrypt
bleach
canonicaljson
-
daemonize
+
cryptography
ijson
immutabledict
jinja2
jsonschema
-
lxml
matrix-common
msgpack
netaddr
+
packaging
phonenumbers
pillow
prometheus-client
-
psutil
-
psycopg2
pyasn1
+
pyasn1-modules
pydantic
-
pyicu
pymacaroons
-
pynacl
pyopenssl
-
pysaml2
pyyaml
-
requests
-
setuptools
+
service-identity
signedjson
sortedcontainers
treq
twisted
typing-extensions
unpaddedbase64
-
] ++ lib.optional enableSystemd systemd
-
++ lib.optionals enableRedis [ hiredis txredisapi ];
+
]
+
++ twisted.optional-dependencies.tls;
-
nativeCheckInputs = [ mock parameterized openssl ];
+
passthru.optional-dependencies = with python3.pkgs; {
+
postgres = if isPyPy then [
+
psycopg2cffi
+
] else [
+
psycopg2
+
];
+
saml2 = [
+
pysaml2
+
];
+
oidc = [
+
authlib
+
];
+
systemd = [
+
systemd
+
];
+
url-preview = [
+
lxml
+
];
+
sentry = [
+
sentry-sdk
+
];
+
opentracing = [
+
jaeger-client
+
opentracing
+
];
+
jwt = [
+
authlib
+
];
+
redis = [
+
hiredis
+
txredisapi
+
];
+
cache-memory = [
+
pympler
+
];
+
user-search = [
+
pyicu
+
];
+
};
+
+
nativeCheckInputs = [
+
openssl
+
] ++ (with python3.pkgs; [
+
mock
+
parameterized
+
])
+
++ lib.flatten (lib.attrValues passthru.optional-dependencies);
doCheck = !stdenv.isDarwin;
···
meta = with lib; {
homepage = "https://matrix.org";
+
changelog = "https://github.com/matrix-org/synapse/releases/tag/v${version}";
description = "Matrix reference homeserver";
license = licenses.asl20;
maintainers = teams.matrix.members;
+2 -2
pkgs/servers/matrix-synapse/plugins/ldap3.nix
···
, fetchPypi
, ldap3
, ldaptor
-
, matrix-synapse
+
, matrix-synapse-unwrapped
, pytestCheckHook
, service-identity
, setuptools
···
propagatedBuildInputs = [ service-identity ldap3 twisted ];
-
nativeCheckInputs = [ ldaptor matrix-synapse pytestCheckHook ];
+
nativeCheckInputs = [ ldaptor matrix-synapse-unwrapped pytestCheckHook ];
pythonImportsCheck = [ "ldap_auth_provider" ];
+2 -2
pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix
···
-
{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse }:
+
{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped }:
buildPythonPackage rec {
pname = "matrix-synapse-mjolnir-antispam";
···
sourceRoot = "./${src.name}/synapse_antispam";
-
buildInputs = [ matrix-synapse ];
+
buildInputs = [ matrix-synapse-unwrapped ];
doCheck = false; # no tests
pythonImportsCheck = [ "mjolnir" ];
+2 -2
pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix
···
-
{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse, twisted }:
+
{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped, twisted }:
buildPythonPackage rec {
pname = "matrix-synapse-shared-secret-auth";
···
doCheck = false;
pythonImportsCheck = [ "shared_secret_authenticator" ];
-
buildInputs = [ matrix-synapse ];
+
buildInputs = [ matrix-synapse-unwrapped ];
propagatedBuildInputs = [ twisted ];
meta = with lib; {
+44
pkgs/servers/matrix-synapse/wrapper.nix
···
+
{ lib
+
, stdenv
+
, makeWrapper
+
, matrix-synapse-unwrapped
+
, extras ? [
+
"postgres"
+
"url-preview"
+
"user-search"
+
] ++ lib.optional (lib.meta.availableOn stdenv.hostPlatform matrix-synapse-unwrapped.python.pkgs.systemd) "systemd"
+
, plugins ? [ ]
+
, ...
+
}:
+
+
let
+
extraPackages = lib.concatMap (extra: matrix-synapse-unwrapped.optional-dependencies.${extra}) (lib.unique extras);
+
+
pluginsEnv = matrix-synapse-unwrapped.python.buildEnv.override {
+
extraLibs = plugins;
+
};
+
+
searchPath = lib.makeSearchPathOutput "lib" matrix-synapse-unwrapped.python.sitePackages (extraPackages ++ [ pluginsEnv ]);
+
in
+
stdenv.mkDerivation {
+
name = (lib.appendToName "wrapped" matrix-synapse-unwrapped).name;
+
+
nativeBuildInputs = [
+
makeWrapper
+
];
+
+
buildCommand = ''
+
for bin in ${matrix-synapse-unwrapped}/bin/*; do
+
echo $bin
+
makeWrapper "$bin" "$out/bin/$(basename $bin)" \
+
--set PYTHONPATH ${searchPath}
+
done;
+
'';
+
+
passthru = {
+
unwrapped = matrix-synapse-unwrapped;
+
+
# for backward compatibility
+
inherit (matrix-synapse-unwrapped) plugins tools;
+
};
+
}
+4 -5
pkgs/top-level/all-packages.nix
···
matrix-sliding-sync = callPackage ../servers/matrix-synapse/sliding-sync { };
-
matrix-synapse = callPackage ../servers/matrix-synapse { };
-
-
matrix-synapse-plugins = recurseIntoAttrs matrix-synapse.plugins;
-
-
matrix-synapse-tools = recurseIntoAttrs matrix-synapse.tools;
+
matrix-synapse = callPackage ../servers/matrix-synapse/wrapper.nix { };
+
matrix-synapse-unwrapped = callPackage ../servers/matrix-synapse/default.nix { };
+
matrix-synapse-plugins = recurseIntoAttrs matrix-synapse-unwrapped.plugins;
+
matrix-synapse-tools = recurseIntoAttrs matrix-synapse-unwrapped.tools;
matrix-appservice-irc = callPackage ../servers/matrix-synapse/matrix-appservice-irc { };