Merge pull request #117126 from Izorkin/update-mastodon-init-db

nixos/mastodon: fix init db on remote postgresql

Kerstin c28d4da8 757b8221

Changed files
+383 -143
nixos
doc
manual
from_md
release-notes
release-notes
modules
services
web-apps
tests
+17
nixos/doc/manual/from_md/release-notes/rl-2305.section.xml
···
</listitem>
<listitem>
<para>
+
In <literal>mastodon</literal> it is now necessary to specify
+
location of file with <literal>PostgreSQL</literal> database
+
password. In
+
<literal>services.mastodon.database.passwordFile</literal>
+
parameter default value
+
<literal>/var/lib/mastodon/secrets/db-password</literal> has
+
been changed to <literal>null</literal>.
+
</para>
+
</listitem>
+
<listitem>
+
<para>
The <literal>nix.readOnlyStore</literal> option has been
renamed to <literal>boot.readOnlyNixStore</literal> to clarify
that it configures the NixOS boot process, not the Nix daemon.
···
<para>
The minimal ISO image now uses the
<literal>nixos/modules/profiles/minimal.nix</literal> profile.
+
</para>
+
</listitem>
+
<listitem>
+
<para>
+
<literal>mastodon</literal> now supports connection to a
+
remote <literal>PostgreSQL</literal> database.
</para>
</listitem>
<listitem>
+4
nixos/doc/manual/release-notes/rl-2305.section.md
···
- Qt 5.12 and 5.14 have been removed, as the corresponding branches have been EOL upstream for a long time. This affected under 10 packages in nixpkgs, largely unmaintained upstream as well, however, out-of-tree package expressions may need to be updated manually.
+
- In `mastodon` it is now necessary to specify location of file with `PostgreSQL` database password. In `services.mastodon.database.passwordFile` parameter default value `/var/lib/mastodon/secrets/db-password` has been changed to `null`.
+
- The `nix.readOnlyStore` option has been renamed to `boot.readOnlyNixStore` to clarify that it configures the NixOS boot process, not the Nix daemon.
## Other Notable Changes {#sec-release-23.05-notable-changes}
···
- To reduce closure size in `nixos/modules/profiles/minimal.nix` profile disabled installation documentations and manuals. Also disabled `logrotate` and `udisks2` services.
- The minimal ISO image now uses the `nixos/modules/profiles/minimal.nix` profile.
+
+
- `mastodon` now supports connection to a remote `PostgreSQL` database.
- A new `virtualisation.rosetta` module was added to allow running `x86_64` binaries through [Rosetta](https://developer.apple.com/documentation/apple-silicon/about-the-rosetta-translation-environment) inside virtualised NixOS guests on Apple silicon. This feature works by default with the [UTM](https://docs.getutm.app/) virtualisation [package](https://search.nixos.org/packages?channel=unstable&show=utm&from=0&size=1&sort=relevance&type=packages&query=utm).
+46 -12
nixos/modules/services/web-apps/mastodon.nix
···
-
{ config, lib, pkgs, ... }:
+
{ lib, pkgs, config, options, ... }:
let
cfg = config.services.mastodon;
+
opt = options.services.mastodon;
+
# We only want to create a database if we're actually going to connect to it.
databaseActuallyCreateLocally = cfg.database.createLocally && cfg.database.host == "/run/postgresql";
···
REDIS_HOST = cfg.redis.host;
REDIS_PORT = toString(cfg.redis.port);
DB_HOST = cfg.database.host;
-
DB_PORT = toString(cfg.database.port);
DB_NAME = cfg.database.name;
LOCAL_DOMAIN = cfg.localDomain;
SMTP_SERVER = cfg.smtp.host;
···
TRUSTED_PROXY_IP = cfg.trustedProxy;
}
-
// (if cfg.smtp.authenticate then { SMTP_LOGIN = cfg.smtp.user; } else {})
+
// lib.optionalAttrs (cfg.database.host != "/run/postgresql" && cfg.database.port != null) { DB_PORT = toString cfg.database.port; }
+
// lib.optionalAttrs cfg.smtp.authenticate { SMTP_LOGIN = cfg.smtp.user; }
// cfg.extraConfig;
systemCallsList = [ "@cpu-emulation" "@debug" "@keyring" "@ipc" "@mount" "@obsolete" "@privileged" "@setuid" ];
···
};
port = lib.mkOption {
-
type = lib.types.port;
-
default = 5432;
+
type = lib.types.nullOr lib.types.port;
+
default = if cfg.database.createLocally then null else 5432;
+
defaultText = lib.literalExpression ''
+
if config.${opt.database.createLocally}
+
then null
+
else 5432
+
'';
description = lib.mdDoc "Database host port.";
};
···
passwordFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
-
default = "/var/lib/mastodon/secrets/db-password";
-
example = "/run/keys/mastodon-db-password";
+
default = null;
+
example = "/var/lib/mastodon/secrets/db-password";
description = lib.mdDoc ''
A file containing the password corresponding to
{option}`database.user`.
···
assertions = [
{
assertion = databaseActuallyCreateLocally -> (cfg.user == cfg.database.user);
-
message = ''For local automatic database provisioning (services.mastodon.database.createLocally == true) with peer authentication (services.mastodon.database.host == "/run/postgresql") to work services.mastodon.user and services.mastodon.database.user must be identical.'';
+
message = ''
+
For local automatic database provisioning (services.mastodon.database.createLocally == true) with peer
+
authentication (services.mastodon.database.host == "/run/postgresql") to work services.mastodon.user
+
and services.mastodon.database.user must be identical.
+
'';
+
}
+
{
+
assertion = !databaseActuallyCreateLocally -> (cfg.database.host != "/run/postgresql");
+
message = ''
+
<option>services.mastodon.database.host</option> needs to be set if
+
<option>services.mastodon.database.createLocally</option> is not enabled.
+
'';
}
{
assertion = cfg.smtp.authenticate -> (cfg.smtp.user != null);
···
OTP_SECRET="$(cat ${cfg.otpSecretFile})"
VAPID_PRIVATE_KEY="$(cat ${cfg.vapidPrivateKeyFile})"
VAPID_PUBLIC_KEY="$(cat ${cfg.vapidPublicKeyFile})"
+
'' + lib.optionalString (cfg.database.passwordFile != null) ''
DB_PASS="$(cat ${cfg.database.passwordFile})"
-
'' + (if cfg.smtp.authenticate then ''
+
'' + lib.optionalString cfg.smtp.authenticate ''
SMTP_PASSWORD="$(cat ${cfg.smtp.passwordFile})"
-
'' else "") + ''
+
'' + ''
EOF
'';
environment = env;
···
};
systemd.services.mastodon-init-db = lib.mkIf cfg.automaticMigrations {
-
script = ''
+
script = lib.optionalString (!databaseActuallyCreateLocally) ''
+
umask 077
+
+
export PGPASSFILE
+
PGPASSFILE=$(mktemp)
+
cat > $PGPASSFILE <<EOF
+
${cfg.database.host}:${toString cfg.database.port}:${cfg.database.name}:${cfg.database.user}:$(cat ${cfg.database.passwordFile})
+
EOF
+
+
'' + ''
if [ `psql ${cfg.database.name} -c \
"select count(*) from pg_class c \
join pg_namespace s on s.oid = c.relnamespace \
···
else
rails db:migrate
fi
+
'' + lib.optionalString (!databaseActuallyCreateLocally) ''
+
rm $PGPASSFILE
+
unset PGPASSFILE
'';
path = [ cfg.package pkgs.postgresql ];
-
environment = env;
+
environment = env // lib.optionalAttrs (!databaseActuallyCreateLocally) {
+
PGHOST = cfg.database.host;
+
PGUSER = cfg.database.user;
+
};
serviceConfig = {
Type = "oneshot";
EnvironmentFile = [ "/var/lib/mastodon/.secrets_env" ];
+1 -1
nixos/tests/all-tests.nix
···
mailhog = handleTest ./mailhog.nix {};
man = handleTest ./man.nix {};
mariadb-galera = handleTest ./mysql/mariadb-galera.nix {};
-
mastodon = handleTestOn ["x86_64-linux" "i686-linux" "aarch64-linux"] ./web-apps/mastodon.nix {};
+
mastodon = discoverTests (import ./web-apps/mastodon { inherit handleTestOn; });
matomo = handleTest ./matomo.nix {};
matrix-appservice-irc = handleTest ./matrix/appservice-irc.nix {};
matrix-conduit = handleTest ./matrix/conduit.nix {};
-130
nixos/tests/web-apps/mastodon.nix
···
-
import ../make-test-python.nix ({pkgs, ...}:
-
let
-
cert = pkgs: pkgs.runCommand "selfSignedCerts" { buildInputs = [ pkgs.openssl ]; } ''
-
openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -nodes -subj '/CN=mastodon.local' -days 36500
-
mkdir -p $out
-
cp key.pem cert.pem $out
-
'';
-
-
hosts = ''
-
192.168.2.101 mastodon.local
-
'';
-
-
in
-
{
-
name = "mastodon";
-
meta.maintainers = with pkgs.lib.maintainers; [ erictapen izorkin turion ];
-
-
nodes = {
-
server = { pkgs, ... }: {
-
-
virtualisation.memorySize = 2048;
-
-
networking = {
-
interfaces.eth1 = {
-
ipv4.addresses = [
-
{ address = "192.168.2.101"; prefixLength = 24; }
-
];
-
};
-
extraHosts = hosts;
-
firewall.allowedTCPPorts = [ 80 443 ];
-
};
-
-
security = {
-
pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
-
};
-
-
services.redis.servers.mastodon = {
-
enable = true;
-
bind = "127.0.0.1";
-
port = 31637;
-
};
-
-
services.mastodon = {
-
enable = true;
-
configureNginx = true;
-
localDomain = "mastodon.local";
-
enableUnixSocket = false;
-
smtp = {
-
createLocally = false;
-
fromAddress = "mastodon@mastodon.local";
-
};
-
extraConfig = {
-
EMAIL_DOMAIN_ALLOWLIST = "example.com";
-
};
-
};
-
-
services.nginx = {
-
virtualHosts."mastodon.local" = {
-
enableACME = pkgs.lib.mkForce false;
-
sslCertificate = "${cert pkgs}/cert.pem";
-
sslCertificateKey = "${cert pkgs}/key.pem";
-
};
-
};
-
};
-
-
client = { pkgs, ... }: {
-
environment.systemPackages = [ pkgs.jq ];
-
networking = {
-
interfaces.eth1 = {
-
ipv4.addresses = [
-
{ address = "192.168.2.102"; prefixLength = 24; }
-
];
-
};
-
extraHosts = hosts;
-
};
-
-
security = {
-
pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
-
};
-
};
-
};
-
-
testScript = ''
-
start_all()
-
-
server.wait_for_unit("nginx.service")
-
server.wait_for_unit("redis-mastodon.service")
-
server.wait_for_unit("postgresql.service")
-
server.wait_for_unit("mastodon-sidekiq.service")
-
server.wait_for_unit("mastodon-streaming.service")
-
server.wait_for_unit("mastodon-web.service")
-
server.wait_for_open_port(55000)
-
server.wait_for_open_port(55001)
-
-
# Check that mastodon-media-auto-remove is scheduled
-
server.succeed("systemctl status mastodon-media-auto-remove.timer")
-
-
# Check Mastodon version from remote client
-
client.succeed("curl --fail https://mastodon.local/api/v1/instance | jq -r '.version' | grep '${pkgs.mastodon.version}'")
-
-
# Check access from remote client
-
client.succeed("curl --fail https://mastodon.local/about | grep 'Mastodon hosted on mastodon.local'")
-
client.succeed("curl --fail $(curl https://mastodon.local/api/v1/instance 2> /dev/null | jq -r .thumbnail) --output /dev/null")
-
-
# Simple check tootctl commands
-
# Check Mastodon version
-
server.succeed("mastodon-tootctl version | grep '${pkgs.mastodon.version}'")
-
-
# Manage accounts
-
server.succeed("mastodon-tootctl email_domain_blocks add example.com")
-
server.succeed("mastodon-tootctl email_domain_blocks list | grep example.com")
-
server.fail("mastodon-tootctl email_domain_blocks list | grep mastodon.local")
-
server.fail("mastodon-tootctl accounts create alice --email=alice@example.com")
-
server.succeed("mastodon-tootctl email_domain_blocks remove example.com")
-
server.succeed("mastodon-tootctl accounts create bob --email=bob@example.com")
-
server.succeed("mastodon-tootctl accounts approve bob")
-
server.succeed("mastodon-tootctl accounts delete bob")
-
-
# Manage IP access
-
server.succeed("mastodon-tootctl ip_blocks add 192.168.0.0/16 --severity=no_access")
-
server.succeed("mastodon-tootctl ip_blocks export | grep 192.168.0.0/16")
-
server.fail("mastodon-tootctl ip_blocks export | grep 172.16.0.0/16")
-
client.fail("curl --fail https://mastodon.local/about")
-
server.succeed("mastodon-tootctl ip_blocks remove 192.168.0.0/16")
-
client.succeed("curl --fail https://mastodon.local/about")
-
-
server.shutdown()
-
client.shutdown()
-
'';
-
})
+9
nixos/tests/web-apps/mastodon/default.nix
···
+
{ system ? builtins.currentSystem, handleTestOn }:
+
let
+
supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ];
+
+
in
+
{
+
standard = handleTestOn supportedSystems ./standard.nix { inherit system; };
+
remote-postgresql = handleTestOn supportedSystems ./remote-postgresql.nix { inherit system; };
+
}
+160
nixos/tests/web-apps/mastodon/remote-postgresql.nix
···
+
import ../../make-test-python.nix ({pkgs, ...}:
+
let
+
cert = pkgs: pkgs.runCommand "selfSignedCerts" { buildInputs = [ pkgs.openssl ]; } ''
+
openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -nodes -subj '/CN=mastodon.local' -days 36500
+
mkdir -p $out
+
cp key.pem cert.pem $out
+
'';
+
+
hosts = ''
+
192.168.2.103 mastodon.local
+
'';
+
+
in
+
{
+
name = "mastodon-remote-postgresql";
+
meta.maintainers = with pkgs.lib.maintainers; [ erictapen izorkin turion ];
+
+
nodes = {
+
database = {
+
networking = {
+
interfaces.eth1 = {
+
ipv4.addresses = [
+
{ address = "192.168.2.102"; prefixLength = 24; }
+
];
+
};
+
extraHosts = hosts;
+
firewall.allowedTCPPorts = [ 5432 ];
+
};
+
+
services.postgresql = {
+
enable = true;
+
enableTCPIP = true;
+
authentication = ''
+
hostnossl mastodon_local mastodon_test 192.168.2.201/32 md5
+
'';
+
initialScript = pkgs.writeText "postgresql_init.sql" ''
+
CREATE ROLE mastodon_test LOGIN PASSWORD 'SoDTZcISc3f1M1LJsRLT';
+
CREATE DATABASE mastodon_local TEMPLATE template0 ENCODING UTF8;
+
GRANT ALL PRIVILEGES ON DATABASE mastodon_local TO mastodon_test;
+
'';
+
};
+
};
+
+
nginx = {
+
networking = {
+
interfaces.eth1 = {
+
ipv4.addresses = [
+
{ address = "192.168.2.103"; prefixLength = 24; }
+
];
+
};
+
extraHosts = hosts;
+
firewall.allowedTCPPorts = [ 80 443 ];
+
};
+
+
security = {
+
pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
+
};
+
+
services.nginx = {
+
enable = true;
+
recommendedProxySettings = true;
+
virtualHosts."mastodon.local" = {
+
root = "/var/empty";
+
forceSSL = true;
+
enableACME = pkgs.lib.mkForce false;
+
sslCertificate = "${cert pkgs}/cert.pem";
+
sslCertificateKey = "${cert pkgs}/key.pem";
+
locations."/" = {
+
tryFiles = "$uri @proxy";
+
};
+
locations."@proxy" = {
+
proxyPass = "http://192.168.2.201:55001";
+
proxyWebsockets = true;
+
};
+
locations."/api/v1/streaming/" = {
+
proxyPass = "http://192.168.2.201:55002";
+
proxyWebsockets = true;
+
};
+
};
+
};
+
};
+
+
server = { pkgs, ... }: {
+
virtualisation.memorySize = 2048;
+
+
environment = {
+
etc = {
+
"mastodon/password-posgressql-db".text = ''
+
SoDTZcISc3f1M1LJsRLT
+
'';
+
};
+
};
+
+
networking = {
+
interfaces.eth1 = {
+
ipv4.addresses = [
+
{ address = "192.168.2.201"; prefixLength = 24; }
+
];
+
};
+
extraHosts = hosts;
+
firewall.allowedTCPPorts = [ 55001 55002 ];
+
};
+
+
services.mastodon = {
+
enable = true;
+
configureNginx = false;
+
localDomain = "mastodon.local";
+
enableUnixSocket = false;
+
database = {
+
createLocally = false;
+
host = "192.168.2.102";
+
port = 5432;
+
name = "mastodon_local";
+
user = "mastodon_test";
+
passwordFile = "/etc/mastodon/password-posgressql-db";
+
};
+
smtp = {
+
createLocally = false;
+
fromAddress = "mastodon@mastodon.local";
+
};
+
extraConfig = {
+
BIND = "0.0.0.0";
+
EMAIL_DOMAIN_ALLOWLIST = "example.com";
+
RAILS_SERVE_STATIC_FILES = "true";
+
TRUSTED_PROXY_IP = "192.168.2.103";
+
};
+
};
+
};
+
+
client = { pkgs, ... }: {
+
environment.systemPackages = [ pkgs.jq ];
+
networking = {
+
interfaces.eth1 = {
+
ipv4.addresses = [
+
{ address = "192.168.2.202"; prefixLength = 24; }
+
];
+
};
+
extraHosts = hosts;
+
};
+
+
security = {
+
pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
+
};
+
};
+
};
+
+
testScript = import ./script.nix {
+
inherit pkgs;
+
extraInit = ''
+
nginx.wait_for_unit("nginx.service")
+
nginx.wait_for_open_port(443)
+
database.wait_for_unit("postgresql.service")
+
database.wait_for_open_port(5432)
+
'';
+
extraShutdown = ''
+
nginx.shutdown()
+
database.shutdown()
+
'';
+
};
+
})
+54
nixos/tests/web-apps/mastodon/script.nix
···
+
{ pkgs
+
, extraInit ? ""
+
, extraShutdown ? ""
+
}:
+
+
''
+
start_all()
+
+
${extraInit}
+
+
server.wait_for_unit("redis-mastodon.service")
+
server.wait_for_unit("mastodon-sidekiq.service")
+
server.wait_for_unit("mastodon-streaming.service")
+
server.wait_for_unit("mastodon-web.service")
+
server.wait_for_open_port(55000)
+
server.wait_for_open_port(55001)
+
+
# Check that mastodon-media-auto-remove is scheduled
+
server.succeed("systemctl status mastodon-media-auto-remove.timer")
+
+
# Check Mastodon version from remote client
+
client.succeed("curl --fail https://mastodon.local/api/v1/instance | jq -r '.version' | grep '${pkgs.mastodon.version}'")
+
+
# Check access from remote client
+
client.succeed("curl --fail https://mastodon.local/about | grep 'Mastodon hosted on mastodon.local'")
+
client.succeed("curl --fail $(curl https://mastodon.local/api/v1/instance 2> /dev/null | jq -r .thumbnail) --output /dev/null")
+
+
# Simple check tootctl commands
+
# Check Mastodon version
+
server.succeed("mastodon-tootctl version | grep '${pkgs.mastodon.version}'")
+
+
# Manage accounts
+
server.succeed("mastodon-tootctl email_domain_blocks add example.com")
+
server.succeed("mastodon-tootctl email_domain_blocks list | grep example.com")
+
server.fail("mastodon-tootctl email_domain_blocks list | grep mastodon.local")
+
server.fail("mastodon-tootctl accounts create alice --email=alice@example.com")
+
server.succeed("mastodon-tootctl email_domain_blocks remove example.com")
+
server.succeed("mastodon-tootctl accounts create bob --email=bob@example.com")
+
server.succeed("mastodon-tootctl accounts approve bob")
+
server.succeed("mastodon-tootctl accounts delete bob")
+
+
# Manage IP access
+
server.succeed("mastodon-tootctl ip_blocks add 192.168.0.0/16 --severity=no_access")
+
server.succeed("mastodon-tootctl ip_blocks export | grep 192.168.0.0/16")
+
server.fail("mastodon-tootctl ip_blocks export | grep 172.16.0.0/16")
+
client.fail("curl --fail https://mastodon.local/about")
+
server.succeed("mastodon-tootctl ip_blocks remove 192.168.0.0/16")
+
client.succeed("curl --fail https://mastodon.local/about")
+
+
server.shutdown()
+
client.shutdown()
+
+
${extraShutdown}
+
''
+92
nixos/tests/web-apps/mastodon/standard.nix
···
+
import ../../make-test-python.nix ({pkgs, ...}:
+
let
+
cert = pkgs: pkgs.runCommand "selfSignedCerts" { buildInputs = [ pkgs.openssl ]; } ''
+
openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -nodes -subj '/CN=mastodon.local' -days 36500
+
mkdir -p $out
+
cp key.pem cert.pem $out
+
'';
+
+
hosts = ''
+
192.168.2.101 mastodon.local
+
'';
+
+
in
+
{
+
name = "mastodon-standard";
+
meta.maintainers = with pkgs.lib.maintainers; [ erictapen izorkin turion ];
+
+
nodes = {
+
server = { pkgs, ... }: {
+
+
virtualisation.memorySize = 2048;
+
+
networking = {
+
interfaces.eth1 = {
+
ipv4.addresses = [
+
{ address = "192.168.2.101"; prefixLength = 24; }
+
];
+
};
+
extraHosts = hosts;
+
firewall.allowedTCPPorts = [ 80 443 ];
+
};
+
+
security = {
+
pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
+
};
+
+
services.redis.servers.mastodon = {
+
enable = true;
+
bind = "127.0.0.1";
+
port = 31637;
+
};
+
+
services.mastodon = {
+
enable = true;
+
configureNginx = true;
+
localDomain = "mastodon.local";
+
enableUnixSocket = false;
+
smtp = {
+
createLocally = false;
+
fromAddress = "mastodon@mastodon.local";
+
};
+
extraConfig = {
+
EMAIL_DOMAIN_ALLOWLIST = "example.com";
+
};
+
};
+
+
services.nginx = {
+
virtualHosts."mastodon.local" = {
+
enableACME = pkgs.lib.mkForce false;
+
sslCertificate = "${cert pkgs}/cert.pem";
+
sslCertificateKey = "${cert pkgs}/key.pem";
+
};
+
};
+
};
+
+
client = { pkgs, ... }: {
+
environment.systemPackages = [ pkgs.jq ];
+
networking = {
+
interfaces.eth1 = {
+
ipv4.addresses = [
+
{ address = "192.168.2.102"; prefixLength = 24; }
+
];
+
};
+
extraHosts = hosts;
+
};
+
+
security = {
+
pki.certificateFiles = [ "${cert pkgs}/cert.pem" ];
+
};
+
};
+
};
+
+
testScript = import ./script.nix {
+
inherit pkgs;
+
extraInit = ''
+
server.wait_for_unit("nginx.service")
+
server.wait_for_open_port(443)
+
server.wait_for_unit("postgresql.service")
+
server.wait_for_open_port(5432)
+
'';
+
};
+
})