Merge pull request #144174 from mkg20001/lxdimage

Changed files
+456
nixos
pkgs
development
python-modules
confight
inotify
tools
virtualization
lxd-image-server
top-level
+1
nixos/modules/module-list.nix
···
./services/networking/libreswan.nix
./services/networking/lldpd.nix
./services/networking/logmein-hamachi.nix
+
./services/networking/lxd-image-server.nix
./services/networking/mailpile.nix
./services/networking/magic-wormhole-mailbox-server.nix
./services/networking/matterbridge.nix
+138
nixos/modules/services/networking/lxd-image-server.nix
···
+
{ config, pkgs, lib, ... }:
+
+
with lib;
+
+
let
+
cfg = config.services.lxd-image-server;
+
format = pkgs.formats.toml {};
+
+
location = "/var/www/simplestreams";
+
in
+
{
+
options = {
+
services.lxd-image-server = {
+
enable = mkEnableOption "lxd-image-server";
+
+
group = mkOption {
+
type = types.str;
+
description = "Group assigned to the user and the webroot directory.";
+
default = "nginx";
+
example = "www-data";
+
};
+
+
settings = mkOption {
+
type = format.type;
+
description = ''
+
Configuration for lxd-image-server.
+
+
Example see <link xlink:href="https://github.com/Avature/lxd-image-server/blob/master/config.toml"/>.
+
'';
+
default = {};
+
};
+
+
nginx = {
+
enable = mkEnableOption "nginx";
+
domain = mkOption {
+
type = types.str;
+
description = "Domain to use for nginx virtual host.";
+
example = "images.example.org";
+
};
+
};
+
};
+
};
+
+
config = mkMerge [
+
(mkIf (cfg.enable) {
+
users.users.lxd-image-server = {
+
isSystemUser = true;
+
group = cfg.group;
+
};
+
users.groups.${cfg.group} = {};
+
+
environment.etc."lxd-image-server/config.toml".source = format.generate "config.toml" cfg.settings;
+
+
services.logrotate.paths.lxd-image-server = {
+
path = "/var/log/lxd-image-server/lxd-image-server.log";
+
frequency = "daily";
+
keep = 21;
+
user = "lxd-image-server";
+
group = cfg.group;
+
extraConfig = ''
+
missingok
+
compress
+
delaycompress
+
copytruncate
+
notifempty
+
'';
+
};
+
+
systemd.tmpfiles.rules = [
+
"d /var/www/simplestreams 0755 lxd-image-server ${cfg.group}"
+
];
+
+
systemd.services.lxd-image-server = {
+
wantedBy = [ "multi-user.target" ];
+
after = [ "network.target" ];
+
+
description = "LXD Image Server";
+
+
script = ''
+
${pkgs.lxd-image-server}/bin/lxd-image-server init
+
${pkgs.lxd-image-server}/bin/lxd-image-server watch
+
'';
+
+
serviceConfig = {
+
User = "lxd-image-server";
+
Group = cfg.group;
+
DynamicUser = true;
+
LogsDirectory = "lxd-image-server";
+
RuntimeDirectory = "lxd-image-server";
+
ExecReload = "${pkgs.lxd-image-server}/bin/lxd-image-server reload";
+
ReadWritePaths = [ location ];
+
};
+
};
+
})
+
# this is seperate so it can be enabled on mirrored hosts
+
(mkIf (cfg.nginx.enable) {
+
# https://github.com/Avature/lxd-image-server/blob/master/resources/nginx/includes/lxd-image-server.pkg.conf
+
services.nginx.virtualHosts = {
+
"${cfg.nginx.domain}" = {
+
forceSSL = true;
+
enableACME = mkDefault true;
+
+
root = location;
+
+
locations = {
+
"/streams/v1/" = {
+
index = "index.json";
+
};
+
+
# Serve json files with content type header application/json
+
"~ \.json$" = {
+
extraConfig = ''
+
add_header Content-Type application/json;
+
'';
+
};
+
+
"~ \.tar.xz$" = {
+
extraConfig = ''
+
add_header Content-Type application/octet-stream;
+
'';
+
};
+
+
"~ \.tar.gz$" = {
+
extraConfig = ''
+
add_header Content-Type application/octet-stream;
+
'';
+
};
+
+
# Deny access to document root and the images folder
+
"~ ^/(images/)?$" = {
+
return = "403";
+
};
+
};
+
};
+
};
+
})
+
];
+
}
+1
nixos/tests/all-tests.nix
···
lxd = handleTest ./lxd.nix {};
lxd-image = handleTest ./lxd-image.nix {};
lxd-nftables = handleTest ./lxd-nftables.nix {};
+
lxd-image-server = handleTest ./lxd-image-server.nix {};
#logstash = handleTest ./logstash.nix {};
lorri = handleTest ./lorri/default.nix {};
magic-wormhole-mailbox-server = handleTest ./magic-wormhole-mailbox-server.nix {};
+127
nixos/tests/lxd-image-server.nix
···
+
import ./make-test-python.nix ({ pkgs, ...} :
+
+
let
+
# Since we don't have access to the internet during the tests, we have to
+
# pre-fetch lxd containers beforehand.
+
#
+
# I've chosen to import Alpine Linux, because its image is turbo-tiny and,
+
# generally, sufficient for our tests.
+
alpine-meta = pkgs.fetchurl {
+
url = "https://tarballs.nixos.org/alpine/3.12/lxd.tar.xz";
+
hash = "sha256-1tcKaO9lOkvqfmG/7FMbfAEToAuFy2YMewS8ysBKuLA=";
+
};
+
+
alpine-rootfs = pkgs.fetchurl {
+
url = "https://tarballs.nixos.org/alpine/3.12/rootfs.tar.xz";
+
hash = "sha256-Tba9sSoaiMtQLY45u7p5DMqXTSDgs/763L/SQp0bkCA=";
+
};
+
+
lxd-config = pkgs.writeText "config.yaml" ''
+
storage_pools:
+
- name: default
+
driver: dir
+
config:
+
source: /var/lxd-pool
+
+
networks:
+
- name: lxdbr0
+
type: bridge
+
config:
+
ipv4.address: auto
+
ipv6.address: none
+
+
profiles:
+
- name: default
+
devices:
+
eth0:
+
name: eth0
+
network: lxdbr0
+
type: nic
+
root:
+
path: /
+
pool: default
+
type: disk
+
'';
+
+
+
in {
+
name = "lxd-image-server";
+
+
meta = with pkgs.lib.maintainers; {
+
maintainers = [ mkg20001 ];
+
};
+
+
machine = { lib, ... }: {
+
virtualisation = {
+
cores = 2;
+
+
memorySize = 2048;
+
diskSize = 4096;
+
+
lxc.lxcfs.enable = true;
+
lxd.enable = true;
+
};
+
+
security.pki.certificates = [
+
(builtins.readFile ./common/acme/server/ca.cert.pem)
+
];
+
+
services.nginx = {
+
enable = true;
+
};
+
+
services.lxd-image-server = {
+
enable = true;
+
nginx = {
+
enable = true;
+
domain = "acme.test";
+
};
+
};
+
+
services.nginx.virtualHosts."acme.test" = {
+
enableACME = false;
+
sslCertificate = ./common/acme/server/acme.test.cert.pem;
+
sslCertificateKey = ./common/acme/server/acme.test.key.pem;
+
};
+
+
networking.hosts = {
+
"::1" = [ "acme.test" ];
+
};
+
};
+
+
testScript = ''
+
machine.wait_for_unit("sockets.target")
+
machine.wait_for_unit("lxd.service")
+
machine.wait_for_file("/var/lib/lxd/unix.socket")
+
+
# It takes additional second for lxd to settle
+
machine.sleep(1)
+
+
# lxd expects the pool's directory to already exist
+
machine.succeed("mkdir /var/lxd-pool")
+
+
+
machine.succeed(
+
"cat ${lxd-config} | lxd init --preseed"
+
)
+
+
machine.succeed(
+
"lxc image import ${alpine-meta} ${alpine-rootfs} --alias alpine"
+
)
+
+
loc = "/var/www/simplestreams/images/iats/alpine/amd64/default/v1"
+
+
with subtest("push image to server"):
+
machine.succeed("lxc launch alpine test")
+
machine.succeed("lxc stop test")
+
machine.succeed("lxc publish --public test --alias=testimg")
+
machine.succeed("lxc image export testimg")
+
machine.succeed("ls >&2")
+
machine.succeed("mkdir -p " + loc)
+
machine.succeed("mv *.tar.gz " + loc)
+
+
with subtest("pull image from server"):
+
machine.succeed("lxc remote add img https://acme.test --protocol=simplestreams")
+
machine.succeed("lxc image list img: >&2")
+
'';
+
})
+30
pkgs/development/python-modules/confight/default.nix
···
+
{ lib
+
, buildPythonPackage
+
, fetchPypi
+
, toml
+
}:
+
+
buildPythonPackage rec {
+
pname = "confight";
+
version = "1.3.1";
+
+
src = fetchPypi {
+
inherit pname version;
+
sha256 = "sha256-fJr7f9Y/zEpCedWYd04AMuhkOFqZLJOw4sDiz8SDQ/Y=";
+
};
+
+
propagatedBuildInputs = [
+
toml
+
];
+
+
pythonImportsCheck = [ "confight" ];
+
+
doCheck = false;
+
+
meta = with lib; {
+
description = "Python context manager for managing pid files";
+
homepage = "https://github.com/avature/confight";
+
license = with licenses; [ mit ];
+
maintainers = with maintainers; [ mkg20001 ];
+
};
+
}
+32
pkgs/development/python-modules/inotify/default.nix
···
+
{ lib
+
, buildPythonPackage
+
, fetchFromGitHub
+
, nose
+
}:
+
+
buildPythonPackage rec {
+
pname = "inotify";
+
version = "unstable-2020-08-27";
+
+
src = fetchFromGitHub {
+
owner = "dsoprea";
+
repo = "PyInotify";
+
rev = "f77596ae965e47124f38d7bd6587365924dcd8f7";
+
sha256 = "X0gu4s1R/Kg+tmf6s8SdZBab2HisJl4FxfdwKktubVc=";
+
fetchSubmodules = false;
+
};
+
+
checkInputs = [
+
nose
+
];
+
+
# dunno what's wrong but the module works regardless
+
doCheck = false;
+
+
meta = with lib; {
+
homepage = "https://github.com/dsoprea/PyInotify";
+
description = "Monitor filesystems events on Linux platforms with inotify";
+
license = licenses.gpl2;
+
platforms = platforms.linux;
+
};
+
}
+47
pkgs/tools/virtualization/lxd-image-server/default.nix
···
+
{ lib
+
, openssl
+
, rsync
+
, python3
+
, fetchFromGitHub
+
}:
+
+
python3.pkgs.buildPythonApplication rec {
+
pname = "lxd-image-server";
+
version = "0.0.4";
+
+
src = fetchFromGitHub {
+
owner = "Avature";
+
repo = "lxd-image-server";
+
rev = version;
+
sha256 = "yx8aUmMfSzyWaM6M7+WcL6ouuWwOpqLzODWSdNgwCwo=";
+
};
+
+
patches = [
+
./state.patch
+
./run.patch
+
];
+
+
propagatedBuildInputs = with python3.pkgs; [
+
setuptools
+
attrs
+
click
+
inotify
+
cryptography
+
confight
+
python-pidfile
+
];
+
+
makeWrapperArgs = [
+
''--prefix PATH ':' "${lib.makeBinPath [ openssl rsync ]}"''
+
];
+
+
doCheck = false;
+
+
meta = with lib; {
+
description = "Creates and manages a simplestreams lxd image server on top of nginx";
+
homepage = "https://github.com/Avature/lxd-image-server";
+
license = licenses.apsl20;
+
platforms = platforms.unix;
+
maintainers = with maintainers; [ mkg20001 ];
+
};
+
}
+25
pkgs/tools/virtualization/lxd-image-server/run.patch
···
+
From df2ce9fb48a3790407646a388e0d220a75496c52 Mon Sep 17 00:00:00 2001
+
From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= <mkg20001@gmail.com>
+
Date: Wed, 3 Nov 2021 14:23:38 +0100
+
Subject: [PATCH] /var/run -> /run
+
+
---
+
lxd_image_server/tools/config.py | 2 +-
+
1 file changed, 1 insertion(+), 1 deletion(-)
+
+
diff --git a/lxd_image_server/tools/config.py b/lxd_image_server/tools/config.py
+
index 60e8973..23d392a 100644
+
--- a/lxd_image_server/tools/config.py
+
+++ b/lxd_image_server/tools/config.py
+
@@ -9,7 +9,7 @@ import confight
+
class Config():
+
+
_lock = Lock()
+
- pidfile = Path('/var/run/lxd-image-server/pidfile')
+
+ pidfile = Path('/run/lxd-image-server/pidfile')
+
data = {}
+
+
@classmethod
+
--
+
2.33.0
+
+49
pkgs/tools/virtualization/lxd-image-server/state.patch
···
+
From 17a1e09eaf8957174425d05200be9ee3e77229f9 Mon Sep 17 00:00:00 2001
+
From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= <mkg20001@gmail.com>
+
Date: Thu, 21 Oct 2021 00:39:08 +0200
+
Subject: [PATCH] Remove system-state changing code
+
+
This is already done by the module on nixOS
+
---
+
lxd_image_server/cli.py | 15 +--------------
+
1 file changed, 1 insertion(+), 14 deletions(-)
+
+
diff --git a/lxd_image_server/cli.py b/lxd_image_server/cli.py
+
index d276e6d..f759bf2 100644
+
--- a/lxd_image_server/cli.py
+
+++ b/lxd_image_server/cli.py
+
@@ -140,30 +140,17 @@ def reload_config():
+
@cli.command()
+
@click.option('--root_dir', default='/var/www/simplestreams',
+
show_default=True)
+
-@click.option('--ssl_dir', default='/etc/nginx/ssl', show_default=True,
+
- callback=lambda ctx, param, val: Path(val))
+
@click.pass_context
+
-def init(ctx, root_dir, ssl_dir):
+
+def init(ctx, root_dir):
+
if not Path(root_dir).exists():
+
logger.error('Root directory does not exists')
+
else:
+
- if not ssl_dir.exists():
+
- os.makedirs(str(ssl_dir))
+
-
+
- if not (ssl_dir / 'nginx.key').exists():
+
- generate_cert(str(ssl_dir))
+
-
+
img_dir = str(Path(root_dir, 'images'))
+
streams_dir = str(Path(root_dir, 'streams/v1'))
+
if not Path(img_dir).exists():
+
os.makedirs(img_dir)
+
if not Path(streams_dir).exists():
+
os.makedirs(streams_dir)
+
- conf_path = Path('/etc/nginx/sites-enabled/simplestreams.conf')
+
- if not conf_path.exists():
+
- conf_path.symlink_to(
+
- '/etc/nginx/sites-available/simplestreams.conf')
+
- os.system('nginx -s reload')
+
+
if not Path(root_dir, 'streams', 'v1', 'images.json').exists():
+
ctx.invoke(update, img_dir=Path(root_dir, 'images'),
+
--
+
2.33.0
+
+2
pkgs/top-level/all-packages.nix
···
lxcfs = callPackage ../os-specific/linux/lxcfs { };
lxd = callPackage ../tools/admin/lxd { };
+
lxd-image-server = callPackage ../tools/virtualization/lxd-image-server { };
+
lzfse = callPackage ../tools/compression/lzfse { };
lzham = callPackage ../tools/compression/lzham { };
+4
pkgs/top-level/python-packages.nix
···
confuse = callPackage ../development/python-modules/confuse { };
+
confight = callPackage ../development/python-modules/confight { };
+
connexion = callPackage ../development/python-modules/connexion { };
consonance = callPackage ../development/python-modules/consonance { };
···
injector = callPackage ../development/python-modules/injector { };
inkex = callPackage ../development/python-modules/inkex { };
+
+
inotify = callPackage ../development/python-modules/inotify { };
inotify-simple = callPackage ../development/python-modules/inotify-simple { };