pyrox.dev
1{ config, lib, pkgs, ... }:
2let
3 inherit (lib) types;
4
5 cfg = config.services.ollama;
6 ollamaPackage = cfg.package.override {
7 inherit (cfg) acceleration;
8 linuxPackages = config.boot.kernelPackages // {
9 nvidia_x11 = config.hardware.nvidia.package;
10 };
11 };
12in
13{
14 options = {
15 services.ollama = {
16 enable = lib.mkEnableOption "ollama server for local large language models";
17 package = lib.mkPackageOption pkgs "ollama" { };
18 home = lib.mkOption {
19 type = types.str;
20 default = "%S/ollama";
21 example = "/home/foo";
22 description = ''
23 The home directory that the ollama service is started in.
24
25 See also `services.ollama.writablePaths` and `services.ollama.sandbox`.
26 '';
27 };
28 models = lib.mkOption {
29 type = types.str;
30 default = "%S/ollama/models";
31 example = "/path/to/ollama/models";
32 description = ''
33 The directory that the ollama service will read models from and download new models to.
34
35 See also `services.ollama.writablePaths` and `services.ollama.sandbox`
36 if downloading models or other mutation of the filesystem is required.
37 '';
38 };
39 sandbox = lib.mkOption {
40 type = types.bool;
41 default = true;
42 example = false;
43 description = ''
44 Whether to enable systemd's sandboxing capabilities.
45
46 This sets [`DynamicUser`](
47 https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#DynamicUser=
48 ), which runs the server as a unique user with read-only access to most of the filesystem.
49
50 See also `services.ollama.writablePaths`.
51 '';
52 };
53 writablePaths = lib.mkOption {
54 type = types.listOf types.str;
55 default = [ ];
56 example = [ "/home/foo" "/mnt/foo" ];
57 description = ''
58 Paths that the server should have write access to.
59
60 This sets [`ReadWritePaths`](
61 https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#ReadWritePaths=
62 ), which allows specified paths to be written to through the default sandboxing.
63
64 See also `services.ollama.sandbox`.
65 '';
66 };
67 listenAddress = lib.mkOption {
68 type = types.str;
69 default = "127.0.0.1:11434";
70 example = "0.0.0.0:11111";
71 description = ''
72 The address which the ollama server HTTP interface binds and listens to.
73 '';
74 };
75 acceleration = lib.mkOption {
76 type = types.nullOr (types.enum [ false "rocm" "cuda" ]);
77 default = null;
78 example = "rocm";
79 description = ''
80 What interface to use for hardware acceleration.
81
82 - `null`: default behavior
83 if `nixpkgs.config.rocmSupport` is enabled, uses `"rocm"`
84 if `nixpkgs.config.cudaSupport` is enabled, uses `"cuda"`
85 otherwise defaults to `false`
86 - `false`: disable GPU, only use CPU
87 - `"rocm"`: supported by most modern AMD GPUs
88 - `"cuda"`: supported by most modern NVIDIA GPUs
89 '';
90 };
91 environmentVariables = lib.mkOption {
92 type = types.attrsOf types.str;
93 default = { };
94 example = {
95 OLLAMA_LLM_LIBRARY = "cpu";
96 HIP_VISIBLE_DEVICES = "0,1";
97 };
98 description = ''
99 Set arbitrary environment variables for the ollama service.
100
101 Be aware that these are only seen by the ollama server (systemd service),
102 not normal invocations like `ollama run`.
103 Since `ollama run` is mostly a shell around the ollama server, this is usually sufficient.
104 '';
105 };
106 };
107 };
108
109 config = lib.mkIf cfg.enable {
110 systemd.services.ollama = {
111 description = "Server for local large language models";
112 wantedBy = [ "multi-user.target" ];
113 after = [ "network.target" ];
114 environment = cfg.environmentVariables // {
115 HOME = cfg.home;
116 OLLAMA_MODELS = cfg.models;
117 OLLAMA_HOST = cfg.listenAddress;
118 };
119 serviceConfig = {
120 ExecStart = "${lib.getExe ollamaPackage} serve";
121 WorkingDirectory = cfg.home;
122 StateDirectory = [ "ollama" ];
123 DynamicUser = cfg.sandbox;
124 ReadWritePaths = cfg.writablePaths;
125 };
126 };
127
128 environment.systemPackages = [ ollamaPackage ];
129 };
130
131 meta.maintainers = with lib.maintainers; [ abysssol onny ];
132}