pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 inherit (lib) types;
9
10 cfg = config.services.litellm;
11 settingsFormat = pkgs.formats.yaml { };
12in
13{
14 options = {
15 services.litellm = {
16 enable = lib.mkEnableOption "LiteLLM server";
17 package = lib.mkPackageOption pkgs "litellm" { };
18
19 stateDir = lib.mkOption {
20 type = types.path;
21 default = "/var/lib/litellm";
22 example = "/home/foo";
23 description = "State directory of LiteLLM.";
24 };
25
26 host = lib.mkOption {
27 type = types.str;
28 default = "127.0.0.1";
29 example = "0.0.0.0";
30 description = ''
31 The host address which the LiteLLM server HTTP interface listens to.
32 '';
33 };
34
35 port = lib.mkOption {
36 type = types.port;
37 default = 8080;
38 example = 11111;
39 description = ''
40 Which port the LiteLLM server listens to.
41 '';
42 };
43
44 settings = lib.mkOption {
45 type = types.submodule {
46 freeformType = settingsFormat.type;
47 options = {
48 model_list = lib.mkOption {
49 type = settingsFormat.type;
50 description = ''
51 List of supported models on the server, with model-specific configs.
52 '';
53 default = [ ];
54 };
55 router_settings = lib.mkOption {
56 type = settingsFormat.type;
57 description = ''
58 LiteLLM Router settings
59 '';
60 default = { };
61 };
62
63 litellm_settings = lib.mkOption {
64 type = settingsFormat.type;
65 description = ''
66 LiteLLM Module settings
67 '';
68 default = { };
69 };
70
71 general_settings = lib.mkOption {
72 type = settingsFormat.type;
73 description = ''
74 LiteLLM Server settings
75 '';
76 default = { };
77 };
78
79 environment_variables = lib.mkOption {
80 type = settingsFormat.type;
81 description = ''
82 Environment variables to pass to the Lite
83 '';
84 default = { };
85 };
86 };
87 };
88 default = { };
89 description = ''
90 Configuration for LiteLLM.
91 See <https://docs.litellm.ai/docs/proxy/configs> for more.
92 '';
93 };
94
95 environment = lib.mkOption {
96 type = types.attrsOf types.str;
97 default = {
98 SCARF_NO_ANALYTICS = "True";
99 DO_NOT_TRACK = "True";
100 ANONYMIZED_TELEMETRY = "False";
101 };
102 example = ''
103 {
104 NO_DOCS="True";
105 }
106 '';
107 description = ''
108 Extra environment variables for LiteLLM.
109 '';
110 };
111
112 environmentFile = lib.mkOption {
113 description = ''
114 Environment file to be passed to the systemd service.
115 Useful for passing secrets to the service to prevent them from being
116 world-readable in the Nix store.
117 '';
118 type = lib.types.nullOr lib.types.path;
119 default = null;
120 example = "/var/lib/secrets/liteLLMSecrets";
121 };
122
123 openFirewall = lib.mkOption {
124 type = types.bool;
125 default = false;
126 description = ''
127 Whether to open the firewall for LiteLLM.
128 This adds `services.litellm.port` to `networking.firewall.allowedTCPPorts`.
129 '';
130 };
131 };
132 };
133
134 config = lib.mkIf cfg.enable {
135 systemd.services.litellm = {
136 description = "LLM Gateway to provide model access, fallbacks and spend tracking across 100+ LLMs.";
137 wantedBy = [ "multi-user.target" ];
138 after = [ "network.target" ];
139
140 environment = cfg.environment;
141
142 serviceConfig =
143 let
144 configFile = settingsFormat.generate "config.yaml" cfg.settings;
145 in
146 {
147 ExecStart = "${lib.getExe cfg.package} --host \"${cfg.host}\" --port ${toString cfg.port} --config ${configFile}";
148 EnvironmentFile = lib.optional (cfg.environmentFile != null) cfg.environmentFile;
149 WorkingDirectory = cfg.stateDir;
150 StateDirectory = "litellm";
151 RuntimeDirectory = "litellm";
152 RuntimeDirectoryMode = "0755";
153 PrivateTmp = true;
154 DynamicUser = true;
155 DevicePolicy = "closed";
156 LockPersonality = true;
157 PrivateUsers = true;
158 ProtectHome = true;
159 ProtectHostname = true;
160 ProtectKernelLogs = true;
161 ProtectKernelModules = true;
162 ProtectKernelTunables = true;
163 ProtectControlGroups = true;
164 RestrictNamespaces = true;
165 RestrictRealtime = true;
166 SystemCallArchitectures = "native";
167 UMask = "0077";
168 RestrictAddressFamilies = [
169 "AF_INET"
170 "AF_INET6"
171 "AF_UNIX"
172 ];
173 ProtectClock = true;
174 ProtectProc = "invisible";
175 };
176 };
177
178 networking.firewall = lib.mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port ]; };
179 };
180
181 meta.maintainers = with lib.maintainers; [ ];
182}