pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 inherit (lib) types mkRemovedOptionModule;
9
10 cfg = config.services.tabby;
11 format = pkgs.formats.toml { };
12 tabbyPackage = cfg.package.override {
13 inherit (cfg) acceleration;
14 };
15in
16{
17 imports = [
18 (mkRemovedOptionModule [
19 "services"
20 "tabby"
21 "indexInterval"
22 ] "These options are now managed within the tabby WebGUI")
23 ];
24 options = {
25 services.tabby = {
26 enable = lib.mkEnableOption "Self-hosted AI coding assistant using large language models";
27
28 package = lib.mkPackageOption pkgs "tabby" { };
29
30 host = lib.mkOption {
31 type = types.str;
32 default = "127.0.0.1";
33 description = ''
34 Specifies the hostname on which the tabby server HTTP interface listens.
35 '';
36 };
37
38 port = lib.mkOption {
39 type = types.port;
40 default = 11029;
41 description = ''
42 Specifies the bind port on which the tabby server HTTP interface listens.
43 '';
44 };
45
46 model = lib.mkOption {
47 type = types.str;
48 default = "TabbyML/StarCoder-1B";
49 description = ''
50 Specify the model that tabby will use to generate completions.
51
52 This model will be downloaded automatically if it is not already present.
53
54 If you want to utilize an existing model that you've already
55 downloaded you'll need to move it into tabby's state directory which
56 lives in `/var/lib/tabby`. Because the tabby.service is configured to
57 use a DynamicUser the service will need to have been started at least
58 once before you can move the locally existing model into
59 `/var/lib/tabby`. You can set the model to 'none' and tabby will
60 startup and fail to download a model, but will have created the
61 `/var/lib/tabby` directory. You can then copy over the model manually
62 into `/var/lib/tabby`, update the model option to the name you just
63 downloaded and copied over then `nixos-rebuild switch` to start using
64 it.
65
66 $ tabby download --model TabbyML/DeepseekCoder-6.7B
67 $ find ~/.tabby/ | tail -n1
68 /home/ghthor/.tabby/models/TabbyML/DeepseekCoder-6.7B/ggml/q8_0.v2.gguf
69 $ sudo rsync -r ~/.tabby/models/ /var/lib/tabby/models/
70 $ sudo chown -R tabby:tabby /var/lib/tabby/models/
71
72 See for Model Options:
73 > https://github.com/TabbyML/registry-tabby
74 '';
75 };
76
77 acceleration = lib.mkOption {
78 type = types.nullOr (
79 types.enum [
80 "cpu"
81 "rocm"
82 "cuda"
83 "metal"
84 ]
85 );
86 default = null;
87 example = "rocm";
88 description = ''
89 Specifies the device to use for hardware acceleration.
90
91 - `cpu`: no acceleration just use the CPU
92 - `rocm`: supported by modern AMD GPUs
93 - `cuda`: supported by modern NVIDIA GPUs
94 - `metal`: supported on darwin aarch64 machines
95
96 Tabby will try and determine what type of acceleration that is
97 already enabled in your configuration when `acceleration = null`.
98
99 - nixpkgs.config.cudaSupport
100 - nixpkgs.config.rocmSupport
101 - if stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64
102
103 IFF multiple acceleration methods are found to be enabled or if you
104 haven't set either `cudaSupport or rocmSupport` you will have to
105 specify the device type manually here otherwise it will default to
106 the first from the list above or to cpu.
107 '';
108 };
109
110 usageCollection = lib.mkOption {
111 type = types.bool;
112 default = false;
113 description = ''
114 Enable sending anonymous usage data.
115
116 See for more details:
117 > https://tabby.tabbyml.com/docs/configuration#usage-collection
118 '';
119 };
120 };
121 };
122
123 # TODO(ghthor): firewall config
124
125 config = lib.mkIf cfg.enable {
126 environment = {
127 systemPackages = [ tabbyPackage ];
128 };
129
130 systemd =
131 let
132 serviceUser = {
133 WorkingDirectory = "/var/lib/tabby";
134 StateDirectory = [ "tabby" ];
135 ConfigurationDirectory = [ "tabby" ];
136 DynamicUser = true;
137 User = "tabby";
138 Group = "tabby";
139 };
140
141 serviceEnv = lib.mkMerge [
142 {
143 TABBY_ROOT = "%S/tabby";
144 }
145 (lib.mkIf (!cfg.usageCollection) {
146 TABBY_DISABLE_USAGE_COLLECTION = "1";
147 })
148 ];
149 in
150 {
151 services.tabby = {
152 wantedBy = [ "multi-user.target" ];
153 description = "Self-hosted AI coding assistant using large language models";
154 after = [ "network.target" ];
155 environment = serviceEnv;
156 serviceConfig = lib.mkMerge [
157 serviceUser
158 {
159 ExecStart = "${lib.getExe tabbyPackage} serve --model ${cfg.model} --host ${cfg.host} --port ${toString cfg.port} --device ${tabbyPackage.featureDevice}";
160 }
161 ];
162 };
163 };
164 };
165
166 meta.maintainers = with lib.maintainers; [ ghthor ];
167}