pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 inherit (lib) types;
9
10 format = pkgs.formats.yaml { };
11 cfg = config.services.private-gpt;
12in
13{
14 options = {
15 services.private-gpt = {
16 enable = lib.mkEnableOption "private-gpt for local large language models";
17 package = lib.mkPackageOption pkgs "private-gpt" { };
18
19 stateDir = lib.mkOption {
20 type = types.path;
21 default = "/var/lib/private-gpt";
22 description = "State directory of private-gpt.";
23 };
24
25 settings = lib.mkOption {
26 type = format.type;
27 default = {
28 llm = {
29 mode = "ollama";
30 tokenizer = "";
31 };
32 embedding = {
33 mode = "ollama";
34 };
35 ollama = {
36 llm_model = "llama3";
37 embedding_model = "nomic-embed-text";
38 api_base = "http://localhost:11434";
39 embedding_api_base = "http://localhost:11434";
40 keep_alive = "5m";
41 tfs_z = 1;
42 top_k = 40;
43 top_p = 0.9;
44 repeat_last_n = 64;
45 repeat_penalty = 1.2;
46 request_timeout = 120;
47 };
48 vectorstore = {
49 database = "qdrant";
50 };
51 qdrant = {
52 path = "/var/lib/private-gpt/vectorstore/qdrant";
53 };
54 data = {
55 local_data_folder = "/var/lib/private-gpt";
56 };
57 openai = { };
58 azopenai = { };
59 };
60 description = ''
61 settings-local.yaml for private-gpt
62 '';
63 };
64 };
65 };
66
67 config = lib.mkIf cfg.enable {
68 systemd.services.private-gpt = {
69 description = "Interact with your documents using the power of GPT, 100% privately, no data leaks";
70 wantedBy = [ "multi-user.target" ];
71 after = [ "network.target" ];
72
73 preStart =
74 let
75 config = format.generate "settings-local.yaml" (cfg.settings // { server.env_name = "local"; });
76 in
77 ''
78 mkdir -p ${cfg.stateDir}/{settings,huggingface,matplotlib,tiktoken_cache}
79 cp ${cfg.package.cl100k_base.tiktoken} ${cfg.stateDir}/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
80 cp ${pkgs.python3Packages.private-gpt}/${pkgs.python3.sitePackages}/private_gpt/settings.yaml ${cfg.stateDir}/settings/settings.yaml
81 cp "${config}" "${cfg.stateDir}/settings/settings-local.yaml"
82 chmod 600 "${cfg.stateDir}/settings/settings-local.yaml"
83 '';
84
85 environment = {
86 PGPT_PROFILES = "local";
87 PGPT_SETTINGS_FOLDER = "${cfg.stateDir}/settings";
88 HF_HOME = "${cfg.stateDir}/huggingface";
89 TRANSFORMERS_OFFLINE = "1";
90 HF_DATASETS_OFFLINE = "1";
91 MPLCONFIGDIR = "${cfg.stateDir}/matplotlib";
92 };
93
94 serviceConfig = {
95 ExecStart = lib.getExe cfg.package;
96 WorkingDirectory = cfg.stateDir;
97 StateDirectory = "private-gpt";
98 RuntimeDirectory = "private-gpt";
99 RuntimeDirectoryMode = "0755";
100 PrivateTmp = true;
101 DynamicUser = true;
102 DevicePolicy = "closed";
103 LockPersonality = true;
104 MemoryDenyWriteExecute = true;
105 PrivateUsers = true;
106 ProtectHome = true;
107 ProtectHostname = true;
108 ProtectKernelLogs = true;
109 ProtectKernelModules = true;
110 ProtectKernelTunables = true;
111 ProtectControlGroups = true;
112 ProcSubset = "pid";
113 RestrictNamespaces = true;
114 RestrictRealtime = true;
115 SystemCallArchitectures = "native";
116 UMask = "0077";
117 };
118 };
119 };
120
121 meta.maintainers = [ ];
122}