comparing 46af5888723ac9b4088a97c7c2c8e201e2d63e7a and refactor/move-global-system-config on soopy.moe/gensokyo

-4

.git-crypt/.gitattributes

···

       1
       1
       -
       # Do not edit this file.  To specify the files to encrypt, create your own

     

       2
       2
       -
       # .gitattributes file in the directory where your files are.

     

       3
       3
       -
       * !filter !diff

     

       4
       4
       -
       *.gpg binary

.git-crypt/keys/default/0/7595B36DF6C2E95E10E528662932BA0FA3DDD7D6.gpg

This is a binary file and will not be displayed.

-1

.gitattributes

···

       1
       1
       -
       *.cry filter=git-crypt diff=git-crypt

+42 -2

docs/utils.md

···

       1
       1
        
       # utility functions

     

       2
       2
        
       

     

       3
       3
        
       ## `_utils.mkVhost`

     

       4
       4
       -
       make virtual host with sensible defaults

     

       4
       4
       +
       `attrset -> attrset`

     

       5
       5
       +
       make a virtual host with sensible defaults

     

       5
       6
        
       

     

       6
       7
        
       pass in a set to override the defaults.

     

       7
       8
        
       

     

       9
       9
       +
       ### Example

     

       10
       10
       +
       ```nix

     

       11
       11
       +
       services.nginx.virtualHosts."balls.example" = _utils.mkVhost {};

     

       12
       12
       +
       ```

     

       13
       13
       +
       

     

       8
       14
        
       ## `_utils.mkSimpleProxy`

     

       15
       15
       +
       `attrset -> attrset`

     

       16
       16
       +
       

     

       9
       17
        
       make a simple reverse proxy

     

       10
       18
        
       

     

       11
       19
        
       takes a set:

     
···

       14
       22
        
         port,

     

       15
       23
        
         protocol ? "http",

     

       16
       24
        
         location ? "/",

     

       17
       17
       -
         websockets ? false

     

       25
       25
       +
         websockets ? false,

     

       26
       26
       +
         extraConfig ? {}

     

       18
       27
        
       }

     

       19
       28
        
       ```

     

       29
       29
       +
       

     

       30
       30
       +
       It is recommended to override/add attributes with `extraConfig` to

     

       31
       31
       +
       preserve defaults.

     

       32
       32
       +
       

     

       33
       33
       +
       Items in `extraConfig` are merged verbatim to the base attrset with defaults.

     

       34
       34
       +
       They are overridden based on their order.

     

       35
       35
       +
       

     

       36
       36
       +
       ## `_utils.genSecrets`

     

       37
       37
       +
       `namespace[str] -> files[list[str]] -> value[attrset] -> attrset`

     

       38
       38
       +
       a

     

       39
       39
       +
       generate an attrset to be passed into sops.secrets.

     

       40
       40
       +
       

     

       41
       41
       +
       ### Example

     

       42
       42
       +
       ```nix

     

       43
       43
       +
       { _utils, ... }:

     

       44
       44
       +
       let

     

       45
       45
       +
         secrets = [

     

       46
       46
       +
           "secure_secret"

     

       47
       47
       +
           # this is a directory structure, so secrets will be stored as a file in /run/secrets/service/test/secret.

     

       48
       48
       +
           "service/test/secret"

     

       49
       49
       +
         ];

     

       50
       50
       +
       in {

     

       51
       51
       +
         sops.secrets = _utils.genSecrets "" secrets {}; # it's recommended to use a namespace, but having none is still fine.

     

       52
       52
       +
         # -> sops.secrets."secure_secret" = {};

     

       53
       53
       +
         #    sops.secrets."service/test/secret" = {};

     

       54
       54
       +
         sops.secrets = _utils.genSecrets "balls" ["balls_secret"] {owner = "balls"};

     

       55
       55
       +
         # -> sops.secrets."balls/balls_secret" = {owner = "balls";};

     

       56
       56
       +
       }

     

       57
       57
       +
       ```

     

       58
       58
       +
       

     

       59
       59
       +
       See https://github.com/soopyc/nix-on-koumakan/blob/b7983776143c15c91df69ef34ba4264a22047ec6/systems/koumakan/services/fedivese/akkoma.nix#L8-L34 for a more extensive example

+1 -9

global/core.nix

···

       1
       1
        
       {pkgs, ...}: {

     

       2
       2
        
         imports = [

     

       3
       3
        
           ./upgrade-diff.nix

     

       4
       4
       +
           ./system

     

       4
       5
        
         ];

     

       5
       6
        
         # Set default i18n configuration

     

       6
       7
        
         i18n.defaultLocale = "en_US.UTF-8";

     
···

       8
       9
        
           font = "Lat2-Terminus16";

     

       9
       10
        
           keyMap = "us";

     

       10
       11
        
         };

     

       11
       11
       -
       

     

       12
       12
       -
         hardware.enableRedistributableFirmware = true;

     

       13
       13
       -
         services.fwupd.enable = true;

     

       14
       14
       -
       

     

       15
       15
       -
         # # Enable crash dumps globally

     

       16
       16
       -
         # boot.crashDump = {

     

       17
       17
       -
         #   enable = true;

     

       18
       18
       -
         #   reservedMemory = "128M";

     

       19
       19
       -
         # };

     

       20
       12
        
       

     

       21
       13
        
         time.timeZone = "Asia/Hong_Kong";

     

       22
       14

+1

global/programs/nix.nix

···

       5
       5
        
           experimental-features = [

     

       6
       6
        
             "nix-command"

     

       7
       7
        
             "flakes"

     

       8
       8
       +
             "repl-flake"

     

       8
       9
        
           ];

     

       9
       10
        
       

     

       10
       11
        
           substituters = [

+5

global/system/default.nix

···

       1
       1
       +
       {...}: {

     

       2
       2
       +
         imports = [

     

       3
       3
       +
           ./firmware.nix

     

       4
       4
       +
         ];

     

       5
       5
       +
       }

+4

global/system/firmware.nix

···

       1
       1
       +
       {...}: {

     

       2
       2
       +
         hardware.enableRedistributableFirmware = true;

     

       3
       3
       +
         services.fwupd.enable = true;

     

       4
       4
       +
       }

+29 -15

global/utils.nix

···

       1
       1
        
       # see /docs/utils.md for a usage guide

     

       2
       2
        
       {

     

       3
       3
        
         inputs,

     

       4
       4
       -
         # system,

     

       4
       4
       +
         system,

     

       5
       5
        
         ...

     

       6
       6
        
       }: let

     

       7
       7
        
         lib = inputs.nixpkgs.lib;

     

       8
       8
        
       in rec {

     

       9
       9
       -
         mkVhost = {...} @ opts:

     

       10
       10
       -
           {

     

       11
       11
       -
             # ideally mkOverride/mkDefault would be used, but i have 0 idea how it works.

     

       12
       12
       -
             forceSSL = true;

     

       13
       13
       -
             useACMEHost = "global.c.soopy.moe";

     

       14
       14
       -
             kTLS = true;

     

       15
       15
       -
           }

     

       16
       16
       -
           // opts;

     

       9
       9
       +
         mkVhost = opts:

     

       10
       10
       +
           lib.mkMerge [

     

       11
       11
       +
             {

     

       12
       12
       +
               forceSSL = lib.mkDefault true;

     

       13
       13
       +
               useACMEHost = lib.mkDefault "global.c.soopy.moe";

     

       14
       14
       +
               kTLS = lib.mkDefault true;

     

       15
       15
       +
       

     

       16
       16
       +
               locations."/_cgi/error/" = {

     

       17
       17
       +
                 alias = "${inputs.mystia.packages.${system}.staticly}/nginx_error_pages/";

     

       18
       18
       +
               };

     

       19
       19
       +
               extraConfig = ''

     

       20
       20
       +
                 error_page 503 /_cgi/error/503.html;

     

       21
       21
       +
                 error_page 502 /_cgi/error/502.html;

     

       22
       22
       +
                 error_page 404 /_cgi/error/404.html;

     

       23
       23
       +
               '';

     

       24
       24
       +
             }

     

       25
       25
       +
             opts

     

       26
       26
       +
           ];

     

       17
       27
        
       

     

       18
       28
        
         mkSimpleProxy = {

     

       19
       29
        
           port,

     

       20
       30
        
           protocol ? "http",

     

       21
       31
        
           location ? "/",

     

       22
       32
        
           websockets ? false,

     

       33
       33
       +
           extraConfig ? {},

     

       23
       34
        
         }:

     

       24
       24
       -
           mkVhost {

     

       25
       25
       -
             locations."${location}" = {

     

       26
       26
       -
               proxyPass = "${protocol}://localhost:${toString port}";

     

       27
       27
       -
               proxyWebsockets = websockets;

     

       28
       28
       -
             };

     

       29
       29
       -
           };

     

       35
       35
       +
           mkVhost (lib.mkMerge [

     

       36
       36
       +
             extraConfig

     

       37
       37
       +
             {

     

       38
       38
       +
               locations."${location}" = {

     

       39
       39
       +
                 proxyPass = "${protocol}://localhost:${toString port}";

     

       40
       40
       +
                 proxyWebsockets = websockets;

     

       41
       41
       +
               };

     

       42
       42
       +
             }

     

       43
       43
       +
           ]);

     

       30
       44
        
       

     

       31
       45
        
         genSecrets = namespace: files: value:

     

       32
       46
        
           lib.genAttrs (

+1 -1

systems/koumakan/networking/interface.nix

···

       1
       1
        
       {...}: {

     

       2
       2
       -
         networking.networkmanager.ethernet.macAddress = builtins.readFile ./nma.cry;

     

       2
       2
       +
         networking.networkmanager.ethernet.macAddress = "stable";

     

       3
       3
        
       }

systems/koumakan/networking/nma.cry

This is a binary file and will not be displayed.

+1 -1

systems/koumakan/security/pam.nix

···

       1
       1
        
       {...}: {

     

       2
       2
        
         security.pam.yubico = {

     

       3
       3
        
           enable = true;

     

       4
       4
       -
           id = builtins.readFile ./ykid.cry;

     

       4
       4
       +
           id = "91582";

     

       5
       5
        
           mode = "client";

     

       6
       6
        
           control = "requisite";

     

       7
       7
        
         };

systems/koumakan/security/ykid.cry

This is a binary file and will not be displayed.

+4 -3

systems/koumakan/services/attic.nix

···

       39
       39
        
           };

     

       40
       40
        
         };

     

       41
       41
        
       

     

       42
       42
       -
         services.nginx.virtualHosts."nonbunary.soopy.moe" =

     

       43
       43
       -
           _utils.mkSimpleProxy {port = 38191;}

     

       44
       44
       -
           // {

     

       42
       42
       +
         services.nginx.virtualHosts."nonbunary.soopy.moe" = _utils.mkSimpleProxy {

     

       43
       43
       +
           port = 38191;

     

       44
       44
       +
           extraConfig = {

     

       45
       45
        
             extraConfig = ''

     

       46
       46
        
               client_max_body_size 1G;

     

       47
       47
        
               proxy_read_timeout 3h;

     
···

       49
       49
        
               proxy_send_timeout 3h;

     

       50
       50
        
             '';

     

       51
       51
        
           };

     

       52
       52
       +
         };

     

       52
       53
        
       }

+3 -3

systems/koumakan/services/static-sites/keine.nix

···

       1
       1
       -
       {...}: {

     

       2
       2
       -
         services.nginx.virtualHosts."keine.soopy.moe" = {

     

       3
       3
       -
           useACMEHost = "global.c.soopy.moe";

     

       1
       1
       +
       {_utils, ...}: {

     

       2
       2
       +
         services.nginx.virtualHosts."keine.soopy.moe" = _utils.mkVhost {

     

       3
       3
       +
           forceSSL = false;

     

       4
       4
        
           addSSL = true; # Don't force SSL on a mirror (implications TBD)

     

       5
       5
        
       

     

       6
       6
        
           root = "/srv/www/keine";

+78 -64

utils/nitter-guest-account.py

···

       3
       3
        
       # thank you!

     

       4
       4
        
       import sys

     

       5
       5
        
       import json

     

       6
       6
       +
       import time

     

       7
       7
       +
       import random

     

       6
       8
        
       import typing

     

       7
       9
        
       import traceback

     

       8
       10
        
       from base64 import b64encode

     
···

       256
       258
        
       	success(f'flow token => {flow_token}')

     

       257
       259
        
       

     

       258
       260
        
       	info('Fetching final account object...')

     

       259
       259
       -
       	tasks = send_req('post', TASKS_ENDPOINT, headers=request_headers, json=get_tasks_body(flow_token)).json()

     

       260
       260
       -
       	try:

     

       261
       261
       +
       	backoff_time = 0

     

       262
       262
       +
       	exception = None

     

       263
       263
       +
       	for attempt in range(0, 6):

     

       264
       264
       +
       		debug(f"Attempt #{attempt + 1}")

     

       265
       265
       +
       		tasks = send_req('post', TASKS_ENDPOINT, headers=request_headers, json=get_tasks_body(flow_token)).json()

     

       261
       266
        
       		try:

     

       262
       262
       -
       			open_account_task = next(filter(lambda i: i.get('subtask_id') == "OpenAccount", tasks['subtasks']))

     

       263
       263
       -
       			account = open_account_task['open_account']

     

       264
       264
       -
       		except StopIteration:

     

       265
       265
       -
       			debug("resulting tasks =>", format_json(tasks), override=True)

     

       266
       266
       -
       			error("Unable to acquire guest account credentials as it isn't present in the API response.")

     

       267
       267
       -
       			error("This might be because of a wide variety of reasons, but it most likely is due to your IP being rate-limited.")

     

       268
       268
       -
       			error("Try again with a new IP address or in 24 hours after this attempt.")

     

       269
       269
       -
       			return 1

     

       267
       267
       +
       			try:

     

       268
       268
       +
       				open_account_task = next(filter(lambda i: i.get('subtask_id') == "OpenAccount", tasks['subtasks']))

     

       269
       269
       +
       				account = open_account_task['open_account']

     

       270
       270
       +
       			except StopIteration as e:

     

       271
       271
       +
       				backoff_time += random.randint(5000,10000) / 1000

     

       272
       272
       +
       				exception = e

     

       273
       273
       +
       				warn(f"attempt #{attempt + 1} failed to acquire token, retrying in {backoff_time}s.")

     

       274
       274
       +
       				time.sleep(backoff_time)

     

       275
       275
       +
       				continue

     

       270
       276
        
       

     

       271
       271
       -
       		if args.outfile == "-":

     

       272
       272
       -
       			debug("outfile is `-`, printing to stdout")

     

       273
       273
       -
       			print(format_json(account))

     

       274
       274
       -
       			return 0

     

       277
       277
       +
       			info(f"Attempt #{attempt + 1} succeeded")

     

       278
       278
       +
       			if args.outfile == "-":

     

       279
       279
       +
       				debug("outfile is `-`, printing to stdout")

     

       280
       280
       +
       				print(format_json(account))

     

       281
       281
       +
       				return 0

     

       275
       282
        
       

     

       276
       276
       -
       		# Sanity checks

     

       277
       277
       -
       		try:

     

       278
       278
       -
       			debug(f"attempting to read file: {args.outfile}")

     

       279
       279
       -
       			with open(args.outfile) as f:

     

       280
       280
       -
       				old_data = json.load(f)

     

       281
       281
       -
       		except FileNotFoundError:

     

       282
       282
       -
       			# that's okay, we might be able to create it later.

     

       283
       283
       -
       			old_data = []

     

       284
       284
       -
       		except PermissionError:

     

       285
       285
       -
       			# that's not okay. we will need to access the file later anyways.

     

       286
       286
       -
       			error("unable to read file due to a permission error.")

     

       287
       287
       -
       			error("please make sure this script has read and write access to the file.")

     

       288
       288
       -
       			print(format_json(account))

     

       289
       289
       -
       			return 1

     

       290
       290
       -
       		except json.JSONDecodeError:

     

       291
       291
       -
       			error("could not parse the provided JSON file.")

     

       292
       292
       -
       			if not prompt_bool("Do you want to overwrite the file?", default=None):

     

       293
       293
       -
       				warn("Not overwriting file, printing to stdout instead.")

     

       283
       283
       +
       			# Sanity checks

     

       284
       284
       +
       			try:

     

       285
       285
       +
       				debug(f"attempting to read file: {args.outfile}")

     

       286
       286
       +
       				with open(args.outfile) as f:

     

       287
       287
       +
       					old_data = json.load(f)

     

       288
       288
       +
       			except FileNotFoundError:

     

       289
       289
       +
       				# that's okay, we might be able to create it later.

     

       290
       290
       +
       				old_data = []

     

       291
       291
       +
       			except PermissionError:

     

       292
       292
       +
       				# that's not okay. we will need to access the file later anyways.

     

       293
       293
       +
       				error("unable to read file due to a permission error.")

     

       294
       294
       +
       				error("please make sure this script has read and write access to the file.")

     

       294
       295
        
       				print(format_json(account))

     

       295
       296
        
       				return 1

     

       296
       296
       -
       			debug('assuming old data is an empty array because we are overwriting')

     

       297
       297
       -
       			old_data = []

     

       298
       298
       -
       		if type(old_data) != list:

     

       299
       299
       -
       			error("top-level object of the existing JSON file is not a list.")

     

       300
       300
       -
       			error("due to the implementation, the file must be overwritten.")

     

       301
       301
       -
       			if not (prompt_bool("Do you want to overwrite?", default=None)):

     

       302
       302
       -
       				warn("Not overwriting existing data, printing to stdout instead.")

     

       297
       297
       +
       			except json.JSONDecodeError:

     

       298
       298
       +
       				error("could not parse the provided JSON file.")

     

       299
       299
       +
       				if not prompt_bool("Do you want to overwrite the file?", default=None):

     

       300
       300
       +
       					warn("Not overwriting file, printing to stdout instead.")

     

       301
       301
       +
       					print(format_json(account))

     

       302
       302
       +
       					return 1

     

       303
       303
       +
       				debug('assuming old data is an empty array because we are overwriting')

     

       304
       304
       +
       				old_data = []

     

       305
       305
       +
       			if type(old_data) != list:

     

       306
       306
       +
       				error("top-level object of the existing JSON file is not a list.")

     

       307
       307
       +
       				error("due to the implementation, the file must be overwritten.")

     

       308
       308
       +
       				if not (prompt_bool("Do you want to overwrite?", default=None)):

     

       309
       309
       +
       					warn("Not overwriting existing data, printing to stdout instead.")

     

       310
       310
       +
       					print(format_json(account))

     

       311
       311
       +
       					return 1

     

       312
       312
       +
       				debug("assuming old data is an empty array because we are overwriting")

     

       313
       313
       +
       				old_data = []

     

       314
       314
       +
       

     

       315
       315
       +
       			old_data.append(account)

     

       316
       316
       +
       

     

       317
       317
       +
       			try:

     

       318
       318
       +
       				debug("attempting to write file")

     

       319
       319
       +
       				with open(args.outfile, 'w+') as f:

     

       320
       320
       +
       					f.write(format_json(old_data))

     

       321
       321
       +
       				success(f"successfully written to file {args.outfile}")

     

       322
       322
       +
       				return 0

     

       323
       323
       +
       			except PermissionError:

     

       324
       324
       +
       				error("unable to write to file due to permission error.")

     

       325
       325
       +
       				error("please make sure this script has write access to the file.")

     

       303
       326
        
       				print(format_json(account))

     

       304
       327
        
       				return 1

     

       305
       305
       -
       			debug("assuming old data is an empty array because we are overwriting")

     

       306
       306
       -
       			old_data = []

     

       307
       307
       -
       

     

       308
       308
       -
       		old_data.append(account)

     

       328
       328
       +
       			except Exception as e:

     

       329
       329
       +
       				error("Unable to write to file due to an uncaught error:", e)

     

       330
       330
       +
       				tb = ''.join(traceback.TracebackException.from_exception(e).format())

     

       331
       331
       +
       				debug("exception stacktrace\n" + tb)

     

       332
       332
       +
       				print(format_json(account))

     

       309
       333
        
       

     

       310
       310
       -
       		try:

     

       311
       311
       -
       			debug("attempting to write file")

     

       312
       312
       -
       			with open(args.outfile, 'w+') as f:

     

       313
       313
       -
       				f.write(format_json(old_data))

     

       314
       314
       -
       			success(f"successfully written to file {args.outfile}")

     

       315
       315
       -
       			return 0

     

       316
       316
       -
       		except PermissionError:

     

       317
       317
       -
       			error("unable to write to file due to permission error.")

     

       318
       318
       -
       			error("please make sure this script has write access to the file.")

     

       319
       319
       -
       			print(format_json(account))

     

       320
       320
       -
       			return 1

     

       321
       321
       -
       		except Exception as e:

     

       322
       322
       -
       			error("Unable to write to file due to an uncaught error:", e)

     

       323
       323
       -
       			tb = ''.join(traceback.TracebackException.from_exception(e).format())

     

       324
       324
       -
       			debug("exception stacktrace\n" + tb)

     

       325
       325
       -
       			print(format_json(account))

     

       334
       334
       +
       		except Exception:

     

       335
       335
       +
       			debug("resulting tasks =>", format_json(tasks), override=True)

     

       336
       336
       +
       			error("an unhandled error occurred. the tasks object is printed to avoid losing otherwise successful data.")

     

       337
       337
       +
       			error("please file a bug report and attach the traceback below.")

     

       338
       338
       +
       			raise

     

       326
       339
        
       

     

       327
       327
       -
       	except Exception:

     

       328
       328
       -
       		debug("resulting tasks =>", format_json(tasks), override=True)

     

       329
       329
       -
       		error("an unhandled error occurred. the tasks object is printed to avoid losing otherwise successful data.")

     

       330
       330
       -
       		error("please file a bug report and attach the traceback below.")

     

       331
       331
       -
       		raise

     

       340
       340
       +
       	if exception != None:

     

       341
       341
       +
       		debug("resulting tasks =>", format_json(tasks))

     

       342
       342
       +
       		error("Unable to acquire guest account credentials with 5 attempts as it isn't present in any of the API responses.")

     

       343
       343
       +
       		error("This might be because of a wide variety of reasons, but it most likely is due to your IP being rate-limited.")

     

       344
       344
       +
       		error("Try again with a new IP address or in 24 hours after this attempt.")

     

       345
       345
       +
       		return 1

     

       332
       346
        
       

     

       333
       347
        
       	return 0

     

       334
       348

Compare changes