commit d8d0e036cb6caa2f5dcc93eb86583a22df71216f · soopy.moe/gensokyo

-46

systems/koumakan/services/proxies/minio.nix

···

       2
       2
        
         _utils,

     

       3
       3
        
         lib,

     

       4
       4
        
         config,

     

       5
       5
       -
         inputs,

     

       6
       5
        
         ...

     

       7
       6
        
       }:

     

       8
       7
        
       let

     
···

       35
       34
        
         systemd.services.minio.environment = {

     

       36
       35
        
           MINIO_BROWSER_REDIRECT_URL = "https://s3.soopy.moe/_static";

     

       37
       36
        
           MINIO_BROWSER_LOGIN_ANIMATION = "false";

     

       38
       38
       -
         };

     

       39
       39
       -
       

     

       40
       40
       -
         services.nginx.virtualHosts = {

     

       41
       41
       -
           "s3.soopy.moe" = _utils.mkSimpleProxy {

     

       42
       42
       -
             port = 26531;

     

       43
       43
       -
             extraConfig = {

     

       44
       44
       -
               extraConfig = ''

     

       45
       45
       -
                 client_max_body_size 32G;

     

       46
       46
       -
               '';

     

       47
       47
       -
       

     

       48
       48
       -
               locations."= /_static" = _utils.mkNginxFile {

     

       49
       49
       -
                 content = ''

     

       50
       50
       -
                   <!doctype html>

     

       51
       51
       -
                   <html lang="en">

     

       52
       52
       -
                   <head>

     

       53
       53
       -
                     <title>horrors of gensokyo</title>

     

       54
       54
       -
                     <style>

     

       55
       55
       -
                       :root {font-family: "monospace";}

     

       56
       56
       -
                     </style>

     

       57
       57
       -
                   </head>

     

       58
       58
       -
                   <body>

     

       59
       59
       -
                     <h1>gensokyo filedump - public buckets</h1>

     

       60
       60
       -
                     <ul>

     

       61
       61
       -
                       <li><a href="//cache.soopy.moe">nix-cache</a></li>

     

       62
       62
       -
                       <li>lwjgl-nix</li>

     

       63
       63
       -
                     </ul>

     

       64
       64
       -
                   </body>

     

       65
       65
       -
                   </html>

     

       66
       66
       -
                 '';

     

       67
       67
       -
               };

     

       68
       68
       -
             };

     

       69
       69
       -
           };

     

       70
       70
       -
       

     

       71
       71
       -
           "cache.soopy.moe" = _utils.mkVhost {

     

       72
       72
       -
             locations."/".proxyPass = "http://localhost:26531/nix-cache/";

     

       73
       73
       -
       

     

       74
       74
       -
             locations."= /" = {

     

       75
       75
       -
               tryFiles = "/listing.html =500";

     

       76
       76
       -
               root = inputs.mystia.packages.x86_64-linux.s3-listing.override {

     

       77
       77
       -
                 bucketName = "nix-cache";

     

       78
       78
       -
                 bucketUrl = "https://s3.soopy.moe/nix-cache/";

     

       79
       79
       -
                 bucketWebsiteUrl = "https://cache.soopy.moe";

     

       80
       80
       -
               };

     

       81
       81
       -
             };

     

       82
       82
       -
           };

     

       83
       37
        
         };

     

       84
       38
        
       

     

       85
       39
        
         systemd.services.vmagent.serviceConfig.LoadCredential = [

+71 -4

systems/koumakan/services/storage/garage.nix

···

       1
       1
        
       {

     

       2
       2
        
         _utils,

     

       3
       3
        
         config,

     

       4
       4
       +
         lib,

     

       4
       5
        
         pkgs,

     

       5
       6
        
         ...

     

       6
       7
        
       }:

     
···

       16
       17
        
       in

     

       17
       18
        
       {

     

       18
       19
        
         imports = [ secrets.generate ];

     

       20
       20
       +
       

     

       21
       21
       +
         users = {

     

       22
       22
       +
           users.garage = {

     

       23
       23
       +
             isSystemUser = true;

     

       24
       24
       +
             group = "garage";

     

       25
       25
       +
           };

     

       26
       26
       +
           groups.garage = {};

     

       27
       27
       +
         };

     

       28
       28
       +
       

     

       19
       29
        
         services.garage = {

     

       20
       30
        
           enable = true;

     

       21
       31
        
           package = pkgs.garage_2;

     
···

       30
       40
        
             replication_factor = 1; # we only have the resources for a single node atm.

     

       31
       41
        
             compression_level = 4;

     

       32
       42
        
       

     

       33
       33
       -
             rpc_bind_addr = "100.100.16.16:39931";

     

       34
       34
       -
             rpc_public_addr = "koumakan.mist-nessie.ts.net:39931";

     

       35
       35
       -
             rpc_secret_file = secrets.get "rpc_secret";

     

       36
       36
       -
       

     

       37
       43
        
             s3_api = {

     

       38
       44
        
               s3_region = "ap-east-1";

     

       39
       45
        
               api_bind_addr = "[::1]:39930";

     

       40
       46
        
               root_domain = ".s3.soopy.moe";

     

       41
       47
        
             };

     

       42
       48
        
       

     

       49
       49
       +
             rpc_bind_addr = "100.100.16.16:39931";

     

       50
       50
       +
             rpc_public_addr = "koumakan.mist-nessie.ts.net:39931";

     

       51
       51
       +
             rpc_secret_file = secrets.get "rpc_secret";

     

       52
       52
       +
       

     

       43
       53
        
             admin = {

     

       44
       54
        
               api_bind_addr = "100.100.16.16:39932";

     

       45
       55
        
               admin_token_file = secrets.get "admin_token";

     

       46
       56
        
               metrics_token_file = secrets.get "metrics_token";

     

       47
       57
        
             };

     

       48
       58
        
           };

     

       59
       59
       +
         };

     

       60
       60
       +
       

     

       61
       61
       +
         systemd.services.garage.serviceConfig = {

     

       62
       62
       +
           DynamicUser = false; # we need to use a mounted filesystem and systemd explodes when i already have a mountpoint at /var/lib/garage/data.

     

       63
       63
       +
           User = config.users.users.garage.name;

     

       64
       64
       +
           Group = config.users.groups.garage.name;

     

       65
       65
       +
           Restart = "on-failure";

     

       66
       66
       +
         };

     

       67
       67
       +
       

     

       68
       68
       +
         services.nginx.virtualHosts.".s3.soopy.moe" = _utils.mkSimpleProxy {

     

       69
       69
       +
           port = 39930;

     

       70
       70
       +
           extraConfig = {

     

       71
       71
       +
             extraConfig = ''

     

       72
       72
       +
               client_max_body_size 32G;

     

       73
       73
       +
               proxy_max_temp_file_size 0;

     

       74
       74
       +
             '';

     

       75
       75
       +
       

     

       76
       76
       +
             locations."= /_static" = _utils.mkNginxFile {

     

       77
       77
       +
               content = ''

     

       78
       78
       +
                 <!doctype html>

     

       79
       79
       +
                 <html lang="en">

     

       80
       80
       +
                 <head>

     

       81
       81
       +
                   <title>horrors of gensokyo</title>

     

       82
       82
       +
                   <style>

     

       83
       83
       +
                     :root {font-family: "monospace";}

     

       84
       84
       +
                   </style>

     

       85
       85
       +
                 </head>

     

       86
       86
       +
                 <body>

     

       87
       87
       +
                   <h1>gensokyo filedump - public buckets</h1>

     

       88
       88
       +
                   <ul>

     

       89
       89
       +
                     <li><a href="//cache.soopy.moe">nix-cache</a></li>

     

       90
       90
       +
                     <li>lwjgl-nix</li>

     

       91
       91
       +
                   </ul>

     

       92
       92
       +
                 </body>

     

       93
       93
       +
                 </html>

     

       94
       94
       +
               '';

     

       95
       95
       +
             };

     

       96
       96
       +
           };

     

       97
       97
       +
         };

     

       98
       98
       +
       

     

       99
       99
       +
         systemd.services.vmagent.serviceConfig.LoadCredential = [

     

       100
       100
       +
           "garage_token:${secrets.get "metrics_token"}"

     

       101
       101
       +
         ];

     

       102
       102
       +
       

     

       103
       103
       +
         services.vmagent.prometheusConfig.scrape_configs = lib.singleton {

     

       104
       104
       +
           job_name = "garage-job";

     

       105
       105
       +
           scheme = "http";

     

       106
       106
       +
           static_configs = lib.singleton { targets = lib.singleton "localhost:39932"; };

     

       107
       107
       +
           relabel_configs = lib.singleton {

     

       108
       108
       +
             target_label = "instance";

     

       109
       109
       +
             replacement = config.networking.fqdnOrHostName;

     

       110
       110
       +
           };

     

       111
       111
       +
       

     

       112
       112
       +
           # https://github.com/NixOS/nixpkgs/issues/367447

     

       113
       113
       +
           # https://docs.victoriametrics.com/sd_configs/#scrape_configs

     

       114
       114
       +
           # hard coding because we can't use %{ENV_VAR} syntax (yet) when checking.

     

       115
       115
       +
           bearer_token_file = "/run/credentials/vmagent.service/garage_token";

     

       49
       116
        
         };

     

       50
       117
        
       }

systems/koumakan/services/telemetry/grafana/dashboards/garage.json

···

       1
       1
       +
       {"__inputs":[{"name":"DS_DS_PROMETHEUS","label":"DS_PROMETHEUS","description":"","type":"datasource","pluginId":"prometheus","pluginName":"Prometheus"}],"__elements":{},"__requires":[{"type":"grafana","id":"grafana","name":"Grafana","version":"9.2.0"},{"type":"datasource","id":"prometheus","name":"Prometheus","version":"1.0.0"},{"type":"panel","id":"timeseries","name":"Time series","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":null,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":24,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":9,"w":8,"x":0,"y":0},"id":10,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"sum(rate(block_bytes_read{job=\"garage\"}[$__rate_interval])    )","hide":false,"interval":"","legendFormat":"Disk bytes read","refId":"A"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"-sum(rate(block_bytes_written{job=\"garage\"}[$__rate_interval])    )","hide":false,"interval":"","legendFormat":"Disk bytes written","refId":"B"}],"title":"Disk I/O","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"reqps"},"overrides":[]},"gridPos":{"h":9,"w":8,"x":8,"y":0},"id":3,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"editorMode":"code","exemplar":true,"expr":"sum by (api_endpoint) (rate(api_s3_request_counter {job=\"garage\"}[$__rate_interval]))","hide":false,"interval":"","legendFormat":"{{api_endpoint}}","range":true,"refId":"A"}],"title":"API requests","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"reqps"},"overrides":[]},"gridPos":{"h":9,"w":8,"x":16,"y":0},"id":9,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"sum(rate(web_request_counter {job=\"garage\"}[$__rate_interval]))","hide":false,"interval":"","legendFormat":"Web request rate","refId":"A"}],"title":"Web requests","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"reqps"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":9},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"sum by (rpc_endpoint) (rate(rpc_request_counter {job=\"garage\"}[$__rate_interval]))","hide":false,"interval":"","legendFormat":"{{rpc_endpoint}}","refId":"A"}],"title":"RPC requests","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"reqps"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":9},"id":4,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"editorMode":"code","exemplar":true,"expr":"sum by (api_endpoint, status_code) (rate(api_s3_error_counter {job=\"garage\"}[$__rate_interval]))","hide":false,"interval":"","legendFormat":"{{api_endpoint}} {{status_code}}","range":true,"refId":"A"}],"title":"API errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"reqps"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":9},"id":11,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"sum by(status_code) (rate(web_error_counter {job=\"garage\"}[$__rate_interval]))","hide":false,"interval":"","legendFormat":"{{status_code}}","refId":"A"}],"title":"Web errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"__systemRef":"hideSeriesFrom","matcher":{"id":"byNames","options":{"mode":"exclude","names":["10.83.2.3:3903"],"prefix":"All except:","readOnly":true}},"properties":[{"id":"custom.hideFrom","value":{"legend":false,"tooltip":false,"viz":true}}]}]},"gridPos":{"h":8,"w":8,"x":0,"y":17},"id":6,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"block_resync_queue_length{job=\"garage\"}","interval":"","legendFormat":"{{instance}}","refId":"A"}],"title":"Resync queue length","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":17},"id":7,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"sum by(table_name) (table_gc_todo_queue_length{job=\"garage\"})","interval":"","legendFormat":"{{ table_name}}","refId":"A"}],"title":"Table GC queue length","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":17},"id":8,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"sum by(table_name) (table_merkle_updater_todo_queue_length{job=\"garage\"})","interval":"","legendFormat":"{{ table_name}}","refId":"A"}],"title":"Table Merkle updater queue length","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":25},"id":12,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${DS_DS_PROMETHEUS}"},"exemplar":true,"expr":"block_resync_errored_blocks{job=\"garage\"}","interval":"","legendFormat":"{{instance}}","refId":"A"}],"title":"Resync errored blocks","type":"timeseries"}],"refresh":"30s","schemaVersion":37,"style":"dark","tags":[],"templating":{"list":[]},"time":{"from":"now-6h","to":"now"},"timepicker":{},"timezone":"","title":"Garage","uid":"ys3pnpZ4k","version":26,"weekStart":""}