···

       
167
       
167
       
 
       


     

       
168
       
168
       
 
       
    todo: get raspi vcgencmd outputs into metrics

     

       
169
       
169
       
 
       


     

       
170
       
170
       
+
       
- nginx on gateway

     

       
171
       
171
       
+
       


     

       
172
       
172
       
+
       
    ```nginx

     

       
173
       
173
       
+
       
    # in http

     

       
174
       
174
       
+
       


     

       
175
       
175
       
+
       
    ##

     

       
176
       
176
       
+
       
    # cozy cache

     

       
177
       
177
       
+
       
    ##

     

       
178
       
178
       
+
       
    proxy_cache_path /var/cache/nginx keys_zone=cozy_zone:10m;

     

       
179
       
179
       
+
       


     

       
180
       
180
       
+
       
    ##

     

       
181
       
181
       
+
       
    # cozy limit

     

       
182
       
182
       
+
       
    ##

     

       
183
       
183
       
+
       
    limit_req_zone $binary_remote_addr zone=cozy_ip_limit:10m rate=50r/s;

     

       
184
       
184
       
+
       
    limit_req_zone $server_name zone=cozy_global_limit:10m rate=1000r/s;

     

       
185
       
185
       
+
       


     

       
186
       
186
       
+
       
    # in sites-available/linsks.bsky.bad-example.com

     

       
187
       
187
       
+
       


     

       
188
       
188
       
+
       
    upstream cozy_link_aggregator {

     

       
189
       
189
       
+
       
      server link-aggregator:6789;

     

       
190
       
190
       
+
       
      keepalive 16;

     

       
191
       
191
       
+
       
    }

     

       
192
       
192
       
+
       


     

       
193
       
193
       
+
       
    server {

     

       
194
       
194
       
+
       
      listen 8080;

     

       
195
       
195
       
+
       
      listen [::]:8080;

     

       
196
       
196
       
+
       


     

       
197
       
197
       
+
       
      server_name links.bsky.bad-example.com;

     

       
198
       
198
       
+
       


     

       
199
       
199
       
+
       
      proxy_cache cozy_zone;

     

       
200
       
200
       
+
       
      proxy_cache_background_update on;

     

       
201
       
201
       
+
       
      proxy_cache_key "$scheme$proxy_host$uri$is_args$args$http_accept";

     

       
202
       
202
       
+
       
      proxy_cache_lock on; # make simlutaneous requests for the same uri wait for it to appear in cache instead of hitting origin

     

       
203
       
203
       
+
       
      proxy_cache_lock_age 1s;

     

       
204
       
204
       
+
       
      proxy_cache_lock_timeout 2s;

     

       
205
       
205
       
+
       
      proxy_cache_valid 10s; # default -- should be explicitly set in the response headers

     

       
206
       
206
       
+
       
      proxy_cache_valid any 15s; # non-200s default

     

       
207
       
207
       
+
       
      proxy_read_timeout 5s;

     

       
208
       
208
       
+
       
      proxy_send_timeout 15s;

     

       
209
       
209
       
+
       
      proxy_socket_keepalive on;

     

       
210
       
210
       
+
       


     

       
211
       
211
       
+
       
      limit_req zone=cozy_ip_limit nodelay burst=100;

     

       
212
       
212
       
+
       
      limit_req zone=cozy_global_limit;

     

       
213
       
213
       
+
       
      limit_req_status 429;

     

       
214
       
214
       
+
       


     

       
215
       
215
       
+
       
      location / {

     

       
216
       
216
       
+
       
        proxy_pass http://cozy_link_aggregator;

     

       
217
       
217
       
+
       
        include proxy_params;

     

       
218
       
218
       
+
       
        proxy_http_version 1.1;

     

       
219
       
219
       
+
       
        proxy_set_header Connection ""; # for keepalive

     

       
220
       
220
       
+
       
      }

     

       
221
       
221
       
+
       
    }

     

       
222
       
222
       
+
       
    ```

     

       
223
       
223
       
+
       


     

       
170
       
224
       
 
       
---

     

       
171
       
225
       
 
       


     

       
172
       
226
       
 
       
some todos

     
···

       
179
       
233
       
 
       
  - [x] configure systemd to keep it alive

     

       
180
       
234
       
 
       
- [x] configure caddy cache

     

       
181
       
235
       
 
       
- [x] configure caddy rate-limit

     

       
182
       
182
       
-
       
- [ ] configure caddy to use a health check (once it's added)

     

       
183
       
183
       
-
       
- [ ] configure caddy to only expose cache metrics to tailnet :/

     

       
236
       
236
       
+
       
- [ ] configure ~caddy~ nginx to use a health check (once it's added)

     

       
237
       
237
       
+
       
- [ ] ~configure caddy to only expose cache metrics to tailnet :/~

     

       
184
       
238
       
 
       
- [x] make some grafana dashboards

     

       
185
       
239
       
 
       
- [ ] raspi: mount /dev/sda on boot

     

       
186
       
240
       
 
       
- [ ] raspi: run link_aggregator via systemd so it starts on startup (and restarts?)

     

       
187
       
241
       
 
       


     

       
242
       
242
       
+
       
- [x] use nginx instead of caddy

     

       
243
       
243
       
+
       
- [x] nginx: enable cache

     

       
244
       
244
       
+
       
- [x] nginx: rate-limit

     

       
245
       
245
       
+
       
- [ ] nginx: get metrics

     

···

       
118
       
118
       
 
       
- [ ] jetstream: connect retry: only reset counter after some *time* has passed.

     

       
119
       
119
       
 
       
- [x] either count or estimate the total number of links added (distinct from link targets)

     

       
120
       
120
       
 
       


     

       
121
       
121
       
+
       
cache

     

       
122
       
122
       
+
       
- [ ] set api response headers

     

       
123
       
123
       
+
       
  - [ ] put "stale-while-revalidate" in Cache-Control w/ num seconds

     

       
124
       
124
       
+
       
  - [ ] put "stale-if-error" in Cache-Control w/ num seconds

     

       
125
       
125
       
+
       
  - [ ] set Expires or Cache-Control expires

     

       
126
       
126
       
+
       
  - [ ] add Accept to vary response

     

       
127
       
127
       
+
       
- [ ] cache vary: might need to take bsky account privacy setting into account (unless this ends up being in query)

     

       
128
       
128
       
+
       


     

       
121
       
129
       
 
       
data fixes

     

       
122
       
130
       
 
       
- [x] add rkey to linkers 🤦‍♀️

     

       
123
       
131
       
 
       
- [x] don't remove deleted links from the reverse records -- null them out. this will keep things stable for paging.