comparing 90da2f14db4d26ef142cdde4b51a868856b7ea7e and migrate on ptr.pet/nsid-tracker

.gitignore

···

       29
       29
        
       

     

       30
       30
        
       result

     

       31
       31
        
       server/bsky_zstd_dictionary

     

       32
       32
       +
       fjall_data*

     

       33
       33
       +
       .fjall_data*

+13 -1

client/src/lib/api.ts

···

       1
       1
        
       import { dev } from "$app/environment";

     

       2
       2
       -
       import type { Events } from "./types";

     

       2
       2
       +
       import type { Events, Since } from "./types";

     

       3
       3
        
       import { PUBLIC_API_URL } from "$env/static/public";

     

       4
       4
        
       

     

       5
       5
        
       export const fetchEvents = async (): Promise<Events> => {

     
···

       13
       13
        
         const data = await response.json();

     

       14
       14
        
         return data;

     

       15
       15
        
       };

     

       16
       16
       +
       

     

       17
       17
       +
       export const fetchTrackingSince = async (): Promise<Since> => {

     

       18
       18
       +
         const response = await fetch(

     

       19
       19
       +
           `${dev ? "http" : "https"}://${PUBLIC_API_URL}/since`,

     

       20
       20
       +
         );

     

       21
       21
       +
         if (!response.ok) {

     

       22
       22
       +
           throw new Error(`(${response.status}): ${await response.json()}`);

     

       23
       23
       +
         }

     

       24
       24
       +
       

     

       25
       25
       +
         const data = await response.json();

     

       26
       26
       +
         return data;

     

       27
       27
       +
       };

+2 -4

client/src/lib/components/StatsCard.svelte

···

       40
       40
        
           const colors = $derived(colorClasses[colorScheme]);

     

       41
       41
        
       </script>

     

       42
       42
        
       

     

       43
       43
       -
       <div

     

       44
       44
       -
           class="bg-gradient-to-r {colors.bg} p-3 md:p-6 rounded-lg border {colors.border}"

     

       45
       45
       -
       >

     

       43
       43
       +
       <div class="bg-gradient-to-r {colors.bg} p-3 rounded-lg border {colors.border}">

     

       46
       44
        
           <h3 class="text-base font-medium {colors.titleText} mb-2">

     

       47
       45
        
               {title}

     

       48
       46
        
           </h3>

     

       49
       49
       -
           <p class="text-xl md:text-3xl font-bold {colors.valueText}">

     

       47
       47
       +
           <p class="text-xl md:text-2xl font-bold {colors.valueText}">

     

       50
       48
        
               <NumberFlow {value} />

     

       51
       49
        
           </p>

     

       52
       50
        
       </div>

client/src/lib/types.ts

···

       13
       13
        
         count: number;

     

       14
       14
        
         deleted_count: number;

     

       15
       15
        
       };

     

       16
       16
       +
       export type Since = {

     

       17
       17
       +
         since: number;

     

       18
       18
       +
       };

     

       16
       19
        
       

     

       17
       20
        
       export type SortOption = "total" | "created" | "deleted" | "date";

+20 -26

client/src/routes/+page.svelte

···

       4
       4
        
           import { onMount, onDestroy } from "svelte";

     

       5
       5
        
           import { writable } from "svelte/store";

     

       6
       6
        
           import { PUBLIC_API_URL } from "$env/static/public";

     

       7
       7
       -
           import { fetchEvents } from "$lib/api";

     

       7
       7
       +
           import { fetchEvents, fetchTrackingSince } from "$lib/api";

     

       8
       8
        
           import { createRegexFilter } from "$lib/filter";

     

       9
       9
        
           import StatsCard from "$lib/components/StatsCard.svelte";

     

       10
       10
        
           import StatusBadge from "$lib/components/StatusBadge.svelte";

     
···

       13
       13
        
           import SortControls from "$lib/components/SortControls.svelte";

     

       14
       14
        
           import BskyToggle from "$lib/components/BskyToggle.svelte";

     

       15
       15
        
           import RefreshControl from "$lib/components/RefreshControl.svelte";

     

       16
       16
       +
           import { formatTimestamp } from "$lib/format";

     

       16
       17
        
       

     

       17
       18
        
           const events = writable(new Map<string, EventRecord>());

     

       18
       19
        
           const pendingUpdates = new Map<string, EventRecord>();

     
···

       28
       29
        
                   .toArray();

     

       29
       30
        
           });

     

       30
       31
        
           let per_second = $state(0);

     

       32
       32
       +
           let tracking_since = $state(0);

     

       31
       33
        
       

     

       32
       34
        
           let all: EventRecord = $derived(

     

       33
       35
        
               eventsList.reduce(

     
···

       53
       55
        
           let dontShowBsky = $state(false);

     

       54
       56
        
           let sortBy: SortOption = $state("total");

     

       55
       57
        
           let refreshRate = $state("");

     

       56
       56
       -
           let previousRefreshRate = $state("");

     

       58
       58
       +
           let changedByUser = $state(false);

     

       57
       59
        
       

     

       58
       60
        
           let websocket: WebSocket | null = null;

     

       59
       61
        
           let isStreamOpen = $state(false);

     
···

       118
       120
        
                       }

     

       119
       121
        
                       return map;

     

       120
       122
        
                   });

     

       123
       123
       +
                   tracking_since = (await fetchTrackingSince()).since;

     

       121
       124
        
               } catch (err) {

     

       122
       125
        
                   error =

     

       123
       126
        
                       err instanceof Error

     
···

       127
       130
        
               }

     

       128
       131
        
           };

     

       129
       132
        
       

     

       130
       130
       -
           // Set refresh rate when sort mode changes

     

       131
       131
       -
           $effect(() => {

     

       132
       132
       -
               if (sortBy === "date" && refreshRate === "") {

     

       133
       133
       -
                   // Only set to 2 if we're in real-time mode

     

       134
       134
       -
                   refreshRate = "2";

     

       135
       135
       -
               } else if (

     

       136
       136
       -
                   sortBy !== "date" &&

     

       137
       137
       -
                   refreshRate === "2" &&

     

       138
       138
       -
                   previousRefreshRate === "2"

     

       139
       139
       -
               ) {

     

       140
       140
       -
                   // Return to real-time mode if we auto-set the refresh rate

     

       141
       141
       -
                   refreshRate = "";

     

       142
       142
       -
               }

     

       143
       143
       -
           });

     

       144
       144
       -
       

     

       145
       133
        
           // Update the refresh timer when refresh rate changes

     

       146
       134
        
           $effect(() => {

     

       147
       135
        
               if (updateTimer) {

     
···

       240
       228
        
           >

     

       241
       229
        
               <h1 class="text-4xl font-bold mr-4 text-gray-900">lexicon tracker</h1>

     

       242
       230
        
               <p class="text-lg mt-1 text-gray-600">

     

       243
       243
       -
                   tracks lexicons seen on the jetstream

     

       231
       231
       +
                   tracks lexicons seen on the jetstream {tracking_since === 0

     

       232
       232
       +
                       ? ""

     

       233
       233
       +
                       : `(since: ${formatTimestamp(tracking_since)})`}

     

       244
       234
        
               </p>

     

       245
       235
        
           </div>

     

       246
       236
        
       </header>

     

       247
       237
        
       <div class="md:max-w-[61vw] mx-auto p-2">

     

       248
       248
       -
           <div

     

       249
       249
       -
               class="min-w-fit grid grid-cols-2 xl:grid-cols-4 gap-2 2xl:gap-6 2xl:mx-16 mb-8"

     

       250
       250
       -
           >

     

       238
       238
       +
           <div class="min-w-fit grid grid-cols-2 xl:grid-cols-4 gap-2 2xl:gap-6 mb-8">

     

       251
       239
        
               <StatsCard

     

       252
       240
        
                   title="total creation"

     

       253
       241
        
                   value={all.count}

     
···

       289
       277
        
                           {filterRegex}

     

       290
       278
        
                           onFilterChange={(value) => (filterRegex = value)}

     

       291
       279
        
                       />

     

       292
       292
       -
                       <SortControls

     

       293
       293
       -
                           {sortBy}

     

       294
       294
       -
                           onSortChange={(value: SortOption) => (sortBy = value)}

     

       295
       295
       -
                       />

     

       296
       280
        
                       <BskyToggle

     

       297
       281
        
                           {dontShowBsky}

     

       298
       282
        
                           onBskyToggle={() => (dontShowBsky = !dontShowBsky)}

     

       299
       283
        
                       />

     

       284
       284
       +
                       <SortControls

     

       285
       285
       +
                           {sortBy}

     

       286
       286
       +
                           onSortChange={(value: SortOption) => {

     

       287
       287
       +
                               sortBy = value;

     

       288
       288
       +
                               if (refreshRate === "" && sortBy === "date")

     

       289
       289
       +
                                   refreshRate = "2";

     

       290
       290
       +
                               else if (refreshRate === "2" && changedByUser === false)

     

       291
       291
       +
                                   refreshRate = "";

     

       292
       292
       +
                           }}

     

       293
       293
       +
                       />

     

       300
       294
        
                       <RefreshControl

     

       301
       295
        
                           {refreshRate}

     

       302
       296
        
                           onRefreshChange={(value) => {

     

       303
       303
       -
                               previousRefreshRate = refreshRate;

     

       304
       297
        
                               refreshRate = value;

     

       298
       298
       +
                               changedByUser = refreshRate !== "";

     

       305
       299
        
                           }}

     

       306
       300
        
                       />

     

       307
       301
        
                   </div>

+5 -2

nix/server.nix

···

       1
       1
        
       {

     

       2
       2
       -
       rustPlatform,

     

       3
       3
       -
       ...

     

       2
       2
       +
         rustPlatform,

     

       3
       3
       +
         cmake,

     

       4
       4
       +
         ...

     

       4
       5
        
       }:

     

       5
       6
        
       rustPlatform.buildRustPackage {

     

       6
       7
        
         pname = "nsid-tracker-server";

     

       7
       8
        
         version = "main";

     

       8
       9
        
       

     

       9
       10
        
         src = ../server;

     

       11
       11
       +
       

     

       12
       12
       +
         nativeBuildInputs = [ cmake ];

     

       10
       13
        
       

     

       11
       14
        
         cargoLock = {

     

       12
       15
        
           lockFile = ../server/Cargo.lock;

+1 -1

server/.gitignore

···

       1
       1
        
       target

     

       2
       2
       -
       .fjall_data

     

       2
       2
       +
       .fjall_data*

+195 -64

server/Cargo.lock

···

       18
       18
        
       checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"

     

       19
       19
        
       

     

       20
       20
        
       [[package]]

     

       21
       21
       -
       name = "ahash"

     

       22
       22
       -
       version = "0.8.12"

     

       23
       23
       -
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       24
       24
       -
       checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"

     

       25
       25
       -
       dependencies = [

     

       26
       26
       -
        "cfg-if",

     

       27
       27
       -
        "getrandom 0.3.3",

     

       28
       28
       -
        "once_cell",

     

       29
       29
       -
        "version_check",

     

       30
       30
       -
        "zerocopy",

     

       31
       31
       -
       ]

     

       32
       32
       -
       

     

       33
       33
       -
       [[package]]

     

       34
       21
        
       name = "aho-corasick"

     

       35
       22
        
       version = "1.1.3"

     

       36
       23
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       216
       203
        
       ]

     

       217
       204
        
       

     

       218
       205
        
       [[package]]

     

       206
       206
       +
       name = "branches"

     

       207
       207
       +
       version = "0.2.2"

     

       208
       208
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       209
       209
       +
       checksum = "a918aa7a861caeba57e502465c30e3a0d74ae02ee0b9db2933602fdb6a3a90e5"

     

       210
       210
       +
       dependencies = [

     

       211
       211
       +
        "rustc_version",

     

       212
       212
       +
       ]

     

       213
       213
       +
       

     

       214
       214
       +
       [[package]]

     

       219
       215
        
       name = "brotli"

     

       220
       216
        
       version = "8.0.1"

     

       221
       217
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       313
       309
        
       checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"

     

       314
       310
        
       

     

       315
       311
        
       [[package]]

     

       312
       312
       +
       name = "cmake"

     

       313
       313
       +
       version = "0.1.54"

     

       314
       314
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       315
       315
       +
       checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"

     

       316
       316
       +
       dependencies = [

     

       317
       317
       +
        "cc",

     

       318
       318
       +
       ]

     

       319
       319
       +
       

     

       320
       320
       +
       [[package]]

     

       316
       321
        
       name = "combine"

     

       317
       322
        
       version = "4.6.7"

     

       318
       323
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       363
       368
        
       ]

     

       364
       369
        
       

     

       365
       370
        
       [[package]]

     

       371
       371
       +
       name = "crossbeam-deque"

     

       372
       372
       +
       version = "0.8.6"

     

       373
       373
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       374
       374
       +
       checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"

     

       375
       375
       +
       dependencies = [

     

       376
       376
       +
        "crossbeam-epoch",

     

       377
       377
       +
        "crossbeam-utils",

     

       378
       378
       +
       ]

     

       379
       379
       +
       

     

       380
       380
       +
       [[package]]

     

       366
       381
        
       name = "crossbeam-epoch"

     

       367
       382
        
       version = "0.9.18"

     

       368
       383
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       426
       441
        
       version = "0.1.0"

     

       427
       442
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       428
       443
        
       checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"

     

       444
       444
       +
       

     

       445
       445
       +
       [[package]]

     

       446
       446
       +
       name = "either"

     

       447
       447
       +
       version = "1.15.0"

     

       448
       448
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       449
       449
       +
       checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"

     

       429
       450
        
       

     

       430
       451
        
       [[package]]

     

       431
       452
        
       name = "enum_dispatch"

     
···

       633
       654
        
       checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"

     

       634
       655
        
       

     

       635
       656
        
       [[package]]

     

       657
       657
       +
       name = "hermit-abi"

     

       658
       658
       +
       version = "0.5.2"

     

       659
       659
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       660
       660
       +
       checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"

     

       661
       661
       +
       

     

       662
       662
       +
       [[package]]

     

       636
       663
        
       name = "http"

     

       637
       664
        
       version = "1.3.1"

     

       638
       665
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       742
       769
        
        "bitflags",

     

       743
       770
        
        "cfg-if",

     

       744
       771
        
        "libc",

     

       772
       772
       +
       ]

     

       773
       773
       +
       

     

       774
       774
       +
       [[package]]

     

       775
       775
       +
       name = "itertools"

     

       776
       776
       +
       version = "0.14.0"

     

       777
       777
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       778
       778
       +
       checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"

     

       779
       779
       +
       dependencies = [

     

       780
       780
       +
        "either",

     

       745
       781
        
       ]

     

       746
       782
        
       

     

       747
       783
        
       [[package]]

     
···

       935
       971
        
       ]

     

       936
       972
        
       

     

       937
       973
        
       [[package]]

     

       974
       974
       +
       name = "num_cpus"

     

       975
       975
       +
       version = "1.17.0"

     

       976
       976
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       977
       977
       +
       checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"

     

       978
       978
       +
       dependencies = [

     

       979
       979
       +
        "hermit-abi",

     

       980
       980
       +
        "libc",

     

       981
       981
       +
       ]

     

       982
       982
       +
       

     

       983
       983
       +
       [[package]]

     

       938
       984
        
       name = "object"

     

       939
       985
        
       version = "0.36.7"

     

       940
       986
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       956
       1002
        
       checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"

     

       957
       1003
        
       

     

       958
       1004
        
       [[package]]

     

       1005
       1005
       +
       name = "ordered-varint"

     

       1006
       1006
       +
       version = "2.0.0"

     

       1007
       1007
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1008
       1008
       +
       checksum = "e9cc9f18ab4bad1e01726bda1259feb8f11e5e76308708a966b4c0136e9db34c"

     

       1009
       1009
       +
       

     

       1010
       1010
       +
       [[package]]

     

       959
       1011
        
       name = "overload"

     

       960
       1012
        
       version = "0.1.1"

     

       961
       1013
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       962
       1014
        
       checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"

     

       963
       1015
        
       

     

       964
       1016
        
       [[package]]

     

       965
       965
       -
       name = "papaya"

     

       966
       966
       -
       version = "0.2.3"

     

       967
       967
       -
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       968
       968
       -
       checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f"

     

       969
       969
       -
       dependencies = [

     

       970
       970
       -
        "equivalent",

     

       971
       971
       -
        "seize",

     

       972
       972
       -
       ]

     

       973
       973
       -
       

     

       974
       974
       -
       [[package]]

     

       975
       1017
        
       name = "parking_lot"

     

       976
       1018
        
       version = "0.12.4"

     

       977
       1019
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1031
       1073
        
       checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"

     

       1032
       1074
        
       

     

       1033
       1075
        
       [[package]]

     

       1034
       1034
       -
       name = "pingora-limits"

     

       1035
       1035
       -
       version = "0.5.0"

     

       1036
       1036
       -
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1037
       1037
       -
       checksum = "a719a8cb5558ca06bd6076c97b8905d500ea556da89e132ba53d4272844f95b9"

     

       1038
       1038
       -
       dependencies = [

     

       1039
       1039
       -
        "ahash",

     

       1040
       1040
       -
       ]

     

       1041
       1041
       -
       

     

       1042
       1042
       -
       [[package]]

     

       1043
       1076
        
       name = "pkg-config"

     

       1044
       1077
        
       version = "0.3.32"

     

       1045
       1078
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1075
       1108
        
       ]

     

       1076
       1109
        
       

     

       1077
       1110
        
       [[package]]

     

       1111
       1111
       +
       name = "quanta"

     

       1112
       1112
       +
       version = "0.12.6"

     

       1113
       1113
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1114
       1114
       +
       checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"

     

       1115
       1115
       +
       dependencies = [

     

       1116
       1116
       +
        "crossbeam-utils",

     

       1117
       1117
       +
        "libc",

     

       1118
       1118
       +
        "once_cell",

     

       1119
       1119
       +
        "raw-cpuid",

     

       1120
       1120
       +
        "wasi 0.11.1+wasi-snapshot-preview1",

     

       1121
       1121
       +
        "web-sys",

     

       1122
       1122
       +
        "winapi",

     

       1123
       1123
       +
       ]

     

       1124
       1124
       +
       

     

       1125
       1125
       +
       [[package]]

     

       1078
       1126
        
       name = "quick_cache"

     

       1079
       1127
        
       version = "0.6.15"

     

       1080
       1128
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1109
       1157
        
       ]

     

       1110
       1158
        
       

     

       1111
       1159
        
       [[package]]

     

       1160
       1160
       +
       name = "raw-cpuid"

     

       1161
       1161
       +
       version = "11.5.0"

     

       1162
       1162
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1163
       1163
       +
       checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146"

     

       1164
       1164
       +
       dependencies = [

     

       1165
       1165
       +
        "bitflags",

     

       1166
       1166
       +
       ]

     

       1167
       1167
       +
       

     

       1168
       1168
       +
       [[package]]

     

       1169
       1169
       +
       name = "rayon"

     

       1170
       1170
       +
       version = "1.10.0"

     

       1171
       1171
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1172
       1172
       +
       checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"

     

       1173
       1173
       +
       dependencies = [

     

       1174
       1174
       +
        "either",

     

       1175
       1175
       +
        "rayon-core",

     

       1176
       1176
       +
       ]

     

       1177
       1177
       +
       

     

       1178
       1178
       +
       [[package]]

     

       1179
       1179
       +
       name = "rayon-core"

     

       1180
       1180
       +
       version = "1.12.1"

     

       1181
       1181
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1182
       1182
       +
       checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"

     

       1183
       1183
       +
       dependencies = [

     

       1184
       1184
       +
        "crossbeam-deque",

     

       1185
       1185
       +
        "crossbeam-utils",

     

       1186
       1186
       +
       ]

     

       1187
       1187
       +
       

     

       1188
       1188
       +
       [[package]]

     

       1189
       1189
       +
       name = "rclite"

     

       1190
       1190
       +
       version = "0.2.7"

     

       1191
       1191
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1192
       1192
       +
       checksum = "2f528dfeba924f5fc67bb84a17fe043451d1b392758016ce2d9e9116649b0f35"

     

       1193
       1193
       +
       dependencies = [

     

       1194
       1194
       +
        "branches",

     

       1195
       1195
       +
       ]

     

       1196
       1196
       +
       

     

       1197
       1197
       +
       [[package]]

     

       1112
       1198
        
       name = "redox_syscall"

     

       1113
       1199
        
       version = "0.5.15"

     

       1114
       1200
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1227
       1313
        
       checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"

     

       1228
       1314
        
       

     

       1229
       1315
        
       [[package]]

     

       1316
       1316
       +
       name = "rustc_version"

     

       1317
       1317
       +
       version = "0.4.1"

     

       1318
       1318
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1319
       1319
       +
       checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"

     

       1320
       1320
       +
       dependencies = [

     

       1321
       1321
       +
        "semver",

     

       1322
       1322
       +
       ]

     

       1323
       1323
       +
       

     

       1324
       1324
       +
       [[package]]

     

       1230
       1325
        
       name = "rustix"

     

       1231
       1326
        
       version = "1.0.8"

     

       1232
       1327
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1335
       1430
        
       ]

     

       1336
       1431
        
       

     

       1337
       1432
        
       [[package]]

     

       1433
       1433
       +
       name = "scc"

     

       1434
       1434
       +
       version = "2.3.4"

     

       1435
       1435
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1436
       1436
       +
       checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4"

     

       1437
       1437
       +
       dependencies = [

     

       1438
       1438
       +
        "sdd",

     

       1439
       1439
       +
       ]

     

       1440
       1440
       +
       

     

       1441
       1441
       +
       [[package]]

     

       1338
       1442
        
       name = "schannel"

     

       1339
       1443
        
       version = "0.1.27"

     

       1340
       1444
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1348
       1452
        
       version = "1.2.0"

     

       1349
       1453
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1350
       1454
        
       checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

     

       1455
       1455
       +
       

     

       1456
       1456
       +
       [[package]]

     

       1457
       1457
       +
       name = "sdd"

     

       1458
       1458
       +
       version = "3.0.10"

     

       1459
       1459
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1460
       1460
       +
       checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca"

     

       1351
       1461
        
       

     

       1352
       1462
        
       [[package]]

     

       1353
       1463
        
       name = "security-framework"

     
···

       1373
       1483
        
       ]

     

       1374
       1484
        
       

     

       1375
       1485
        
       [[package]]

     

       1376
       1376
       -
       name = "seize"

     

       1377
       1377
       -
       version = "0.5.0"

     

       1378
       1378
       -
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1379
       1379
       -
       checksum = "e4b8d813387d566f627f3ea1b914c068aac94c40ae27ec43f5f33bde65abefe7"

     

       1380
       1380
       -
       dependencies = [

     

       1381
       1381
       -
        "libc",

     

       1382
       1382
       -
        "windows-sys 0.52.0",

     

       1383
       1383
       -
       ]

     

       1384
       1384
       -
       

     

       1385
       1385
       -
       [[package]]

     

       1386
       1486
        
       name = "self_cell"

     

       1387
       1487
        
       version = "1.2.0"

     

       1388
       1488
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1389
       1489
        
       checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"

     

       1490
       1490
       +
       

     

       1491
       1491
       +
       [[package]]

     

       1492
       1492
       +
       name = "semver"

     

       1493
       1493
       +
       version = "1.0.26"

     

       1494
       1494
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1495
       1495
       +
       checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"

     

       1390
       1496
        
       

     

       1391
       1497
        
       [[package]]

     

       1392
       1498
        
       name = "serde"

     
···

       1450
       1556
        
        "async-trait",

     

       1451
       1557
        
        "axum",

     

       1452
       1558
        
        "axum-tws",

     

       1559
       1559
       +
        "byteview",

     

       1453
       1560
        
        "fjall",

     

       1454
       1561
        
        "futures-util",

     

       1455
       1455
       -
        "papaya",

     

       1456
       1456
       -
        "pingora-limits",

     

       1562
       1562
       +
        "itertools",

     

       1563
       1563
       +
        "ordered-varint",

     

       1564
       1564
       +
        "parking_lot",

     

       1565
       1565
       +
        "quanta",

     

       1566
       1566
       +
        "rayon",

     

       1567
       1567
       +
        "rclite",

     

       1457
       1568
        
        "rkyv",

     

       1458
       1569
        
        "rustls",

     

       1570
       1570
       +
        "scc",

     

       1459
       1571
        
        "serde",

     

       1460
       1572
        
        "serde_json",

     

       1461
       1573
        
        "smol_str",

     

       1574
       1574
       +
        "snmalloc-rs",

     

       1575
       1575
       +
        "threadpool",

     

       1462
       1576
        
        "tikv-jemallocator",

     

       1463
       1577
        
        "tokio",

     

       1464
       1578
        
        "tokio-util",

     
···

       1532
       1646
        
       ]

     

       1533
       1647
        
       

     

       1534
       1648
        
       [[package]]

     

       1649
       1649
       +
       name = "snmalloc-rs"

     

       1650
       1650
       +
       version = "0.3.8"

     

       1651
       1651
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1652
       1652
       +
       checksum = "eb317153089fdfa4d8a2eec059d40a5a23c3bde43995ea23b19121c3f621e74a"

     

       1653
       1653
       +
       dependencies = [

     

       1654
       1654
       +
        "snmalloc-sys",

     

       1655
       1655
       +
       ]

     

       1656
       1656
       +
       

     

       1657
       1657
       +
       [[package]]

     

       1658
       1658
       +
       name = "snmalloc-sys"

     

       1659
       1659
       +
       version = "0.3.8"

     

       1660
       1660
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1661
       1661
       +
       checksum = "065fea53d32bb77bc36cca466cb191f2e5216ebfd0ed360b1d64889ee6e559ea"

     

       1662
       1662
       +
       dependencies = [

     

       1663
       1663
       +
        "cmake",

     

       1664
       1664
       +
       ]

     

       1665
       1665
       +
       

     

       1666
       1666
       +
       [[package]]

     

       1535
       1667
        
       name = "socket2"

     

       1536
       1668
        
       version = "0.5.10"

     

       1537
       1669
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1613
       1745
        
       ]

     

       1614
       1746
        
       

     

       1615
       1747
        
       [[package]]

     

       1748
       1748
       +
       name = "threadpool"

     

       1749
       1749
       +
       version = "1.8.1"

     

       1750
       1750
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1751
       1751
       +
       checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"

     

       1752
       1752
       +
       dependencies = [

     

       1753
       1753
       +
        "num_cpus",

     

       1754
       1754
       +
       ]

     

       1755
       1755
       +
       

     

       1756
       1756
       +
       [[package]]

     

       1616
       1757
        
       name = "tikv-jemalloc-sys"

     

       1617
       1758
        
       version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"

     

       1618
       1759
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1981
       2122
        
       ]

     

       1982
       2123
        
       

     

       1983
       2124
        
       [[package]]

     

       2125
       2125
       +
       name = "web-sys"

     

       2126
       2126
       +
       version = "0.3.77"

     

       2127
       2127
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       2128
       2128
       +
       checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"

     

       2129
       2129
       +
       dependencies = [

     

       2130
       2130
       +
        "js-sys",

     

       2131
       2131
       +
        "wasm-bindgen",

     

       2132
       2132
       +
       ]

     

       2133
       2133
       +
       

     

       2134
       2134
       +
       [[package]]

     

       1984
       2135
        
       name = "webpki-root-certs"

     

       1985
       2136
        
       version = "1.0.1"

     

       1986
       2137
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       2255
       2406
        
       version = "0.8.15"

     

       2256
       2407
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       2257
       2408
        
       checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"

     

       2258
       2258
       -
       

     

       2259
       2259
       -
       [[package]]

     

       2260
       2260
       -
       name = "zerocopy"

     

       2261
       2261
       -
       version = "0.8.26"

     

       2262
       2262
       -
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       2263
       2263
       -
       checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"

     

       2264
       2264
       -
       dependencies = [

     

       2265
       2265
       -
        "zerocopy-derive",

     

       2266
       2266
       -
       ]

     

       2267
       2267
       -
       

     

       2268
       2268
       -
       [[package]]

     

       2269
       2269
       -
       name = "zerocopy-derive"

     

       2270
       2270
       -
       version = "0.8.26"

     

       2271
       2271
       -
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       2272
       2272
       -
       checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"

     

       2273
       2273
       -
       dependencies = [

     

       2274
       2274
       -
        "proc-macro2",

     

       2275
       2275
       -
        "quote",

     

       2276
       2276
       -
        "syn",

     

       2277
       2277
       -
       ]

     

       2278
       2409
        
       

     

       2279
       2410
        
       [[package]]

     

       2280
       2411
        
       name = "zeroize"

+13 -3

server/Cargo.toml

···

       8
       8
        
       async-trait = "0.1"

     

       9
       9
        
       tracing-subscriber = {version = "0.3", features = ["env-filter"]}

     

       10
       10
        
       tracing = "0.1"

     

       11
       11
       -
       tokio = { version = "1", features = ["full"] }

     

       11
       11
       +
       tokio = { version = "1", features = ["full", "parking_lot"] }

     

       12
       12
        
       tokio-util = { version = "0.7", features = ["tracing"] }

     

       13
       13
        
       rustls = { version = "0.23", default-features = false, features = ["log", "ring", "std"] }

     

       14
       14
        
       tokio-websockets = { version = "0.12", features = ["client", "rustls-platform-verifier", "getrandom", "ring"] }

     

       15
       15
        
       futures-util = "0.3"

     

       16
       16
        
       axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "tracing", "json", "query"] }

     

       17
       17
        
       axum-tws = { git = "https://github.com/90-008/axum-tws.git", features = ["http2"] }

     

       18
       18
       -
       pingora-limits = "0.5"

     

       19
       18
        
       tower-http = {version = "0.6", features = ["request-id", "trace", "compression-full"]}

     

       20
       19
        
       fjall = { version = "2", default-features = false, features = ["miniz", "lz4"] }

     

       21
       20
        
       rkyv = {version = "0.8", features = ["unaligned"]}

     

       22
       21
        
       smol_str = { version = "0.3", features = ["serde"] }

     

       23
       23
       -
       papaya = "0.2"

     

       24
       22
        
       serde = { version = "1", features = ["derive"] }

     

       25
       23
        
       serde_json = "1.0.141"

     

       24
       24
       +
       scc = "2.3.4"

     

       25
       25
       +
       ordered-varint = "2.0.0"

     

       26
       26
       +
       threadpool = "1.8.1"

     

       27
       27
       +
       quanta = "0.12.6"

     

       28
       28
       +
       itertools = "0.14.0"

     

       29
       29
       +
       byteview = "0.6.1"

     

       30
       30
       +
       rayon = "1.10.0"

     

       31
       31
       +
       parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] }

     

       32
       32
       +
       rclite = "0.2.7"

     

       33
       33
       +
       

     

       34
       34
       +
       [target.'cfg(target_env = "msvc")'.dependencies]

     

       35
       35
       +
       snmalloc-rs = "0.3.8"

     

       26
       36
        
       

     

       27
       37
        
       [target.'cfg(not(target_env = "msvc"))'.dependencies]

     

       28
       38
        
       tikv-jemallocator = "0.6"

+40 -17

server/src/api.rs

···

       2
       2
        
           collections::HashMap,

     

       3
       3
        
           fmt::Display,

     

       4
       4
        
           net::SocketAddr,

     

       5
       5
       -
           ops::Deref,

     

       6
       6
       -
           sync::Arc,

     

       7
       7
       -
           time::{Duration, UNIX_EPOCH},

     

       5
       5
       +
           ops::{Bound, Deref, RangeBounds},

     

       6
       6
       +
           time::Duration,

     

       8
       7
        
       };

     

       9
       8
        
       

     

       10
       9
        
       use anyhow::anyhow;

     
···

       16
       15
        
           routing::get,

     

       17
       16
        
       };

     

       18
       17
        
       use axum_tws::{Message, WebSocketUpgrade};

     

       18
       18
       +
       use rclite::Arc;

     

       19
       19
        
       use serde::{Deserialize, Serialize};

     

       20
       20
        
       use smol_str::SmolStr;

     

       21
       21
        
       use tokio_util::sync::CancellationToken;

     
···

       51
       51
        
               .route("/events", get(events))

     

       52
       52
        
               .route("/stream_events", get(stream_events))

     

       53
       53
        
               .route("/hits", get(hits))

     

       54
       54
       +
               .route("/since", get(since))

     

       54
       55
        
               .route_layer(CompressionLayer::new().br(true).deflate(true).gzip(true).zstd(true))

     

       55
       56
        
               .route_layer(PropagateRequestIdLayer::x_request_id())

     

       56
       57
        
               .route_layer(

     
···

       145
       146
        
           to: Option<u64>,

     

       146
       147
        
       }

     

       147
       148
        
       

     

       148
       148
       -
       #[derive(Serialize)]

     

       149
       149
       +
       #[derive(Debug, Serialize)]

     

       149
       150
        
       struct Hit {

     

       150
       151
        
           timestamp: u64,

     

       151
       152
        
           deleted: bool,

     
···

       153
       154
        
       

     

       154
       155
        
       const MAX_HITS: usize = 100_000;

     

       155
       156
        
       

     

       157
       157
       +
       #[derive(Debug)]

     

       158
       158
       +
       struct HitsRange {

     

       159
       159
       +
           from: Bound<u64>,

     

       160
       160
       +
           to: Bound<u64>,

     

       161
       161
       +
       }

     

       162
       162
       +
       

     

       163
       163
       +
       impl RangeBounds<u64> for HitsRange {

     

       164
       164
       +
           fn start_bound(&self) -> Bound<&u64> {

     

       165
       165
       +
               self.from.as_ref()

     

       166
       166
       +
           }

     

       167
       167
       +
       

     

       168
       168
       +
           fn end_bound(&self) -> Bound<&u64> {

     

       169
       169
       +
               self.to.as_ref()

     

       170
       170
       +
           }

     

       171
       171
       +
       }

     

       172
       172
       +
       

     

       156
       173
        
       async fn hits(

     

       157
       174
        
           State(db): State<Arc<Db>>,

     

       158
       175
        
           Query(params): Query<HitsQuery>,

     

       159
       176
        
       ) -> AppResult<Json<Vec<Hit>>> {

     

       177
       177
       +
           let from = params.to.map(Bound::Included).unwrap_or(Bound::Unbounded);

     

       178
       178
       +
           let to = params.from.map(Bound::Included).unwrap_or(Bound::Unbounded);

     

       160
       179
        
           let maybe_hits = db

     

       161
       161
       -
               .get_hits(

     

       162
       162
       -
                   &params.nsid,

     

       163
       163
       -
                   params.to.unwrap_or(0)

     

       164
       164
       -
                       ..params.from.unwrap_or(

     

       165
       165
       -
                           std::time::SystemTime::now()

     

       166
       166
       -
                               .duration_since(UNIX_EPOCH)

     

       167
       167
       -
                               .expect("oops")

     

       168
       168
       -
                               .as_micros() as u64,

     

       169
       169
       -
                       ),

     

       170
       170
       -
               )?

     

       180
       180
       +
               .get_hits(&params.nsid, HitsRange { from, to })

     

       171
       181
        
               .take(MAX_HITS);

     

       172
       182
        
           let mut hits = Vec::with_capacity(maybe_hits.size_hint().0);

     

       173
       183
        
       

     

       174
       184
        
           for maybe_hit in maybe_hits {

     

       175
       175
       -
               let (timestamp, hit) = maybe_hit?;

     

       185
       185
       +
               let hit = maybe_hit?;

     

       186
       186
       +
               let hit_data = hit.deser()?;

     

       187
       187
       +
       

     

       176
       188
        
               hits.push(Hit {

     

       177
       177
       -
                   timestamp,

     

       178
       178
       -
                   deleted: hit.deleted,

     

       189
       189
       +
                   timestamp: hit.timestamp,

     

       190
       190
       +
                   deleted: hit_data.deleted,

     

       179
       191
        
               });

     

       180
       192
        
           }

     

       181
       193
        
       

     
···

       219
       231
        
               .instrument(span)

     

       220
       232
        
           })

     

       221
       233
        
       }

     

       234
       234
       +
       

     

       235
       235
       +
       #[derive(Debug, Serialize)]

     

       236
       236
       +
       struct Since {

     

       237
       237
       +
           since: u64,

     

       238
       238
       +
       }

     

       239
       239
       +
       

     

       240
       240
       +
       async fn since(db: State<Arc<Db>>) -> AppResult<Json<Since>> {

     

       241
       241
       +
           Ok(Json(Since {

     

       242
       242
       +
               since: db.tracking_since()?,

     

       243
       243
       +
           }))

     

       244
       244
       +
       }

+521

server/src/db/block.rs

···

       1
       1
       +
       use std::{

     

       2
       2
       +
           io::{self, Read, Write},

     

       3
       3
       +
           marker::PhantomData,

     

       4
       4
       +
           usize,

     

       5
       5
       +
       };

     

       6
       6
       +
       

     

       7
       7
       +
       use rkyv::{

     

       8
       8
       +
           Archive, Deserialize, Serialize,

     

       9
       9
       +
           api::high::{HighSerializer, HighValidator},

     

       10
       10
       +
           bytecheck::CheckBytes,

     

       11
       11
       +
           de::Pool,

     

       12
       12
       +
           rancor::{self, Strategy},

     

       13
       13
       +
           ser::allocator::ArenaHandle,

     

       14
       14
       +
           util::AlignedVec,

     

       15
       15
       +
       };

     

       16
       16
       +
       

     

       17
       17
       +
       use crate::{

     

       18
       18
       +
           error::{AppError, AppResult},

     

       19
       19
       +
           utils::{ReadVariableExt, WriteVariableExt},

     

       20
       20
       +
       };

     

       21
       21
       +
       

     

       22
       22
       +
       pub struct Item<T> {

     

       23
       23
       +
           pub timestamp: u64,

     

       24
       24
       +
           pub data: AlignedVec,

     

       25
       25
       +
           phantom: PhantomData<T>,

     

       26
       26
       +
       }

     

       27
       27
       +
       

     

       28
       28
       +
       impl<T> Item<T>

     

       29
       29
       +
       where

     

       30
       30
       +
           T: Archive,

     

       31
       31
       +
           T::Archived: for<'a> CheckBytes<HighValidator<'a, rancor::Error>>

     

       32
       32
       +
               + Deserialize<T, Strategy<Pool, rancor::Error>>,

     

       33
       33
       +
       {

     

       34
       34
       +
           pub fn deser(&self) -> AppResult<T> {

     

       35
       35
       +
               rkyv::from_bytes(&self.data).map_err(AppError::from)

     

       36
       36
       +
           }

     

       37
       37
       +
       }

     

       38
       38
       +
       

     

       39
       39
       +
       impl<T: for<'a> Serialize<HighSerializer<AlignedVec, ArenaHandle<'a>, rancor::Error>>> Item<T> {

     

       40
       40
       +
           pub fn new(timestamp: u64, data: &T) -> Self {

     

       41
       41
       +
               Item {

     

       42
       42
       +
                   timestamp,

     

       43
       43
       +
                   data: unsafe { rkyv::to_bytes(data).unwrap_unchecked() },

     

       44
       44
       +
                   phantom: PhantomData,

     

       45
       45
       +
               }

     

       46
       46
       +
           }

     

       47
       47
       +
       }

     

       48
       48
       +
       

     

       49
       49
       +
       pub struct ItemEncoder<W: Write, T> {

     

       50
       50
       +
           writer: W,

     

       51
       51
       +
           prev_timestamp: u64,

     

       52
       52
       +
           prev_delta: i64,

     

       53
       53
       +
           item_count: usize,

     

       54
       54
       +
           _item: PhantomData<T>,

     

       55
       55
       +
       }

     

       56
       56
       +
       

     

       57
       57
       +
       impl<W: Write, T> ItemEncoder<W, T> {

     

       58
       58
       +
           pub fn new(writer: W, item_count: usize) -> Self {

     

       59
       59
       +
               assert!(item_count > 0);

     

       60
       60
       +
               ItemEncoder {

     

       61
       61
       +
                   writer,

     

       62
       62
       +
                   prev_timestamp: 0,

     

       63
       63
       +
                   prev_delta: 0,

     

       64
       64
       +
                   item_count,

     

       65
       65
       +
                   _item: PhantomData,

     

       66
       66
       +
               }

     

       67
       67
       +
           }

     

       68
       68
       +
       

     

       69
       69
       +
           /// NOTE: this is a best effort estimate of the encoded length of the block.

     

       70
       70
       +
           /// if T contains variable-length data, the encoded length may be larger than this estimate.

     

       71
       71
       +
           pub fn encoded_len(item_count: usize) -> usize {

     

       72
       72
       +
               // items length + item count * delta length + data length

     

       73
       73
       +
               size_of::<usize>() + item_count * size_of::<(i64, T)>()

     

       74
       74
       +
           }

     

       75
       75
       +
       

     

       76
       76
       +
           pub fn encode(&mut self, item: &Item<T>) -> io::Result<()> {

     

       77
       77
       +
               if self.prev_timestamp == 0 {

     

       78
       78
       +
                   self.writer.write_varint(self.item_count)?;

     

       79
       79
       +
                   // self.writer.write_varint(item.timestamp)?;

     

       80
       80
       +
                   self.prev_timestamp = item.timestamp;

     

       81
       81
       +
                   self.write_data(&item.data)?;

     

       82
       82
       +
                   return Ok(());

     

       83
       83
       +
               }

     

       84
       84
       +
       

     

       85
       85
       +
               let delta = (item.timestamp as i128 - self.prev_timestamp as i128) as i64;

     

       86
       86
       +
       

     

       87
       87
       +
               self.writer.write_varint(delta - self.prev_delta)?;

     

       88
       88
       +
               self.prev_timestamp = item.timestamp;

     

       89
       89
       +
               self.prev_delta = delta;

     

       90
       90
       +
       

     

       91
       91
       +
               self.write_data(&item.data)?;

     

       92
       92
       +
       

     

       93
       93
       +
               Ok(())

     

       94
       94
       +
           }

     

       95
       95
       +
       

     

       96
       96
       +
           fn write_data(&mut self, data: &[u8]) -> io::Result<()> {

     

       97
       97
       +
               self.writer.write_varint(data.len())?;

     

       98
       98
       +
               self.writer.write_all(data)?;

     

       99
       99
       +
               Ok(())

     

       100
       100
       +
           }

     

       101
       101
       +
       

     

       102
       102
       +
           pub fn finish(mut self) -> io::Result<W> {

     

       103
       103
       +
               self.writer.flush()?;

     

       104
       104
       +
               Ok(self.writer)

     

       105
       105
       +
           }

     

       106
       106
       +
       }

     

       107
       107
       +
       

     

       108
       108
       +
       pub struct ItemDecoder<R, T> {

     

       109
       109
       +
           reader: R,

     

       110
       110
       +
           current_timestamp: u64,

     

       111
       111
       +
           current_delta: i64,

     

       112
       112
       +
           items_read: usize,

     

       113
       113
       +
           expected: usize,

     

       114
       114
       +
           _item: PhantomData<T>,

     

       115
       115
       +
       }

     

       116
       116
       +
       

     

       117
       117
       +
       impl<R: Read, T: Archive> ItemDecoder<R, T> {

     

       118
       118
       +
           pub fn new(mut reader: R, start_timestamp: u64) -> io::Result<Self> {

     

       119
       119
       +
               let expected = match reader.read_varint() {

     

       120
       120
       +
                   Ok(expected) => expected,

     

       121
       121
       +
                   Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => 0,

     

       122
       122
       +
                   Err(e) => return Err(e.into()),

     

       123
       123
       +
               };

     

       124
       124
       +
       

     

       125
       125
       +
               Ok(ItemDecoder {

     

       126
       126
       +
                   reader,

     

       127
       127
       +
                   current_timestamp: start_timestamp,

     

       128
       128
       +
                   current_delta: 0,

     

       129
       129
       +
                   items_read: 0,

     

       130
       130
       +
                   expected,

     

       131
       131
       +
                   _item: PhantomData,

     

       132
       132
       +
               })

     

       133
       133
       +
           }

     

       134
       134
       +
       

     

       135
       135
       +
           pub fn item_count(&self) -> usize {

     

       136
       136
       +
               self.expected

     

       137
       137
       +
           }

     

       138
       138
       +
       

     

       139
       139
       +
           pub fn decode(&mut self) -> io::Result<Option<Item<T>>> {

     

       140
       140
       +
               if self.items_read == 0 {

     

       141
       141
       +
                   // read the first timestamp

     

       142
       142
       +
                   // let timestamp = match self.reader.read_varint::<u64>() {

     

       143
       143
       +
                   //     Ok(timestamp) => timestamp,

     

       144
       144
       +
                   //     Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),

     

       145
       145
       +
                   //     Err(e) => return Err(e.into()),

     

       146
       146
       +
                   // };

     

       147
       147
       +
                   // self.current_timestamp = timestamp;

     

       148
       148
       +
       

     

       149
       149
       +
                   let Some(data_raw) = self.read_item()? else {

     

       150
       150
       +
                       return Ok(None);

     

       151
       151
       +
                   };

     

       152
       152
       +
       

     

       153
       153
       +
                   self.items_read += 1;

     

       154
       154
       +
                   return Ok(Some(Item {

     

       155
       155
       +
                       timestamp: self.current_timestamp,

     

       156
       156
       +
                       data: data_raw,

     

       157
       157
       +
                       phantom: PhantomData,

     

       158
       158
       +
                   }));

     

       159
       159
       +
               }

     

       160
       160
       +
       

     

       161
       161
       +
               if self.items_read >= self.expected {

     

       162
       162
       +
                   return Ok(None);

     

       163
       163
       +
               }

     

       164
       164
       +
       

     

       165
       165
       +
               let Some(_delta) = self.read_timestamp()? else {

     

       166
       166
       +
                   return Ok(None);

     

       167
       167
       +
               };

     

       168
       168
       +
       

     

       169
       169
       +
               // read data

     

       170
       170
       +
               let data_raw = match self.read_item()? {

     

       171
       171
       +
                   Some(data_raw) => data_raw,

     

       172
       172
       +
                   None => {

     

       173
       173
       +
                       return Err(io::Error::new(

     

       174
       174
       +
                           io::ErrorKind::UnexpectedEof,

     

       175
       175
       +
                           "expected data after delta",

     

       176
       176
       +
                       )

     

       177
       177
       +
                       .into());

     

       178
       178
       +
                   }

     

       179
       179
       +
               };

     

       180
       180
       +
       

     

       181
       181
       +
               self.items_read += 1;

     

       182
       182
       +
               Ok(Some(Item {

     

       183
       183
       +
                   timestamp: self.current_timestamp,

     

       184
       184
       +
                   data: data_raw,

     

       185
       185
       +
                   phantom: PhantomData,

     

       186
       186
       +
               }))

     

       187
       187
       +
           }

     

       188
       188
       +
       

     

       189
       189
       +
           // [10, 11, 12, 14] -> [1, 1, 2] -> [0, 1]

     

       190
       190
       +
           fn read_timestamp(&mut self) -> io::Result<Option<u64>> {

     

       191
       191
       +
               let delta = match self.reader.read_varint::<i64>() {

     

       192
       192
       +
                   Ok(delta) => delta,

     

       193
       193
       +
                   Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),

     

       194
       194
       +
                   Err(e) => return Err(e.into()),

     

       195
       195
       +
               };

     

       196
       196
       +
               self.current_delta += delta;

     

       197
       197
       +
               self.current_timestamp =

     

       198
       198
       +
                   (self.current_timestamp as i128 + self.current_delta as i128) as u64;

     

       199
       199
       +
               Ok(Some(self.current_timestamp))

     

       200
       200
       +
           }

     

       201
       201
       +
       

     

       202
       202
       +
           fn read_item(&mut self) -> io::Result<Option<AlignedVec>> {

     

       203
       203
       +
               let data_len = match self.reader.read_varint::<usize>() {

     

       204
       204
       +
                   Ok(data_len) => data_len,

     

       205
       205
       +
                   Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),

     

       206
       206
       +
                   Err(e) => return Err(e.into()),

     

       207
       207
       +
               };

     

       208
       208
       +
               let mut data_raw = AlignedVec::with_capacity(data_len);

     

       209
       209
       +
               for _ in 0..data_len {

     

       210
       210
       +
                   data_raw.push(0);

     

       211
       211
       +
               }

     

       212
       212
       +
               self.reader.read_exact(data_raw.as_mut_slice())?;

     

       213
       213
       +
               Ok(Some(data_raw))

     

       214
       214
       +
           }

     

       215
       215
       +
       }

     

       216
       216
       +
       

     

       217
       217
       +
       impl<R: Read, T: Archive> Iterator for ItemDecoder<R, T> {

     

       218
       218
       +
           type Item = io::Result<Item<T>>;

     

       219
       219
       +
       

     

       220
       220
       +
           fn next(&mut self) -> Option<Self::Item> {

     

       221
       221
       +
               self.decode().transpose()

     

       222
       222
       +
           }

     

       223
       223
       +
       

     

       224
       224
       +
           fn size_hint(&self) -> (usize, Option<usize>) {

     

       225
       225
       +
               (self.expected, Some(self.expected))

     

       226
       226
       +
           }

     

       227
       227
       +
       }

     

       228
       228
       +
       

     

       229
       229
       +
       #[cfg(test)]

     

       230
       230
       +
       mod test {

     

       231
       231
       +
           use super::*;

     

       232
       232
       +
           use rkyv::{Archive, Deserialize, Serialize};

     

       233
       233
       +
           use std::io::Cursor;

     

       234
       234
       +
       

     

       235
       235
       +
           #[derive(Archive, Deserialize, Serialize, Debug, PartialEq)]

     

       236
       236
       +
           #[rkyv(compare(PartialEq))]

     

       237
       237
       +
           struct TestData {

     

       238
       238
       +
               id: u32,

     

       239
       239
       +
               value: String,

     

       240
       240
       +
           }

     

       241
       241
       +
       

     

       242
       242
       +
           #[test]

     

       243
       243
       +
           fn test_encoder_decoder_single_item() {

     

       244
       244
       +
               let data = TestData {

     

       245
       245
       +
                   id: 123,

     

       246
       246
       +
                   value: "test".to_string(),

     

       247
       247
       +
               };

     

       248
       248
       +
       

     

       249
       249
       +
               let item = Item::new(1000, &data);

     

       250
       250
       +
       

     

       251
       251
       +
               // encode

     

       252
       252
       +
               let mut buffer = Vec::new();

     

       253
       253
       +
               let mut encoder = ItemEncoder::new(&mut buffer, 1);

     

       254
       254
       +
               encoder.encode(&item).unwrap();

     

       255
       255
       +
               encoder.finish().unwrap();

     

       256
       256
       +
       

     

       257
       257
       +
               // decode

     

       258
       258
       +
               let cursor = Cursor::new(buffer);

     

       259
       259
       +
               let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       260
       260
       +
       

     

       261
       261
       +
               let decoded_item = decoder.decode().unwrap().unwrap();

     

       262
       262
       +
               assert_eq!(decoded_item.timestamp, 1000);

     

       263
       263
       +
       

     

       264
       264
       +
               let decoded_data = decoded_item.deser().unwrap();

     

       265
       265
       +
               assert_eq!(decoded_data.id, 123);

     

       266
       266
       +
               assert_eq!(decoded_data.value.as_str(), "test");

     

       267
       267
       +
           }

     

       268
       268
       +
       

     

       269
       269
       +
           #[test]

     

       270
       270
       +
           fn test_encoder_decoder_multiple_items() {

     

       271
       271
       +
               let items = vec![

     

       272
       272
       +
                   Item::new(

     

       273
       273
       +
                       1000,

     

       274
       274
       +
                       &TestData {

     

       275
       275
       +
                           id: 1,

     

       276
       276
       +
                           value: "first".to_string(),

     

       277
       277
       +
                       },

     

       278
       278
       +
                   ),

     

       279
       279
       +
                   Item::new(

     

       280
       280
       +
                       1010,

     

       281
       281
       +
                       &TestData {

     

       282
       282
       +
                           id: 2,

     

       283
       283
       +
                           value: "second".to_string(),

     

       284
       284
       +
                       },

     

       285
       285
       +
                   ),

     

       286
       286
       +
                   Item::new(

     

       287
       287
       +
                       1015,

     

       288
       288
       +
                       &TestData {

     

       289
       289
       +
                           id: 3,

     

       290
       290
       +
                           value: "third".to_string(),

     

       291
       291
       +
                       },

     

       292
       292
       +
                   ),

     

       293
       293
       +
                   Item::new(

     

       294
       294
       +
                       1025,

     

       295
       295
       +
                       &TestData {

     

       296
       296
       +
                           id: 4,

     

       297
       297
       +
                           value: "fourth".to_string(),

     

       298
       298
       +
                       },

     

       299
       299
       +
                   ),

     

       300
       300
       +
               ];

     

       301
       301
       +
       

     

       302
       302
       +
               // encode

     

       303
       303
       +
               let mut buffer = Vec::new();

     

       304
       304
       +
               let mut encoder = ItemEncoder::new(&mut buffer, items.len());

     

       305
       305
       +
       

     

       306
       306
       +
               for item in &items {

     

       307
       307
       +
                   encoder.encode(item).unwrap();

     

       308
       308
       +
               }

     

       309
       309
       +
               encoder.finish().unwrap();

     

       310
       310
       +
       

     

       311
       311
       +
               // decode

     

       312
       312
       +
               let cursor = Cursor::new(buffer);

     

       313
       313
       +
               let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       314
       314
       +
       

     

       315
       315
       +
               let mut decoded_items = Vec::new();

     

       316
       316
       +
               while let Some(item) = decoder.decode().unwrap() {

     

       317
       317
       +
                   decoded_items.push(item);

     

       318
       318
       +
               }

     

       319
       319
       +
       

     

       320
       320
       +
               assert_eq!(decoded_items.len(), 4);

     

       321
       321
       +
       

     

       322
       322
       +
               for (original, decoded) in items.iter().zip(decoded_items.iter()) {

     

       323
       323
       +
                   assert_eq!(original.timestamp, decoded.timestamp);

     

       324
       324
       +
                   assert_eq!(original.deser().unwrap().id, decoded.deser().unwrap().id);

     

       325
       325
       +
                   assert_eq!(

     

       326
       326
       +
                       original.deser().unwrap().value.as_str(),

     

       327
       327
       +
                       decoded.deser().unwrap().value.as_str()

     

       328
       328
       +
                   );

     

       329
       329
       +
               }

     

       330
       330
       +
           }

     

       331
       331
       +
       

     

       332
       332
       +
           #[test]

     

       333
       333
       +
           fn test_encoder_decoder_with_iterator() {

     

       334
       334
       +
               let items = vec![

     

       335
       335
       +
                   Item::new(

     

       336
       336
       +
                       2000,

     

       337
       337
       +
                       &TestData {

     

       338
       338
       +
                           id: 10,

     

       339
       339
       +
                           value: "a".to_string(),

     

       340
       340
       +
                       },

     

       341
       341
       +
                   ),

     

       342
       342
       +
                   Item::new(

     

       343
       343
       +
                       2005,

     

       344
       344
       +
                       &TestData {

     

       345
       345
       +
                           id: 20,

     

       346
       346
       +
                           value: "b".to_string(),

     

       347
       347
       +
                       },

     

       348
       348
       +
                   ),

     

       349
       349
       +
                   Item::new(

     

       350
       350
       +
                       2012,

     

       351
       351
       +
                       &TestData {

     

       352
       352
       +
                           id: 30,

     

       353
       353
       +
                           value: "c".to_string(),

     

       354
       354
       +
                       },

     

       355
       355
       +
                   ),

     

       356
       356
       +
               ];

     

       357
       357
       +
       

     

       358
       358
       +
               // encode

     

       359
       359
       +
               let mut buffer = Vec::new();

     

       360
       360
       +
               let mut encoder = ItemEncoder::new(&mut buffer, items.len());

     

       361
       361
       +
       

     

       362
       362
       +
               for item in &items {

     

       363
       363
       +
                   encoder.encode(item).unwrap();

     

       364
       364
       +
               }

     

       365
       365
       +
               encoder.finish().unwrap();

     

       366
       366
       +
       

     

       367
       367
       +
               // decode

     

       368
       368
       +
               let cursor = Cursor::new(buffer);

     

       369
       369
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 2000).unwrap();

     

       370
       370
       +
       

     

       371
       371
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       372
       372
       +
               let decoded_items = decoded_items.unwrap();

     

       373
       373
       +
       

     

       374
       374
       +
               assert_eq!(decoded_items.len(), 3);

     

       375
       375
       +
               assert_eq!(decoded_items[0].timestamp, 2000);

     

       376
       376
       +
               assert_eq!(decoded_items[1].timestamp, 2005);

     

       377
       377
       +
               assert_eq!(decoded_items[2].timestamp, 2012);

     

       378
       378
       +
       

     

       379
       379
       +
               assert_eq!(decoded_items[0].deser().unwrap().id, 10);

     

       380
       380
       +
               assert_eq!(decoded_items[1].deser().unwrap().id, 20);

     

       381
       381
       +
               assert_eq!(decoded_items[2].deser().unwrap().id, 30);

     

       382
       382
       +
           }

     

       383
       383
       +
       

     

       384
       384
       +
           #[test]

     

       385
       385
       +
           fn test_delta_compression() {

     

       386
       386
       +
               let items = vec![

     

       387
       387
       +
                   Item::new(

     

       388
       388
       +
                       1000,

     

       389
       389
       +
                       &TestData {

     

       390
       390
       +
                           id: 1,

     

       391
       391
       +
                           value: "a".to_string(),

     

       392
       392
       +
                       },

     

       393
       393
       +
                   ),

     

       394
       394
       +
                   Item::new(

     

       395
       395
       +
                       1010,

     

       396
       396
       +
                       &TestData {

     

       397
       397
       +
                           id: 2,

     

       398
       398
       +
                           value: "b".to_string(),

     

       399
       399
       +
                       },

     

       400
       400
       +
                   ), // delta = 10

     

       401
       401
       +
                   Item::new(

     

       402
       402
       +
                       1020,

     

       403
       403
       +
                       &TestData {

     

       404
       404
       +
                           id: 3,

     

       405
       405
       +
                           value: "c".to_string(),

     

       406
       406
       +
                       },

     

       407
       407
       +
                   ), // delta = 10, delta-of-delta = 0

     

       408
       408
       +
                   Item::new(

     

       409
       409
       +
                       1025,

     

       410
       410
       +
                       &TestData {

     

       411
       411
       +
                           id: 4,

     

       412
       412
       +
                           value: "d".to_string(),

     

       413
       413
       +
                       },

     

       414
       414
       +
                   ), // delta = 5, delta-of-delta = -5

     

       415
       415
       +
               ];

     

       416
       416
       +
       

     

       417
       417
       +
               let mut buffer = Vec::new();

     

       418
       418
       +
               let mut encoder = ItemEncoder::new(&mut buffer, items.len());

     

       419
       419
       +
       

     

       420
       420
       +
               for item in &items {

     

       421
       421
       +
                   encoder.encode(item).unwrap();

     

       422
       422
       +
               }

     

       423
       423
       +
               encoder.finish().unwrap();

     

       424
       424
       +
       

     

       425
       425
       +
               // decode and verify

     

       426
       426
       +
               let cursor = Cursor::new(buffer);

     

       427
       427
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       428
       428
       +
       

     

       429
       429
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       430
       430
       +
               let decoded_items = decoded_items.unwrap();

     

       431
       431
       +
       

     

       432
       432
       +
               for (original, decoded) in items.iter().zip(decoded_items.iter()) {

     

       433
       433
       +
                   assert_eq!(original.timestamp, decoded.timestamp);

     

       434
       434
       +
                   assert_eq!(original.deser().unwrap().id, decoded.deser().unwrap().id);

     

       435
       435
       +
               }

     

       436
       436
       +
           }

     

       437
       437
       +
       

     

       438
       438
       +
           #[test]

     

       439
       439
       +
           fn test_empty_decode() {

     

       440
       440
       +
               let buffer = Vec::new();

     

       441
       441
       +
               let cursor = Cursor::new(buffer);

     

       442
       442
       +
               let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       443
       443
       +
       

     

       444
       444
       +
               let result = decoder.decode().unwrap();

     

       445
       445
       +
               assert!(result.is_none());

     

       446
       446
       +
           }

     

       447
       447
       +
       

     

       448
       448
       +
           #[test]

     

       449
       449
       +
           fn test_backwards_timestamp() {

     

       450
       450
       +
               let items = vec![

     

       451
       451
       +
                   Item::new(

     

       452
       452
       +
                       1000,

     

       453
       453
       +
                       &TestData {

     

       454
       454
       +
                           id: 1,

     

       455
       455
       +
                           value: "first".to_string(),

     

       456
       456
       +
                       },

     

       457
       457
       +
                   ),

     

       458
       458
       +
                   Item::new(

     

       459
       459
       +
                       900,

     

       460
       460
       +
                       &TestData {

     

       461
       461
       +
                           id: 2,

     

       462
       462
       +
                           value: "second".to_string(),

     

       463
       463
       +
                       },

     

       464
       464
       +
                   ),

     

       465
       465
       +
               ];

     

       466
       466
       +
       

     

       467
       467
       +
               let mut buffer = Vec::new();

     

       468
       468
       +
               let mut encoder = ItemEncoder::new(&mut buffer, items.len());

     

       469
       469
       +
       

     

       470
       470
       +
               for item in &items {

     

       471
       471
       +
                   encoder.encode(item).unwrap();

     

       472
       472
       +
               }

     

       473
       473
       +
               encoder.finish().unwrap();

     

       474
       474
       +
       

     

       475
       475
       +
               let cursor = Cursor::new(buffer);

     

       476
       476
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       477
       477
       +
       

     

       478
       478
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       479
       479
       +
               let decoded_items = decoded_items.unwrap();

     

       480
       480
       +
       

     

       481
       481
       +
               assert_eq!(decoded_items.len(), 2);

     

       482
       482
       +
               assert_eq!(decoded_items[0].timestamp, 1000);

     

       483
       483
       +
               assert_eq!(decoded_items[1].timestamp, 900);

     

       484
       484
       +
           }

     

       485
       485
       +
       

     

       486
       486
       +
           #[test]

     

       487
       487
       +
           fn test_different_data_sizes() {

     

       488
       488
       +
               let small_data = TestData {

     

       489
       489
       +
                   id: 1,

     

       490
       490
       +
                   value: "x".to_string(),

     

       491
       491
       +
               };

     

       492
       492
       +
               let large_data = TestData {

     

       493
       493
       +
                   id: 2,

     

       494
       494
       +
                   value: "a".repeat(1000),

     

       495
       495
       +
               };

     

       496
       496
       +
       

     

       497
       497
       +
               let items = vec![Item::new(1000, &small_data), Item::new(1001, &large_data)];

     

       498
       498
       +
       

     

       499
       499
       +
               let mut buffer = Vec::new();

     

       500
       500
       +
               let mut encoder = ItemEncoder::new(&mut buffer, items.len());

     

       501
       501
       +
       

     

       502
       502
       +
               for item in &items {

     

       503
       503
       +
                   encoder.encode(item).unwrap();

     

       504
       504
       +
               }

     

       505
       505
       +
               encoder.finish().unwrap();

     

       506
       506
       +
       

     

       507
       507
       +
               let cursor = Cursor::new(buffer);

     

       508
       508
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       509
       509
       +
       

     

       510
       510
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       511
       511
       +
               let decoded_items = decoded_items.unwrap();

     

       512
       512
       +
       

     

       513
       513
       +
               assert_eq!(decoded_items.len(), 2);

     

       514
       514
       +
               assert_eq!(decoded_items[0].deser().unwrap().value.as_str(), "x");

     

       515
       515
       +
               assert_eq!(decoded_items[1].deser().unwrap().value.len(), 1000);

     

       516
       516
       +
               assert_eq!(

     

       517
       517
       +
                   decoded_items[1].deser().unwrap().value.as_str(),

     

       518
       518
       +
                   "a".repeat(1000)

     

       519
       519
       +
               );

     

       520
       520
       +
           }

     

       521
       521
       +
       }

+232

server/src/db/handle.rs

···

       1
       1
       +
       use std::{

     

       2
       2
       +
           fmt::Debug,

     

       3
       3
       +
           io::Cursor,

     

       4
       4
       +
           ops::{Bound, Deref, RangeBounds},

     

       5
       5
       +
           sync::atomic::{AtomicU64, Ordering as AtomicOrdering},

     

       6
       6
       +
           time::Duration,

     

       7
       7
       +
       };

     

       8
       8
       +
       

     

       9
       9
       +
       use byteview::ByteView;

     

       10
       10
       +
       use fjall::{Keyspace, Partition, PartitionCreateOptions, Slice};

     

       11
       11
       +
       use itertools::Itertools;

     

       12
       12
       +
       use parking_lot::Mutex;

     

       13
       13
       +
       use rayon::iter::{IntoParallelIterator, ParallelIterator};

     

       14
       14
       +
       use rclite::Arc;

     

       15
       15
       +
       use smol_str::SmolStr;

     

       16
       16
       +
       

     

       17
       17
       +
       use crate::{

     

       18
       18
       +
           db::{EventRecord, NsidHit, block},

     

       19
       19
       +
           error::AppResult,

     

       20
       20
       +
           utils::{CLOCK, DefaultRateTracker, RateTracker, ReadVariableExt, varints_unsigned_encoded},

     

       21
       21
       +
       };

     

       22
       22
       +
       

     

       23
       23
       +
       pub type ItemDecoder = block::ItemDecoder<Cursor<Slice>, NsidHit>;

     

       24
       24
       +
       pub type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;

     

       25
       25
       +
       pub type Item = block::Item<NsidHit>;

     

       26
       26
       +
       

     

       27
       27
       +
       pub struct Block {

     

       28
       28
       +
           pub written: usize,

     

       29
       29
       +
           pub key: ByteView,

     

       30
       30
       +
           pub data: Vec<u8>,

     

       31
       31
       +
       }

     

       32
       32
       +
       

     

       33
       33
       +
       pub struct LexiconHandle {

     

       34
       34
       +
           tree: Partition,

     

       35
       35
       +
           nsid: SmolStr,

     

       36
       36
       +
           buf: Arc<Mutex<Vec<EventRecord>>>,

     

       37
       37
       +
           last_insert: AtomicU64, // relaxed

     

       38
       38
       +
           eps: DefaultRateTracker,

     

       39
       39
       +
       }

     

       40
       40
       +
       

     

       41
       41
       +
       impl Debug for LexiconHandle {

     

       42
       42
       +
           fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

     

       43
       43
       +
               f.debug_struct("LexiconHandle")

     

       44
       44
       +
                   .field("nsid", self.nsid())

     

       45
       45
       +
                   .finish()

     

       46
       46
       +
           }

     

       47
       47
       +
       }

     

       48
       48
       +
       

     

       49
       49
       +
       impl Deref for LexiconHandle {

     

       50
       50
       +
           type Target = Partition;

     

       51
       51
       +
       

     

       52
       52
       +
           fn deref(&self) -> &Self::Target {

     

       53
       53
       +
               &self.tree

     

       54
       54
       +
           }

     

       55
       55
       +
       }

     

       56
       56
       +
       

     

       57
       57
       +
       impl LexiconHandle {

     

       58
       58
       +
           pub fn new(keyspace: &Keyspace, nsid: &str) -> Self {

     

       59
       59
       +
               let opts = PartitionCreateOptions::default()

     

       60
       60
       +
                   .block_size(1024 * 128)

     

       61
       61
       +
                   .compression(fjall::CompressionType::Miniz(9));

     

       62
       62
       +
               Self {

     

       63
       63
       +
                   tree: keyspace.open_partition(nsid, opts).unwrap(),

     

       64
       64
       +
                   nsid: nsid.into(),

     

       65
       65
       +
                   buf: Default::default(),

     

       66
       66
       +
                   last_insert: AtomicU64::new(0),

     

       67
       67
       +
                   eps: RateTracker::new(Duration::from_secs(10)),

     

       68
       68
       +
               }

     

       69
       69
       +
           }

     

       70
       70
       +
       

     

       71
       71
       +
           pub fn nsid(&self) -> &SmolStr {

     

       72
       72
       +
               &self.nsid

     

       73
       73
       +
           }

     

       74
       74
       +
       

     

       75
       75
       +
           pub fn item_count(&self) -> usize {

     

       76
       76
       +
               self.buf.lock().len()

     

       77
       77
       +
           }

     

       78
       78
       +
       

     

       79
       79
       +
           pub fn since_last_activity(&self) -> u64 {

     

       80
       80
       +
               CLOCK.delta_as_nanos(self.last_insert.load(AtomicOrdering::Relaxed), CLOCK.raw())

     

       81
       81
       +
           }

     

       82
       82
       +
       

     

       83
       83
       +
           pub fn suggested_block_size(&self) -> usize {

     

       84
       84
       +
               self.eps.rate() as usize * 60

     

       85
       85
       +
           }

     

       86
       86
       +
       

     

       87
       87
       +
           pub fn queue(&self, events: impl IntoIterator<Item = EventRecord>) {

     

       88
       88
       +
               let mut count = 0;

     

       89
       89
       +
               self.buf.lock().extend(events.into_iter().inspect(|_| {

     

       90
       90
       +
                   count += 1;

     

       91
       91
       +
               }));

     

       92
       92
       +
               self.last_insert.store(CLOCK.raw(), AtomicOrdering::Relaxed);

     

       93
       93
       +
               self.eps.observe(count);

     

       94
       94
       +
           }

     

       95
       95
       +
       

     

       96
       96
       +
           pub fn compact(

     

       97
       97
       +
               &self,

     

       98
       98
       +
               compact_to: usize,

     

       99
       99
       +
               range: impl RangeBounds<u64>,

     

       100
       100
       +
               sort: bool,

     

       101
       101
       +
           ) -> AppResult<()> {

     

       102
       102
       +
               let start_limit = match range.start_bound().cloned() {

     

       103
       103
       +
                   Bound::Included(start) => start,

     

       104
       104
       +
                   Bound::Excluded(start) => start.saturating_add(1),

     

       105
       105
       +
                   Bound::Unbounded => 0,

     

       106
       106
       +
               };

     

       107
       107
       +
               let end_limit = match range.end_bound().cloned() {

     

       108
       108
       +
                   Bound::Included(end) => end,

     

       109
       109
       +
                   Bound::Excluded(end) => end.saturating_sub(1),

     

       110
       110
       +
                   Bound::Unbounded => u64::MAX,

     

       111
       111
       +
               };

     

       112
       112
       +
       

     

       113
       113
       +
               let start_key = varints_unsigned_encoded([start_limit]);

     

       114
       114
       +
               let end_key = varints_unsigned_encoded([end_limit]);

     

       115
       115
       +
       

     

       116
       116
       +
               let blocks_to_compact = self

     

       117
       117
       +
                   .tree

     

       118
       118
       +
                   .range(start_key..end_key)

     

       119
       119
       +
                   .collect::<Result<Vec<_>, _>>()?;

     

       120
       120
       +
               if blocks_to_compact.len() < 2 {

     

       121
       121
       +
                   tracing::info!("{}: nothing to compact", self.nsid);

     

       122
       122
       +
                   return Ok(());

     

       123
       123
       +
               }

     

       124
       124
       +
       

     

       125
       125
       +
               let start_blocks_size = blocks_to_compact.len();

     

       126
       126
       +
               let keys_to_delete = blocks_to_compact.iter().map(|(key, _)| key);

     

       127
       127
       +
               let mut all_items =

     

       128
       128
       +
                   blocks_to_compact

     

       129
       129
       +
                       .iter()

     

       130
       130
       +
                       .try_fold(Vec::new(), |mut acc, (key, value)| {

     

       131
       131
       +
                           let mut timestamps = Cursor::new(key);

     

       132
       132
       +
                           let start_timestamp = timestamps.read_varint()?;

     

       133
       133
       +
                           let decoder = block::ItemDecoder::new(Cursor::new(value), start_timestamp)?;

     

       134
       134
       +
                           let mut items = decoder.collect::<Result<Vec<_>, _>>()?;

     

       135
       135
       +
                           acc.append(&mut items);

     

       136
       136
       +
                           AppResult::Ok(acc)

     

       137
       137
       +
                       })?;

     

       138
       138
       +
       

     

       139
       139
       +
               if sort {

     

       140
       140
       +
                   all_items.sort_unstable_by_key(|e| e.timestamp);

     

       141
       141
       +
               }

     

       142
       142
       +
       

     

       143
       143
       +
               let new_blocks = all_items

     

       144
       144
       +
                   .into_iter()

     

       145
       145
       +
                   .chunks(compact_to)

     

       146
       146
       +
                   .into_iter()

     

       147
       147
       +
                   .map(|chunk| chunk.collect_vec())

     

       148
       148
       +
                   .collect_vec()

     

       149
       149
       +
                   .into_par_iter()

     

       150
       150
       +
                   .map(|chunk| {

     

       151
       151
       +
                       let count = chunk.len();

     

       152
       152
       +
                       Self::encode_block_from_items(chunk, count)

     

       153
       153
       +
                   })

     

       154
       154
       +
                   .collect::<Result<Vec<_>, _>>()?;

     

       155
       155
       +
               let end_blocks_size = new_blocks.len();

     

       156
       156
       +
       

     

       157
       157
       +
               for key in keys_to_delete {

     

       158
       158
       +
                   self.tree.remove(key.clone())?;

     

       159
       159
       +
               }

     

       160
       160
       +
               for block in new_blocks {

     

       161
       161
       +
                   self.tree.insert(block.key, block.data)?;

     

       162
       162
       +
               }

     

       163
       163
       +
       

     

       164
       164
       +
               tracing::info!(

     

       165
       165
       +
                   "{}: compacted {} blocks to {} blocks ({}% reduction)",

     

       166
       166
       +
                   self.nsid,

     

       167
       167
       +
                   start_blocks_size,

     

       168
       168
       +
                   end_blocks_size,

     

       169
       169
       +
                   ((start_blocks_size - end_blocks_size) as f64 / start_blocks_size as f64) * 100.0,

     

       170
       170
       +
               );

     

       171
       171
       +
       

     

       172
       172
       +
               Ok(())

     

       173
       173
       +
           }

     

       174
       174
       +
       

     

       175
       175
       +
           pub fn encode_block_from_items(

     

       176
       176
       +
               items: impl IntoIterator<Item = Item>,

     

       177
       177
       +
               count: usize,

     

       178
       178
       +
           ) -> AppResult<Block> {

     

       179
       179
       +
               if count == 0 {

     

       180
       180
       +
                   return Err(std::io::Error::new(

     

       181
       181
       +
                       std::io::ErrorKind::InvalidInput,

     

       182
       182
       +
                       "no items requested",

     

       183
       183
       +
                   )

     

       184
       184
       +
                   .into());

     

       185
       185
       +
               }

     

       186
       186
       +
               let mut writer =

     

       187
       187
       +
                   ItemEncoder::new(Vec::with_capacity(ItemEncoder::encoded_len(count)), count);

     

       188
       188
       +
               let mut start_timestamp = None;

     

       189
       189
       +
               let mut end_timestamp = None;

     

       190
       190
       +
               let mut written = 0_usize;

     

       191
       191
       +
               for item in items.into_iter().take(count) {

     

       192
       192
       +
                   writer.encode(&item)?;

     

       193
       193
       +
                   if start_timestamp.is_none() {

     

       194
       194
       +
                       start_timestamp = Some(item.timestamp);

     

       195
       195
       +
                   }

     

       196
       196
       +
                   end_timestamp = Some(item.timestamp);

     

       197
       197
       +
                   written += 1;

     

       198
       198
       +
               }

     

       199
       199
       +
               if written != count {

     

       200
       200
       +
                   return Err(std::io::Error::new(

     

       201
       201
       +
                       std::io::ErrorKind::InvalidData,

     

       202
       202
       +
                       "unexpected number of items, invalid data?",

     

       203
       203
       +
                   )

     

       204
       204
       +
                   .into());

     

       205
       205
       +
               }

     

       206
       206
       +
               if let (Some(start_timestamp), Some(end_timestamp)) = (start_timestamp, end_timestamp) {

     

       207
       207
       +
                   let value = writer.finish()?;

     

       208
       208
       +
                   let key = varints_unsigned_encoded([start_timestamp, end_timestamp]);

     

       209
       209
       +
                   return Ok(Block {

     

       210
       210
       +
                       written,

     

       211
       211
       +
                       key,

     

       212
       212
       +
                       data: value,

     

       213
       213
       +
                   });

     

       214
       214
       +
               }

     

       215
       215
       +
               Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "no items are in queue").into())

     

       216
       216
       +
           }

     

       217
       217
       +
       

     

       218
       218
       +
           pub fn take_block_items(&self, item_count: usize) -> Vec<Item> {

     

       219
       219
       +
               let mut buf = self.buf.lock();

     

       220
       220
       +
               let end = item_count.min(buf.len());

     

       221
       221
       +
               buf.drain(..end)

     

       222
       222
       +
                   .map(|event| {

     

       223
       223
       +
                       Item::new(

     

       224
       224
       +
                           event.timestamp,

     

       225
       225
       +
                           &NsidHit {

     

       226
       226
       +
                               deleted: event.deleted,

     

       227
       227
       +
                           },

     

       228
       228
       +
                       )

     

       229
       229
       +
                   })

     

       230
       230
       +
                   .collect()

     

       231
       231
       +
           }

     

       232
       232
       +
       }

+447

server/src/db/mod.rs

···

       1
       1
       +
       use std::{

     

       2
       2
       +
           collections::HashMap,

     

       3
       3
       +
           fmt::Debug,

     

       4
       4
       +
           io::Cursor,

     

       5
       5
       +
           ops::{Bound, Deref, RangeBounds},

     

       6
       6
       +
           path::{Path, PathBuf},

     

       7
       7
       +
           time::Duration,

     

       8
       8
       +
       };

     

       9
       9
       +
       

     

       10
       10
       +
       use byteview::StrView;

     

       11
       11
       +
       use fjall::{Config, Keyspace, Partition, PartitionCreateOptions};

     

       12
       12
       +
       use itertools::{Either, Itertools};

     

       13
       13
       +
       use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};

     

       14
       14
       +
       use rclite::Arc;

     

       15
       15
       +
       use rkyv::{Archive, Deserialize, Serialize, rancor::Error};

     

       16
       16
       +
       use smol_str::{SmolStr, ToSmolStr};

     

       17
       17
       +
       use tokio::sync::broadcast;

     

       18
       18
       +
       use tokio_util::sync::CancellationToken;

     

       19
       19
       +
       

     

       20
       20
       +
       use crate::{

     

       21
       21
       +
           db::handle::{ItemDecoder, LexiconHandle},

     

       22
       22
       +
           error::{AppError, AppResult},

     

       23
       23
       +
           jetstream::JetstreamEvent,

     

       24
       24
       +
           utils::{RateTracker, ReadVariableExt, varints_unsigned_encoded},

     

       25
       25
       +
       };

     

       26
       26
       +
       

     

       27
       27
       +
       mod block;

     

       28
       28
       +
       mod handle;

     

       29
       29
       +
       

     

       30
       30
       +
       #[derive(Clone, Debug, Default, Archive, Deserialize, Serialize, PartialEq)]

     

       31
       31
       +
       #[rkyv(compare(PartialEq), derive(Debug))]

     

       32
       32
       +
       pub struct NsidCounts {

     

       33
       33
       +
           pub count: u128,

     

       34
       34
       +
           pub deleted_count: u128,

     

       35
       35
       +
           pub last_seen: u64,

     

       36
       36
       +
       }

     

       37
       37
       +
       

     

       38
       38
       +
       #[derive(Debug, Default, Archive, Deserialize, Serialize, PartialEq)]

     

       39
       39
       +
       #[rkyv(compare(PartialEq), derive(Debug))]

     

       40
       40
       +
       pub struct NsidHit {

     

       41
       41
       +
           pub deleted: bool,

     

       42
       42
       +
       }

     

       43
       43
       +
       

     

       44
       44
       +
       #[derive(Clone)]

     

       45
       45
       +
       pub struct EventRecord {

     

       46
       46
       +
           pub nsid: SmolStr,

     

       47
       47
       +
           pub timestamp: u64, // seconds

     

       48
       48
       +
           pub deleted: bool,

     

       49
       49
       +
       }

     

       50
       50
       +
       

     

       51
       51
       +
       impl EventRecord {

     

       52
       52
       +
           pub fn from_jetstream(event: JetstreamEvent) -> Option<Self> {

     

       53
       53
       +
               match event {

     

       54
       54
       +
                   JetstreamEvent::Commit {

     

       55
       55
       +
                       time_us, commit, ..

     

       56
       56
       +
                   } => Some(Self {

     

       57
       57
       +
                       nsid: commit.collection.into(),

     

       58
       58
       +
                       timestamp: time_us / 1_000_000,

     

       59
       59
       +
                       deleted: false,

     

       60
       60
       +
                   }),

     

       61
       61
       +
                   JetstreamEvent::Delete {

     

       62
       62
       +
                       time_us, commit, ..

     

       63
       63
       +
                   } => Some(Self {

     

       64
       64
       +
                       nsid: commit.collection.into(),

     

       65
       65
       +
                       timestamp: time_us / 1_000_000,

     

       66
       66
       +
                       deleted: true,

     

       67
       67
       +
                   }),

     

       68
       68
       +
                   _ => None,

     

       69
       69
       +
               }

     

       70
       70
       +
           }

     

       71
       71
       +
       }

     

       72
       72
       +
       

     

       73
       73
       +
       pub struct DbInfo {

     

       74
       74
       +
           pub nsids: HashMap<SmolStr, Vec<usize>>,

     

       75
       75
       +
           pub disk_size: u64,

     

       76
       76
       +
       }

     

       77
       77
       +
       

     

       78
       78
       +
       pub struct DbConfig {

     

       79
       79
       +
           pub ks_config: fjall::Config,

     

       80
       80
       +
           pub min_block_size: usize,

     

       81
       81
       +
           pub max_block_size: usize,

     

       82
       82
       +
           pub max_last_activity: u64,

     

       83
       83
       +
       }

     

       84
       84
       +
       

     

       85
       85
       +
       impl DbConfig {

     

       86
       86
       +
           pub fn path(mut self, path: impl AsRef<Path>) -> Self {

     

       87
       87
       +
               self.ks_config = fjall::Config::new(path);

     

       88
       88
       +
               self

     

       89
       89
       +
           }

     

       90
       90
       +
       

     

       91
       91
       +
           pub fn ks(mut self, f: impl FnOnce(fjall::Config) -> fjall::Config) -> Self {

     

       92
       92
       +
               self.ks_config = f(self.ks_config);

     

       93
       93
       +
               self

     

       94
       94
       +
           }

     

       95
       95
       +
       }

     

       96
       96
       +
       

     

       97
       97
       +
       impl Default for DbConfig {

     

       98
       98
       +
           fn default() -> Self {

     

       99
       99
       +
               Self {

     

       100
       100
       +
                   ks_config: fjall::Config::default(),

     

       101
       101
       +
                   min_block_size: 512,

     

       102
       102
       +
                   max_block_size: 500_000,

     

       103
       103
       +
                   max_last_activity: Duration::from_secs(10).as_nanos() as u64,

     

       104
       104
       +
               }

     

       105
       105
       +
           }

     

       106
       106
       +
       }

     

       107
       107
       +
       

     

       108
       108
       +
       // counts is nsid -> NsidCounts

     

       109
       109
       +
       // hits is tree per nsid: varint start time + varint end time -> block of hits

     

       110
       110
       +
       pub struct Db {

     

       111
       111
       +
           pub cfg: DbConfig,

     

       112
       112
       +
           pub ks: Keyspace,

     

       113
       113
       +
           counts: Partition,

     

       114
       114
       +
           hits: scc::HashIndex<SmolStr, Arc<LexiconHandle>>,

     

       115
       115
       +
           sync_pool: threadpool::ThreadPool,

     

       116
       116
       +
           event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,

     

       117
       117
       +
           eps: RateTracker<100>,

     

       118
       118
       +
           cancel_token: CancellationToken,

     

       119
       119
       +
       }

     

       120
       120
       +
       

     

       121
       121
       +
       impl Db {

     

       122
       122
       +
           pub fn new(cfg: DbConfig, cancel_token: CancellationToken) -> AppResult<Self> {

     

       123
       123
       +
               tracing::info!("opening db...");

     

       124
       124
       +
               let ks = cfg.ks_config.clone().open()?;

     

       125
       125
       +
               Ok(Self {

     

       126
       126
       +
                   cfg,

     

       127
       127
       +
                   hits: Default::default(),

     

       128
       128
       +
                   sync_pool: threadpool::Builder::new()

     

       129
       129
       +
                       .num_threads(rayon::current_num_threads() * 2)

     

       130
       130
       +
                       .build(),

     

       131
       131
       +
                   counts: ks.open_partition(

     

       132
       132
       +
                       "_counts",

     

       133
       133
       +
                       PartitionCreateOptions::default().compression(fjall::CompressionType::None),

     

       134
       134
       +
                   )?,

     

       135
       135
       +
                   ks,

     

       136
       136
       +
                   event_broadcaster: broadcast::channel(1000).0,

     

       137
       137
       +
                   eps: RateTracker::new(Duration::from_secs(1)),

     

       138
       138
       +
                   cancel_token,

     

       139
       139
       +
               })

     

       140
       140
       +
           }

     

       141
       141
       +
       

     

       142
       142
       +
           #[inline(always)]

     

       143
       143
       +
           pub fn shutting_down(&self) -> impl Future<Output = ()> {

     

       144
       144
       +
               self.cancel_token.cancelled()

     

       145
       145
       +
           }

     

       146
       146
       +
       

     

       147
       147
       +
           #[inline(always)]

     

       148
       148
       +
           pub fn is_shutting_down(&self) -> bool {

     

       149
       149
       +
               self.cancel_token.is_cancelled()

     

       150
       150
       +
           }

     

       151
       151
       +
       

     

       152
       152
       +
           #[inline(always)]

     

       153
       153
       +
           pub fn eps(&self) -> usize {

     

       154
       154
       +
               self.eps.rate() as usize

     

       155
       155
       +
           }

     

       156
       156
       +
       

     

       157
       157
       +
           #[inline(always)]

     

       158
       158
       +
           pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {

     

       159
       159
       +
               self.event_broadcaster.subscribe()

     

       160
       160
       +
           }

     

       161
       161
       +
       

     

       162
       162
       +
           pub fn sync(&self, all: bool) -> AppResult<()> {

     

       163
       163
       +
               // prepare all the data

     

       164
       164
       +
               let mut data = Vec::with_capacity(self.hits.len());

     

       165
       165
       +
               let _guard = scc::ebr::Guard::new();

     

       166
       166
       +
               for (_, handle) in self.hits.iter(&_guard) {

     

       167
       167
       +
                   let mut nsid_data = Vec::with_capacity(2);

     

       168
       168
       +
                   let mut total_count = 0;

     

       169
       169
       +
                   let is_too_old = handle.since_last_activity() > self.cfg.max_last_activity;

     

       170
       170
       +
                   // if we disconnect for a long time, we want to sync all of what we

     

       171
       171
       +
                   // have to avoid having many small blocks (even if we run compaction

     

       172
       172
       +
                   // later, it reduces work until we run compaction)

     

       173
       173
       +
                   let block_size = (is_too_old || all)

     

       174
       174
       +
                       .then_some(self.cfg.max_block_size)

     

       175
       175
       +
                       .unwrap_or_else(|| {

     

       176
       176
       +
                           self.cfg

     

       177
       177
       +
                               .max_block_size

     

       178
       178
       +
                               .min(self.cfg.min_block_size.max(handle.suggested_block_size()))

     

       179
       179
       +
                       });

     

       180
       180
       +
                   let count = handle.item_count();

     

       181
       181
       +
                   let data_count = count / block_size;

     

       182
       182
       +
                   if count > 0 && (all || data_count > 0 || is_too_old) {

     

       183
       183
       +
                       for i in 0..data_count {

     

       184
       184
       +
                           nsid_data.push((i, handle.clone(), block_size));

     

       185
       185
       +
                           total_count += block_size;

     

       186
       186
       +
                       }

     

       187
       187
       +
                       // only sync remainder if we haven't met block size

     

       188
       188
       +
                       let remainder = count % block_size;

     

       189
       189
       +
                       if (all || data_count == 0) && remainder > 0 {

     

       190
       190
       +
                           nsid_data.push((data_count, handle.clone(), remainder));

     

       191
       191
       +
                           total_count += remainder;

     

       192
       192
       +
                       }

     

       193
       193
       +
                   }

     

       194
       194
       +
                   tracing::info!(

     

       195
       195
       +
                       "{}: will sync {} blocks ({} count)",

     

       196
       196
       +
                       handle.nsid(),

     

       197
       197
       +
                       nsid_data.len(),

     

       198
       198
       +
                       total_count,

     

       199
       199
       +
                   );

     

       200
       200
       +
                   data.push(nsid_data);

     

       201
       201
       +
               }

     

       202
       202
       +
               drop(_guard);

     

       203
       203
       +
       

     

       204
       204
       +
               // process the blocks

     

       205
       205
       +
               data.into_par_iter()

     

       206
       206
       +
                   .map(|chunk| {

     

       207
       207
       +
                       chunk

     

       208
       208
       +
                           .into_iter()

     

       209
       209
       +
                           .map(|(i, handle, max_block_size)| {

     

       210
       210
       +
                               (i, handle.take_block_items(max_block_size), handle)

     

       211
       211
       +
                           })

     

       212
       212
       +
                           .collect::<Vec<_>>()

     

       213
       213
       +
                           .into_par_iter()

     

       214
       214
       +
                           .map(|(i, items, handle)| {

     

       215
       215
       +
                               let count = items.len();

     

       216
       216
       +
                               let block = LexiconHandle::encode_block_from_items(items, count)?;

     

       217
       217
       +
                               tracing::info!(

     

       218
       218
       +
                                   "{}: encoded block with {} items",

     

       219
       219
       +
                                   handle.nsid(),

     

       220
       220
       +
                                   block.written,

     

       221
       221
       +
                               );

     

       222
       222
       +
                               AppResult::Ok((i, block, handle))

     

       223
       223
       +
                           })

     

       224
       224
       +
                           .collect::<Result<Vec<_>, _>>()

     

       225
       225
       +
                   })

     

       226
       226
       +
                   .try_for_each(|chunk| {

     

       227
       227
       +
                       let chunk = chunk?;

     

       228
       228
       +
                       for (i, block, handle) in chunk {

     

       229
       229
       +
                           self.sync_pool

     

       230
       230
       +
                               .execute(move || match handle.insert(block.key, block.data) {

     

       231
       231
       +
                                   Ok(_) => {

     

       232
       232
       +
                                       tracing::info!("{}: [{i}] synced {}", block.written, handle.nsid())

     

       233
       233
       +
                                   }

     

       234
       234
       +
                                   Err(err) => tracing::error!("failed to sync block: {}", err),

     

       235
       235
       +
                               });

     

       236
       236
       +
                       }

     

       237
       237
       +
                       AppResult::Ok(())

     

       238
       238
       +
                   })?;

     

       239
       239
       +
               self.sync_pool.join();

     

       240
       240
       +
       

     

       241
       241
       +
               Ok(())

     

       242
       242
       +
           }

     

       243
       243
       +
       

     

       244
       244
       +
           pub fn compact(

     

       245
       245
       +
               &self,

     

       246
       246
       +
               nsid: impl AsRef<str>,

     

       247
       247
       +
               max_count: usize,

     

       248
       248
       +
               range: impl RangeBounds<u64>,

     

       249
       249
       +
               sort: bool,

     

       250
       250
       +
           ) -> AppResult<()> {

     

       251
       251
       +
               let Some(handle) = self.get_handle(nsid) else {

     

       252
       252
       +
                   return Ok(());

     

       253
       253
       +
               };

     

       254
       254
       +
               handle.compact(max_count, range, sort)

     

       255
       255
       +
           }

     

       256
       256
       +
       

     

       257
       257
       +
           pub fn compact_all(

     

       258
       258
       +
               &self,

     

       259
       259
       +
               max_count: usize,

     

       260
       260
       +
               range: impl RangeBounds<u64> + Clone,

     

       261
       261
       +
               sort: bool,

     

       262
       262
       +
           ) -> AppResult<()> {

     

       263
       263
       +
               for nsid in self.get_nsids() {

     

       264
       264
       +
                   self.compact(nsid, max_count, range.clone(), sort)?;

     

       265
       265
       +
               }

     

       266
       266
       +
               Ok(())

     

       267
       267
       +
           }

     

       268
       268
       +
       

     

       269
       269
       +
           pub fn major_compact(&self) -> AppResult<()> {

     

       270
       270
       +
               self.compact_all(self.cfg.max_block_size, .., true)?;

     

       271
       271
       +
               let _guard = scc::ebr::Guard::new();

     

       272
       272
       +
               for (_, handle) in self.hits.iter(&_guard) {

     

       273
       273
       +
                   handle.deref().major_compact()?;

     

       274
       274
       +
               }

     

       275
       275
       +
               Ok(())

     

       276
       276
       +
           }

     

       277
       277
       +
       

     

       278
       278
       +
           #[inline(always)]

     

       279
       279
       +
           fn get_handle(&self, nsid: impl AsRef<str>) -> Option<Arc<LexiconHandle>> {

     

       280
       280
       +
               let _guard = scc::ebr::Guard::new();

     

       281
       281
       +
               let handle = match self.hits.peek(nsid.as_ref(), &_guard) {

     

       282
       282
       +
                   Some(handle) => handle.clone(),

     

       283
       283
       +
                   None => {

     

       284
       284
       +
                       if self.ks.partition_exists(nsid.as_ref()) {

     

       285
       285
       +
                           let handle = Arc::new(LexiconHandle::new(&self.ks, nsid.as_ref()));

     

       286
       286
       +
                           let _ = self.hits.insert(SmolStr::new(nsid), handle.clone());

     

       287
       287
       +
                           handle

     

       288
       288
       +
                       } else {

     

       289
       289
       +
                           return None;

     

       290
       290
       +
                       }

     

       291
       291
       +
                   }

     

       292
       292
       +
               };

     

       293
       293
       +
               Some(handle)

     

       294
       294
       +
           }

     

       295
       295
       +
       

     

       296
       296
       +
           #[inline(always)]

     

       297
       297
       +
           fn ensure_handle(&self, nsid: &SmolStr) -> impl Deref<Target = Arc<LexiconHandle>> + use<'_> {

     

       298
       298
       +
               self.hits

     

       299
       299
       +
                   .entry(nsid.clone())

     

       300
       300
       +
                   .or_insert_with(|| Arc::new(LexiconHandle::new(&self.ks, &nsid)))

     

       301
       301
       +
           }

     

       302
       302
       +
       

     

       303
       303
       +
           pub fn ingest_events(&self, events: impl Iterator<Item = EventRecord>) -> AppResult<()> {

     

       304
       304
       +
               for (key, chunk) in events.chunk_by(|event| event.nsid.clone()).into_iter() {

     

       305
       305
       +
                   let mut counts = self.get_count(&key)?;

     

       306
       306
       +
                   let mut count = 0;

     

       307
       307
       +
                   self.ensure_handle(&key).queue(chunk.inspect(|e| {

     

       308
       308
       +
                       // increment count

     

       309
       309
       +
                       counts.last_seen = e.timestamp;

     

       310
       310
       +
                       if e.deleted {

     

       311
       311
       +
                           counts.deleted_count += 1;

     

       312
       312
       +
                       } else {

     

       313
       313
       +
                           counts.count += 1;

     

       314
       314
       +
                       }

     

       315
       315
       +
                       count += 1;

     

       316
       316
       +
                   }));

     

       317
       317
       +
                   self.eps.observe(count);

     

       318
       318
       +
                   self.insert_count(&key, &counts)?;

     

       319
       319
       +
                   if self.event_broadcaster.receiver_count() > 0 {

     

       320
       320
       +
                       let _ = self.event_broadcaster.send((key, counts));

     

       321
       321
       +
                   }

     

       322
       322
       +
               }

     

       323
       323
       +
               Ok(())

     

       324
       324
       +
           }

     

       325
       325
       +
       

     

       326
       326
       +
           #[inline(always)]

     

       327
       327
       +
           fn insert_count(&self, nsid: &str, counts: &NsidCounts) -> AppResult<()> {

     

       328
       328
       +
               self.counts

     

       329
       329
       +
                   .insert(

     

       330
       330
       +
                       nsid,

     

       331
       331
       +
                       unsafe { rkyv::to_bytes::<Error>(counts).unwrap_unchecked() }.as_slice(),

     

       332
       332
       +
                   )

     

       333
       333
       +
                   .map_err(AppError::from)

     

       334
       334
       +
           }

     

       335
       335
       +
       

     

       336
       336
       +
           pub fn get_count(&self, nsid: &str) -> AppResult<NsidCounts> {

     

       337
       337
       +
               let Some(raw) = self.counts.get(nsid)? else {

     

       338
       338
       +
                   return Ok(NsidCounts::default());

     

       339
       339
       +
               };

     

       340
       340
       +
               Ok(unsafe { rkyv::from_bytes_unchecked::<_, Error>(&raw).unwrap_unchecked() })

     

       341
       341
       +
           }

     

       342
       342
       +
       

     

       343
       343
       +
           pub fn get_counts(&self) -> impl Iterator<Item = AppResult<(SmolStr, NsidCounts)>> {

     

       344
       344
       +
               self.counts.iter().map(|res| {

     

       345
       345
       +
                   res.map_err(AppError::from).map(|(key, val)| {

     

       346
       346
       +
                       (

     

       347
       347
       +
                           SmolStr::new(unsafe { str::from_utf8_unchecked(&key) }),

     

       348
       348
       +
                           unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },

     

       349
       349
       +
                       )

     

       350
       350
       +
                   })

     

       351
       351
       +
               })

     

       352
       352
       +
           }

     

       353
       353
       +
       

     

       354
       354
       +
           pub fn get_nsids(&self) -> impl Iterator<Item = StrView> {

     

       355
       355
       +
               self.ks

     

       356
       356
       +
                   .list_partitions()

     

       357
       357
       +
                   .into_iter()

     

       358
       358
       +
                   .filter(|k| k.deref() != "_counts")

     

       359
       359
       +
           }

     

       360
       360
       +
       

     

       361
       361
       +
           pub fn info(&self) -> AppResult<DbInfo> {

     

       362
       362
       +
               let mut nsids = HashMap::new();

     

       363
       363
       +
               for nsid in self.get_nsids() {

     

       364
       364
       +
                   let Some(handle) = self.get_handle(&nsid) else {

     

       365
       365
       +
                       continue;

     

       366
       366
       +
                   };

     

       367
       367
       +
                   let block_lens = handle.iter().rev().try_fold(Vec::new(), |mut acc, item| {

     

       368
       368
       +
                       let (key, value) = item?;

     

       369
       369
       +
                       let mut timestamps = Cursor::new(key);

     

       370
       370
       +
                       let start_timestamp = timestamps.read_varint()?;

     

       371
       371
       +
                       let decoder = ItemDecoder::new(Cursor::new(value), start_timestamp)?;

     

       372
       372
       +
                       acc.push(decoder.item_count());

     

       373
       373
       +
                       AppResult::Ok(acc)

     

       374
       374
       +
                   })?;

     

       375
       375
       +
                   nsids.insert(nsid.to_smolstr(), block_lens);

     

       376
       376
       +
               }

     

       377
       377
       +
               Ok(DbInfo {

     

       378
       378
       +
                   nsids,

     

       379
       379
       +
                   disk_size: self.ks.disk_space(),

     

       380
       380
       +
               })

     

       381
       381
       +
           }

     

       382
       382
       +
       

     

       383
       383
       +
           pub fn get_hits(

     

       384
       384
       +
               &self,

     

       385
       385
       +
               nsid: &str,

     

       386
       386
       +
               range: impl RangeBounds<u64> + std::fmt::Debug,

     

       387
       387
       +
           ) -> impl Iterator<Item = AppResult<handle::Item>> {

     

       388
       388
       +
               let start_limit = match range.start_bound().cloned() {

     

       389
       389
       +
                   Bound::Included(start) => start,

     

       390
       390
       +
                   Bound::Excluded(start) => start.saturating_add(1),

     

       391
       391
       +
                   Bound::Unbounded => 0,

     

       392
       392
       +
               };

     

       393
       393
       +
               let end_limit = match range.end_bound().cloned() {

     

       394
       394
       +
                   Bound::Included(end) => end,

     

       395
       395
       +
                   Bound::Excluded(end) => end.saturating_sub(1),

     

       396
       396
       +
                   Bound::Unbounded => u64::MAX,

     

       397
       397
       +
               };

     

       398
       398
       +
               let end_key = varints_unsigned_encoded([end_limit]);

     

       399
       399
       +
       

     

       400
       400
       +
               let Some(handle) = self.get_handle(nsid) else {

     

       401
       401
       +
                   return Either::Right(std::iter::empty());

     

       402
       402
       +
               };

     

       403
       403
       +
       

     

       404
       404
       +
               let map_block = move |(key, val)| {

     

       405
       405
       +
                   let mut key_reader = Cursor::new(key);

     

       406
       406
       +
                   let start_timestamp = key_reader.read_varint::<u64>()?;

     

       407
       407
       +
                   if start_timestamp < start_limit {

     

       408
       408
       +
                       return Ok(None);

     

       409
       409
       +
                   }

     

       410
       410
       +
                   let items = handle::ItemDecoder::new(Cursor::new(val), start_timestamp)?

     

       411
       411
       +
                       .take_while(move |item| {

     

       412
       412
       +
                           item.as_ref().map_or(true, |item| {

     

       413
       413
       +
                               item.timestamp <= end_limit && item.timestamp >= start_limit

     

       414
       414
       +
                           })

     

       415
       415
       +
                       })

     

       416
       416
       +
                       .map(|res| res.map_err(AppError::from));

     

       417
       417
       +
                   Ok(Some(items))

     

       418
       418
       +
               };

     

       419
       419
       +
       

     

       420
       420
       +
               Either::Left(

     

       421
       421
       +
                   handle

     

       422
       422
       +
                       .range(..end_key)

     

       423
       423
       +
                       .rev()

     

       424
       424
       +
                       .map_while(move |res| res.map_err(AppError::from).and_then(map_block).transpose())

     

       425
       425
       +
                       .collect::<Vec<_>>()

     

       426
       426
       +
                       .into_iter()

     

       427
       427
       +
                       .rev()

     

       428
       428
       +
                       .flatten()

     

       429
       429
       +
                       .flatten(),

     

       430
       430
       +
               )

     

       431
       431
       +
           }

     

       432
       432
       +
       

     

       433
       433
       +
           pub fn tracking_since(&self) -> AppResult<u64> {

     

       434
       434
       +
               // HACK: we should actually store when we started tracking but im lazy

     

       435
       435
       +
               // this should be accurate enough

     

       436
       436
       +
               let Some(handle) = self.get_handle("app.bsky.feed.like") else {

     

       437
       437
       +
                   return Ok(0);

     

       438
       438
       +
               };

     

       439
       439
       +
               let Some((timestamps_raw, _)) = handle.first_key_value()? else {

     

       440
       440
       +
                   return Ok(0);

     

       441
       441
       +
               };

     

       442
       442
       +
               let mut timestamp_reader = Cursor::new(timestamps_raw);

     

       443
       443
       +
               timestamp_reader

     

       444
       444
       +
                   .read_varint::<u64>()

     

       445
       445
       +
                   .map_err(AppError::from)

     

       446
       446
       +
           }

     

       447
       447
       +
       }

-227

server/src/db.rs

···

       1
       1
       -
       use std::{

     

       2
       2
       -
           ops::{Bound, Deref, RangeBounds},

     

       3
       3
       -
           path::Path,

     

       4
       4
       -
           time::Duration,

     

       5
       5
       -
       };

     

       6
       6
       -
       

     

       7
       7
       -
       use fjall::{Config, Keyspace, Partition, PartitionCreateOptions};

     

       8
       8
       -
       use pingora_limits::rate::Rate;

     

       9
       9
       -
       use rkyv::{Archive, Deserialize, Serialize, rancor::Error};

     

       10
       10
       -
       use smol_str::SmolStr;

     

       11
       11
       -
       use tokio::sync::broadcast;

     

       12
       12
       -
       

     

       13
       13
       -
       use crate::{

     

       14
       14
       -
           error::{AppError, AppResult},

     

       15
       15
       -
           jetstream::JetstreamEvent,

     

       16
       16
       -
       };

     

       17
       17
       -
       

     

       18
       18
       -
       #[derive(Clone, Debug, Default, Archive, Deserialize, Serialize, PartialEq)]

     

       19
       19
       -
       #[rkyv(compare(PartialEq), derive(Debug))]

     

       20
       20
       -
       pub struct NsidCounts {

     

       21
       21
       -
           pub count: u128,

     

       22
       22
       -
           pub deleted_count: u128,

     

       23
       23
       -
           pub last_seen: u64,

     

       24
       24
       -
       }

     

       25
       25
       -
       

     

       26
       26
       -
       #[derive(Debug, Default, Archive, Deserialize, Serialize, PartialEq)]

     

       27
       27
       -
       #[rkyv(compare(PartialEq), derive(Debug))]

     

       28
       28
       -
       pub struct NsidHit {

     

       29
       29
       -
           pub deleted: bool,

     

       30
       30
       -
       }

     

       31
       31
       -
       

     

       32
       32
       -
       pub struct EventRecord {

     

       33
       33
       -
           pub nsid: SmolStr,

     

       34
       34
       -
           pub timestamp: u64,

     

       35
       35
       -
           pub deleted: bool,

     

       36
       36
       -
       }

     

       37
       37
       -
       

     

       38
       38
       -
       impl EventRecord {

     

       39
       39
       -
           pub fn from_jetstream(event: JetstreamEvent) -> Option<Self> {

     

       40
       40
       -
               match event {

     

       41
       41
       -
                   JetstreamEvent::Commit {

     

       42
       42
       -
                       time_us, commit, ..

     

       43
       43
       -
                   } => Some(Self {

     

       44
       44
       -
                       nsid: commit.collection.into(),

     

       45
       45
       -
                       timestamp: time_us,

     

       46
       46
       -
                       deleted: false,

     

       47
       47
       -
                   }),

     

       48
       48
       -
                   JetstreamEvent::Delete {

     

       49
       49
       -
                       time_us, commit, ..

     

       50
       50
       -
                   } => Some(Self {

     

       51
       51
       -
                       nsid: commit.collection.into(),

     

       52
       52
       -
                       timestamp: time_us,

     

       53
       53
       -
                       deleted: true,

     

       54
       54
       -
                   }),

     

       55
       55
       -
                   _ => None,

     

       56
       56
       -
               }

     

       57
       57
       -
           }

     

       58
       58
       -
       }

     

       59
       59
       -
       

     

       60
       60
       -
       // counts is nsid -> NsidCounts

     

       61
       61
       -
       // hits is tree per nsid: timestamp -> NsidHit

     

       62
       62
       -
       pub struct Db {

     

       63
       63
       -
           inner: Keyspace,

     

       64
       64
       -
           hits: papaya::HashMap<SmolStr, Partition>,

     

       65
       65
       -
           counts: Partition,

     

       66
       66
       -
           event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,

     

       67
       67
       -
           eps: Rate,

     

       68
       68
       -
       }

     

       69
       69
       -
       

     

       70
       70
       -
       impl Db {

     

       71
       71
       -
           pub fn new(path: impl AsRef<Path>) -> AppResult<Self> {

     

       72
       72
       -
               tracing::info!("opening db...");

     

       73
       73
       -
               let ks = Config::new(path)

     

       74
       74
       -
                   .cache_size(8 * 1024 * 1024) // from talna

     

       75
       75
       -
                   .open()?;

     

       76
       76
       -
               Ok(Self {

     

       77
       77
       -
                   hits: Default::default(),

     

       78
       78
       -
                   counts: ks.open_partition(

     

       79
       79
       -
                       "_counts",

     

       80
       80
       -
                       PartitionCreateOptions::default().compression(fjall::CompressionType::None),

     

       81
       81
       -
                   )?,

     

       82
       82
       -
                   inner: ks,

     

       83
       83
       -
                   event_broadcaster: broadcast::channel(1000).0,

     

       84
       84
       -
                   eps: Rate::new(Duration::from_secs(1)),

     

       85
       85
       -
               })

     

       86
       86
       -
           }

     

       87
       87
       -
       

     

       88
       88
       -
           pub fn eps(&self) -> usize {

     

       89
       89
       -
               self.eps.rate(&()) as usize

     

       90
       90
       -
           }

     

       91
       91
       -
       

     

       92
       92
       -
           pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {

     

       93
       93
       -
               self.event_broadcaster.subscribe()

     

       94
       94
       -
           }

     

       95
       95
       -
       

     

       96
       96
       -
           #[inline(always)]

     

       97
       97
       -
           fn run_in_nsid_tree<T>(

     

       98
       98
       -
               &self,

     

       99
       99
       -
               nsid: &str,

     

       100
       100
       -
               f: impl FnOnce(&Partition) -> AppResult<T>,

     

       101
       101
       -
           ) -> AppResult<T> {

     

       102
       102
       -
               f(self.hits.pin().get_or_insert_with(SmolStr::new(nsid), || {

     

       103
       103
       -
                   let opts = PartitionCreateOptions::default()

     

       104
       104
       -
                       .compression(fjall::CompressionType::Miniz(9))

     

       105
       105
       -
                       .compaction_strategy(fjall::compaction::Strategy::Fifo(fjall::compaction::Fifo {

     

       106
       106
       -
                           limit: 5 * 1024 * 1024 * 1024,        // 5 gb

     

       107
       107
       -
                           ttl_seconds: Some(60 * 60 * 24 * 30), // 30 days

     

       108
       108
       -
                       }));

     

       109
       109
       -
                   self.inner.open_partition(nsid, opts).unwrap()

     

       110
       110
       -
               }))

     

       111
       111
       -
           }

     

       112
       112
       -
       

     

       113
       113
       -
           pub fn record_event(&self, e: EventRecord) -> AppResult<()> {

     

       114
       114
       -
               let EventRecord {

     

       115
       115
       -
                   nsid,

     

       116
       116
       -
                   timestamp,

     

       117
       117
       -
                   deleted,

     

       118
       118
       -
               } = e;

     

       119
       119
       -
       

     

       120
       120
       -
               self.insert_event(&nsid, timestamp, deleted)?;

     

       121
       121
       -
               // increment count

     

       122
       122
       -
               let mut counts = self.get_count(&nsid)?;

     

       123
       123
       -
               counts.last_seen = timestamp;

     

       124
       124
       -
               if deleted {

     

       125
       125
       -
                   counts.deleted_count += 1;

     

       126
       126
       -
               } else {

     

       127
       127
       -
                   counts.count += 1;

     

       128
       128
       -
               }

     

       129
       129
       -
               self.insert_count(&nsid, counts.clone())?;

     

       130
       130
       -
               if self.event_broadcaster.receiver_count() > 0 {

     

       131
       131
       -
                   let _ = self.event_broadcaster.send((SmolStr::new(&nsid), counts));

     

       132
       132
       -
               }

     

       133
       133
       -
               self.eps.observe(&(), 1);

     

       134
       134
       -
               Ok(())

     

       135
       135
       -
           }

     

       136
       136
       -
       

     

       137
       137
       -
           #[inline(always)]

     

       138
       138
       -
           fn insert_event(&self, nsid: &str, timestamp: u64, deleted: bool) -> AppResult<()> {

     

       139
       139
       -
               self.run_in_nsid_tree(nsid, |tree| {

     

       140
       140
       -
                   tree.insert(

     

       141
       141
       -
                       timestamp.to_be_bytes(),

     

       142
       142
       -
                       unsafe { rkyv::to_bytes::<Error>(&NsidHit { deleted }).unwrap_unchecked() }

     

       143
       143
       -
                           .as_slice(),

     

       144
       144
       -
                   )

     

       145
       145
       -
                   .map_err(AppError::from)

     

       146
       146
       -
               })

     

       147
       147
       -
           }

     

       148
       148
       -
       

     

       149
       149
       -
           #[inline(always)]

     

       150
       150
       -
           fn insert_count(&self, nsid: &str, counts: NsidCounts) -> AppResult<()> {

     

       151
       151
       -
               self.counts

     

       152
       152
       -
                   .insert(

     

       153
       153
       -
                       nsid,

     

       154
       154
       -
                       unsafe { rkyv::to_bytes::<Error>(&counts).unwrap_unchecked() }.as_slice(),

     

       155
       155
       -
                   )

     

       156
       156
       -
                   .map_err(AppError::from)

     

       157
       157
       -
           }

     

       158
       158
       -
       

     

       159
       159
       -
           pub fn get_count(&self, nsid: &str) -> AppResult<NsidCounts> {

     

       160
       160
       -
               let Some(raw) = self.counts.get(nsid)? else {

     

       161
       161
       -
                   return Ok(NsidCounts::default());

     

       162
       162
       -
               };

     

       163
       163
       -
               Ok(unsafe { rkyv::from_bytes_unchecked::<_, Error>(&raw).unwrap_unchecked() })

     

       164
       164
       -
           }

     

       165
       165
       -
       

     

       166
       166
       -
           pub fn get_counts(&self) -> impl Iterator<Item = AppResult<(SmolStr, NsidCounts)>> {

     

       167
       167
       -
               self.counts.iter().map(|res| {

     

       168
       168
       -
                   res.map_err(AppError::from).map(|(key, val)| {

     

       169
       169
       -
                       (

     

       170
       170
       -
                           SmolStr::new(unsafe { str::from_utf8_unchecked(&key) }),

     

       171
       171
       -
                           unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },

     

       172
       172
       -
                       )

     

       173
       173
       -
                   })

     

       174
       174
       -
               })

     

       175
       175
       -
           }

     

       176
       176
       -
       

     

       177
       177
       -
           pub fn get_nsids(&self) -> impl Iterator<Item = impl Deref<Target = str>> {

     

       178
       178
       -
               self.inner

     

       179
       179
       -
                   .list_partitions()

     

       180
       180
       -
                   .into_iter()

     

       181
       181
       -
                   .filter(|k| k.deref() != "_counts")

     

       182
       182
       -
           }

     

       183
       183
       -
       

     

       184
       184
       -
           pub fn get_hits(

     

       185
       185
       -
               &self,

     

       186
       186
       -
               nsid: &str,

     

       187
       187
       -
               range: impl RangeBounds<u64>,

     

       188
       188
       -
           ) -> AppResult<Box<dyn Iterator<Item = AppResult<(u64, NsidHit)>>>> {

     

       189
       189
       -
               let start = range.start_bound().cloned().map(u64::to_be_bytes);

     

       190
       190
       -
               let end = range.end_bound().cloned().map(u64::to_be_bytes);

     

       191
       191
       -
       

     

       192
       192
       -
               let _guard = self.hits.guard();

     

       193
       193
       -
               let Some(tree) = self.hits.get(nsid, &_guard) else {

     

       194
       194
       -
                   return Ok(Box::new(std::iter::empty()));

     

       195
       195
       -
               };

     

       196
       196
       -
       

     

       197
       197
       -
               Ok(Box::new(tree.range(TimestampRange { start, end }).map(

     

       198
       198
       -
                   |res| {

     

       199
       199
       -
                       res.map_err(AppError::from).map(|(key, val)| {

     

       200
       200
       -
                           (

     

       201
       201
       -
                               u64::from_be_bytes(key.as_ref().try_into().unwrap()),

     

       202
       202
       -
                               unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },

     

       203
       203
       -
                           )

     

       204
       204
       -
                       })

     

       205
       205
       -
                   },

     

       206
       206
       -
               )))

     

       207
       207
       -
           }

     

       208
       208
       -
       }

     

       209
       209
       -
       

     

       210
       210
       -
       type TimestampRepr = [u8; 8];

     

       211
       211
       -
       

     

       212
       212
       -
       struct TimestampRange {

     

       213
       213
       -
           start: Bound<TimestampRepr>,

     

       214
       214
       -
           end: Bound<TimestampRepr>,

     

       215
       215
       -
       }

     

       216
       216
       -
       

     

       217
       217
       -
       impl RangeBounds<TimestampRepr> for TimestampRange {

     

       218
       218
       -
           #[inline(always)]

     

       219
       219
       -
           fn start_bound(&self) -> Bound<&TimestampRepr> {

     

       220
       220
       -
               self.start.as_ref()

     

       221
       221
       -
           }

     

       222
       222
       -
       

     

       223
       223
       -
           #[inline(always)]

     

       224
       224
       -
           fn end_bound(&self) -> Bound<&TimestampRepr> {

     

       225
       225
       -
               self.end.as_ref()

     

       226
       226
       -
           }

     

       227
       227
       -
       }

+501

server/src/db_old/block.rs

···

       1
       1
       +
       use ordered_varint::Variable;

     

       2
       2
       +
       use rkyv::{

     

       3
       3
       +
           Archive, Deserialize, Serialize,

     

       4
       4
       +
           api::high::{HighSerializer, HighValidator},

     

       5
       5
       +
           bytecheck::CheckBytes,

     

       6
       6
       +
           de::Pool,

     

       7
       7
       +
           rancor::{self, Strategy},

     

       8
       8
       +
           ser::allocator::ArenaHandle,

     

       9
       9
       +
           util::AlignedVec,

     

       10
       10
       +
       };

     

       11
       11
       +
       use std::{

     

       12
       12
       +
           io::{self, Read, Write},

     

       13
       13
       +
           marker::PhantomData,

     

       14
       14
       +
       };

     

       15
       15
       +
       

     

       16
       16
       +
       use crate::error::{AppError, AppResult};

     

       17
       17
       +
       

     

       18
       18
       +
       pub struct Item<T> {

     

       19
       19
       +
           pub timestamp: u64,

     

       20
       20
       +
           data: AlignedVec,

     

       21
       21
       +
           phantom: PhantomData<T>,

     

       22
       22
       +
       }

     

       23
       23
       +
       

     

       24
       24
       +
       impl<T: Archive> Item<T> {

     

       25
       25
       +
           pub fn access(&self) -> &T::Archived {

     

       26
       26
       +
               unsafe { rkyv::access_unchecked::<T::Archived>(&self.data) }

     

       27
       27
       +
           }

     

       28
       28
       +
       }

     

       29
       29
       +
       

     

       30
       30
       +
       impl<T> Item<T>

     

       31
       31
       +
       where

     

       32
       32
       +
           T: Archive,

     

       33
       33
       +
           T::Archived: for<'a> CheckBytes<HighValidator<'a, rancor::Error>>

     

       34
       34
       +
               + Deserialize<T, Strategy<Pool, rancor::Error>>,

     

       35
       35
       +
       {

     

       36
       36
       +
           pub fn deser(&self) -> AppResult<T> {

     

       37
       37
       +
               rkyv::from_bytes(&self.data).map_err(AppError::from)

     

       38
       38
       +
           }

     

       39
       39
       +
       }

     

       40
       40
       +
       

     

       41
       41
       +
       impl<T: for<'a> Serialize<HighSerializer<AlignedVec, ArenaHandle<'a>, rancor::Error>>> Item<T> {

     

       42
       42
       +
           pub fn new(timestamp: u64, data: &T) -> Self {

     

       43
       43
       +
               Item {

     

       44
       44
       +
                   timestamp,

     

       45
       45
       +
                   data: unsafe { rkyv::to_bytes(data).unwrap_unchecked() },

     

       46
       46
       +
                   phantom: PhantomData,

     

       47
       47
       +
               }

     

       48
       48
       +
           }

     

       49
       49
       +
       }

     

       50
       50
       +
       

     

       51
       51
       +
       pub struct ItemEncoder<W: Write, T> {

     

       52
       52
       +
           writer: W,

     

       53
       53
       +
           prev_timestamp: u64,

     

       54
       54
       +
           prev_delta: i64,

     

       55
       55
       +
           _item: PhantomData<T>,

     

       56
       56
       +
       }

     

       57
       57
       +
       

     

       58
       58
       +
       impl<W: Write, T> ItemEncoder<W, T> {

     

       59
       59
       +
           pub fn new(writer: W) -> Self {

     

       60
       60
       +
               ItemEncoder {

     

       61
       61
       +
                   writer,

     

       62
       62
       +
                   prev_timestamp: 0,

     

       63
       63
       +
                   prev_delta: 0,

     

       64
       64
       +
                   _item: PhantomData,

     

       65
       65
       +
               }

     

       66
       66
       +
           }

     

       67
       67
       +
       

     

       68
       68
       +
           pub fn encode(&mut self, item: &Item<T>) -> AppResult<()> {

     

       69
       69
       +
               if self.prev_timestamp == 0 {

     

       70
       70
       +
                   // self.writer.write_varint(item.timestamp)?;

     

       71
       71
       +
                   self.prev_timestamp = item.timestamp;

     

       72
       72
       +
                   self.write_data(&item.data)?;

     

       73
       73
       +
                   return Ok(());

     

       74
       74
       +
               }

     

       75
       75
       +
       

     

       76
       76
       +
               let delta = (item.timestamp as i128 - self.prev_timestamp as i128) as i64;

     

       77
       77
       +
       

     

       78
       78
       +
               self.writer.write_varint(delta - self.prev_delta)?;

     

       79
       79
       +
               self.prev_timestamp = item.timestamp;

     

       80
       80
       +
               self.prev_delta = delta;

     

       81
       81
       +
       

     

       82
       82
       +
               self.write_data(&item.data)?;

     

       83
       83
       +
       

     

       84
       84
       +
               Ok(())

     

       85
       85
       +
           }

     

       86
       86
       +
       

     

       87
       87
       +
           fn write_data(&mut self, data: &[u8]) -> AppResult<()> {

     

       88
       88
       +
               self.writer.write_varint(data.len())?;

     

       89
       89
       +
               self.writer.write_all(data)?;

     

       90
       90
       +
               Ok(())

     

       91
       91
       +
           }

     

       92
       92
       +
       

     

       93
       93
       +
           pub fn finish(mut self) -> AppResult<W> {

     

       94
       94
       +
               self.writer.flush()?;

     

       95
       95
       +
               Ok(self.writer)

     

       96
       96
       +
           }

     

       97
       97
       +
       }

     

       98
       98
       +
       

     

       99
       99
       +
       pub struct ItemDecoder<R, T> {

     

       100
       100
       +
           reader: R,

     

       101
       101
       +
           current_timestamp: u64,

     

       102
       102
       +
           current_delta: i64,

     

       103
       103
       +
           first_item: bool,

     

       104
       104
       +
           _item: PhantomData<T>,

     

       105
       105
       +
       }

     

       106
       106
       +
       

     

       107
       107
       +
       impl<R: Read, T: Archive> ItemDecoder<R, T> {

     

       108
       108
       +
           pub fn new(reader: R, start_timestamp: u64) -> AppResult<Self> {

     

       109
       109
       +
               Ok(ItemDecoder {

     

       110
       110
       +
                   reader,

     

       111
       111
       +
                   current_timestamp: start_timestamp,

     

       112
       112
       +
                   current_delta: 0,

     

       113
       113
       +
                   first_item: true,

     

       114
       114
       +
                   _item: PhantomData,

     

       115
       115
       +
               })

     

       116
       116
       +
           }

     

       117
       117
       +
       

     

       118
       118
       +
           pub fn decode(&mut self) -> AppResult<Option<Item<T>>> {

     

       119
       119
       +
               if self.first_item {

     

       120
       120
       +
                   // read the first timestamp

     

       121
       121
       +
                   // let timestamp = match self.reader.read_varint::<u64>() {

     

       122
       122
       +
                   //     Ok(timestamp) => timestamp,

     

       123
       123
       +
                   //     Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),

     

       124
       124
       +
                   //     Err(e) => return Err(e.into()),

     

       125
       125
       +
                   // };

     

       126
       126
       +
                   // self.current_timestamp = timestamp;

     

       127
       127
       +
       

     

       128
       128
       +
                   let Some(data_raw) = self.read_item()? else {

     

       129
       129
       +
                       return Ok(None);

     

       130
       130
       +
                   };

     

       131
       131
       +
                   self.first_item = false;

     

       132
       132
       +
                   return Ok(Some(Item {

     

       133
       133
       +
                       timestamp: self.current_timestamp,

     

       134
       134
       +
                       data: data_raw,

     

       135
       135
       +
                       phantom: PhantomData,

     

       136
       136
       +
                   }));

     

       137
       137
       +
               }

     

       138
       138
       +
       

     

       139
       139
       +
               let Some(_delta) = self.read_timestamp()? else {

     

       140
       140
       +
                   return Ok(None);

     

       141
       141
       +
               };

     

       142
       142
       +
       

     

       143
       143
       +
               // read data

     

       144
       144
       +
               let data_raw = match self.read_item()? {

     

       145
       145
       +
                   Some(data_raw) => data_raw,

     

       146
       146
       +
                   None => {

     

       147
       147
       +
                       return Err(io::Error::new(

     

       148
       148
       +
                           io::ErrorKind::UnexpectedEof,

     

       149
       149
       +
                           "expected data after delta",

     

       150
       150
       +
                       )

     

       151
       151
       +
                       .into());

     

       152
       152
       +
                   }

     

       153
       153
       +
               };

     

       154
       154
       +
       

     

       155
       155
       +
               Ok(Some(Item {

     

       156
       156
       +
                   timestamp: self.current_timestamp,

     

       157
       157
       +
                   data: data_raw,

     

       158
       158
       +
                   phantom: PhantomData,

     

       159
       159
       +
               }))

     

       160
       160
       +
           }

     

       161
       161
       +
       

     

       162
       162
       +
           // [10, 11, 12, 14] -> [1, 1, 2] -> [0, 1]

     

       163
       163
       +
           fn read_timestamp(&mut self) -> AppResult<Option<u64>> {

     

       164
       164
       +
               let delta = match self.reader.read_varint::<i64>() {

     

       165
       165
       +
                   Ok(delta) => delta,

     

       166
       166
       +
                   Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),

     

       167
       167
       +
                   Err(e) => return Err(e.into()),

     

       168
       168
       +
               };

     

       169
       169
       +
               self.current_delta += delta;

     

       170
       170
       +
               self.current_timestamp =

     

       171
       171
       +
                   (self.current_timestamp as i128 + self.current_delta as i128) as u64;

     

       172
       172
       +
               Ok(Some(self.current_timestamp))

     

       173
       173
       +
           }

     

       174
       174
       +
       

     

       175
       175
       +
           fn read_item(&mut self) -> AppResult<Option<AlignedVec>> {

     

       176
       176
       +
               let data_len = match self.reader.read_varint::<usize>() {

     

       177
       177
       +
                   Ok(data_len) => data_len,

     

       178
       178
       +
                   Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),

     

       179
       179
       +
                   Err(e) => return Err(e.into()),

     

       180
       180
       +
               };

     

       181
       181
       +
               let mut data_raw = AlignedVec::with_capacity(data_len);

     

       182
       182
       +
               for _ in 0..data_len {

     

       183
       183
       +
                   data_raw.push(0);

     

       184
       184
       +
               }

     

       185
       185
       +
               self.reader.read_exact(data_raw.as_mut_slice())?;

     

       186
       186
       +
               Ok(Some(data_raw))

     

       187
       187
       +
           }

     

       188
       188
       +
       }

     

       189
       189
       +
       

     

       190
       190
       +
       impl<R: Read, T: Archive> Iterator for ItemDecoder<R, T> {

     

       191
       191
       +
           type Item = AppResult<Item<T>>;

     

       192
       192
       +
       

     

       193
       193
       +
           fn next(&mut self) -> Option<Self::Item> {

     

       194
       194
       +
               self.decode().transpose()

     

       195
       195
       +
           }

     

       196
       196
       +
       }

     

       197
       197
       +
       

     

       198
       198
       +
       pub trait WriteVariableExt: Write {

     

       199
       199
       +
           fn write_varint(&mut self, value: impl Variable) -> io::Result<usize> {

     

       200
       200
       +
               value.encode_variable(self)

     

       201
       201
       +
           }

     

       202
       202
       +
       }

     

       203
       203
       +
       impl<W: Write> WriteVariableExt for W {}

     

       204
       204
       +
       

     

       205
       205
       +
       pub trait ReadVariableExt: Read {

     

       206
       206
       +
           fn read_varint<T: Variable>(&mut self) -> io::Result<T> {

     

       207
       207
       +
               T::decode_variable(self)

     

       208
       208
       +
           }

     

       209
       209
       +
       }

     

       210
       210
       +
       impl<R: Read> ReadVariableExt for R {}

     

       211
       211
       +
       

     

       212
       212
       +
       #[cfg(test)]

     

       213
       213
       +
       mod test {

     

       214
       214
       +
           use super::*;

     

       215
       215
       +
           use rkyv::{Archive, Deserialize, Serialize};

     

       216
       216
       +
           use std::io::Cursor;

     

       217
       217
       +
       

     

       218
       218
       +
           #[derive(Archive, Deserialize, Serialize, Debug, PartialEq)]

     

       219
       219
       +
           #[rkyv(compare(PartialEq))]

     

       220
       220
       +
           struct TestData {

     

       221
       221
       +
               id: u32,

     

       222
       222
       +
               value: String,

     

       223
       223
       +
           }

     

       224
       224
       +
       

     

       225
       225
       +
           #[test]

     

       226
       226
       +
           fn test_encoder_decoder_single_item() {

     

       227
       227
       +
               let data = TestData {

     

       228
       228
       +
                   id: 123,

     

       229
       229
       +
                   value: "test".to_string(),

     

       230
       230
       +
               };

     

       231
       231
       +
       

     

       232
       232
       +
               let item = Item::new(1000, &data);

     

       233
       233
       +
       

     

       234
       234
       +
               // encode

     

       235
       235
       +
               let mut buffer = Vec::new();

     

       236
       236
       +
               let mut encoder = ItemEncoder::new(&mut buffer);

     

       237
       237
       +
               encoder.encode(&item).unwrap();

     

       238
       238
       +
               encoder.finish().unwrap();

     

       239
       239
       +
       

     

       240
       240
       +
               // decode

     

       241
       241
       +
               let cursor = Cursor::new(buffer);

     

       242
       242
       +
               let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       243
       243
       +
       

     

       244
       244
       +
               let decoded_item = decoder.decode().unwrap().unwrap();

     

       245
       245
       +
               assert_eq!(decoded_item.timestamp, 1000);

     

       246
       246
       +
       

     

       247
       247
       +
               let decoded_data = decoded_item.access();

     

       248
       248
       +
               assert_eq!(decoded_data.id, 123);

     

       249
       249
       +
               assert_eq!(decoded_data.value.as_str(), "test");

     

       250
       250
       +
           }

     

       251
       251
       +
       

     

       252
       252
       +
           #[test]

     

       253
       253
       +
           fn test_encoder_decoder_multiple_items() {

     

       254
       254
       +
               let items = vec![

     

       255
       255
       +
                   Item::new(

     

       256
       256
       +
                       1000,

     

       257
       257
       +
                       &TestData {

     

       258
       258
       +
                           id: 1,

     

       259
       259
       +
                           value: "first".to_string(),

     

       260
       260
       +
                       },

     

       261
       261
       +
                   ),

     

       262
       262
       +
                   Item::new(

     

       263
       263
       +
                       1010,

     

       264
       264
       +
                       &TestData {

     

       265
       265
       +
                           id: 2,

     

       266
       266
       +
                           value: "second".to_string(),

     

       267
       267
       +
                       },

     

       268
       268
       +
                   ),

     

       269
       269
       +
                   Item::new(

     

       270
       270
       +
                       1015,

     

       271
       271
       +
                       &TestData {

     

       272
       272
       +
                           id: 3,

     

       273
       273
       +
                           value: "third".to_string(),

     

       274
       274
       +
                       },

     

       275
       275
       +
                   ),

     

       276
       276
       +
                   Item::new(

     

       277
       277
       +
                       1025,

     

       278
       278
       +
                       &TestData {

     

       279
       279
       +
                           id: 4,

     

       280
       280
       +
                           value: "fourth".to_string(),

     

       281
       281
       +
                       },

     

       282
       282
       +
                   ),

     

       283
       283
       +
               ];

     

       284
       284
       +
       

     

       285
       285
       +
               // encode

     

       286
       286
       +
               let mut buffer = Vec::new();

     

       287
       287
       +
               let mut encoder = ItemEncoder::new(&mut buffer);

     

       288
       288
       +
       

     

       289
       289
       +
               for item in &items {

     

       290
       290
       +
                   encoder.encode(item).unwrap();

     

       291
       291
       +
               }

     

       292
       292
       +
               encoder.finish().unwrap();

     

       293
       293
       +
       

     

       294
       294
       +
               // decode

     

       295
       295
       +
               let cursor = Cursor::new(buffer);

     

       296
       296
       +
               let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       297
       297
       +
       

     

       298
       298
       +
               let mut decoded_items = Vec::new();

     

       299
       299
       +
               while let Some(item) = decoder.decode().unwrap() {

     

       300
       300
       +
                   decoded_items.push(item);

     

       301
       301
       +
               }

     

       302
       302
       +
       

     

       303
       303
       +
               assert_eq!(decoded_items.len(), 4);

     

       304
       304
       +
       

     

       305
       305
       +
               for (original, decoded) in items.iter().zip(decoded_items.iter()) {

     

       306
       306
       +
                   assert_eq!(original.timestamp, decoded.timestamp);

     

       307
       307
       +
                   assert_eq!(original.access().id, decoded.access().id);

     

       308
       308
       +
                   assert_eq!(

     

       309
       309
       +
                       original.access().value.as_str(),

     

       310
       310
       +
                       decoded.access().value.as_str()

     

       311
       311
       +
                   );

     

       312
       312
       +
               }

     

       313
       313
       +
           }

     

       314
       314
       +
       

     

       315
       315
       +
           #[test]

     

       316
       316
       +
           fn test_encoder_decoder_with_iterator() {

     

       317
       317
       +
               let items = vec![

     

       318
       318
       +
                   Item::new(

     

       319
       319
       +
                       2000,

     

       320
       320
       +
                       &TestData {

     

       321
       321
       +
                           id: 10,

     

       322
       322
       +
                           value: "a".to_string(),

     

       323
       323
       +
                       },

     

       324
       324
       +
                   ),

     

       325
       325
       +
                   Item::new(

     

       326
       326
       +
                       2005,

     

       327
       327
       +
                       &TestData {

     

       328
       328
       +
                           id: 20,

     

       329
       329
       +
                           value: "b".to_string(),

     

       330
       330
       +
                       },

     

       331
       331
       +
                   ),

     

       332
       332
       +
                   Item::new(

     

       333
       333
       +
                       2012,

     

       334
       334
       +
                       &TestData {

     

       335
       335
       +
                           id: 30,

     

       336
       336
       +
                           value: "c".to_string(),

     

       337
       337
       +
                       },

     

       338
       338
       +
                   ),

     

       339
       339
       +
               ];

     

       340
       340
       +
       

     

       341
       341
       +
               // encode

     

       342
       342
       +
               let mut buffer = Vec::new();

     

       343
       343
       +
               let mut encoder = ItemEncoder::new(&mut buffer);

     

       344
       344
       +
       

     

       345
       345
       +
               for item in &items {

     

       346
       346
       +
                   encoder.encode(item).unwrap();

     

       347
       347
       +
               }

     

       348
       348
       +
               encoder.finish().unwrap();

     

       349
       349
       +
       

     

       350
       350
       +
               // decode

     

       351
       351
       +
               let cursor = Cursor::new(buffer);

     

       352
       352
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 2000).unwrap();

     

       353
       353
       +
       

     

       354
       354
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       355
       355
       +
               let decoded_items = decoded_items.unwrap();

     

       356
       356
       +
       

     

       357
       357
       +
               assert_eq!(decoded_items.len(), 3);

     

       358
       358
       +
               assert_eq!(decoded_items[0].timestamp, 2000);

     

       359
       359
       +
               assert_eq!(decoded_items[1].timestamp, 2005);

     

       360
       360
       +
               assert_eq!(decoded_items[2].timestamp, 2012);

     

       361
       361
       +
       

     

       362
       362
       +
               assert_eq!(decoded_items[0].access().id, 10);

     

       363
       363
       +
               assert_eq!(decoded_items[1].access().id, 20);

     

       364
       364
       +
               assert_eq!(decoded_items[2].access().id, 30);

     

       365
       365
       +
           }

     

       366
       366
       +
       

     

       367
       367
       +
           #[test]

     

       368
       368
       +
           fn test_delta_compression() {

     

       369
       369
       +
               let items = vec![

     

       370
       370
       +
                   Item::new(

     

       371
       371
       +
                       1000,

     

       372
       372
       +
                       &TestData {

     

       373
       373
       +
                           id: 1,

     

       374
       374
       +
                           value: "a".to_string(),

     

       375
       375
       +
                       },

     

       376
       376
       +
                   ),

     

       377
       377
       +
                   Item::new(

     

       378
       378
       +
                       1010,

     

       379
       379
       +
                       &TestData {

     

       380
       380
       +
                           id: 2,

     

       381
       381
       +
                           value: "b".to_string(),

     

       382
       382
       +
                       },

     

       383
       383
       +
                   ), // delta = 10

     

       384
       384
       +
                   Item::new(

     

       385
       385
       +
                       1020,

     

       386
       386
       +
                       &TestData {

     

       387
       387
       +
                           id: 3,

     

       388
       388
       +
                           value: "c".to_string(),

     

       389
       389
       +
                       },

     

       390
       390
       +
                   ), // delta = 10, delta-of-delta = 0

     

       391
       391
       +
                   Item::new(

     

       392
       392
       +
                       1025,

     

       393
       393
       +
                       &TestData {

     

       394
       394
       +
                           id: 4,

     

       395
       395
       +
                           value: "d".to_string(),

     

       396
       396
       +
                       },

     

       397
       397
       +
                   ), // delta = 5, delta-of-delta = -5

     

       398
       398
       +
               ];

     

       399
       399
       +
       

     

       400
       400
       +
               let mut buffer = Vec::new();

     

       401
       401
       +
               let mut encoder = ItemEncoder::new(&mut buffer);

     

       402
       402
       +
       

     

       403
       403
       +
               for item in &items {

     

       404
       404
       +
                   encoder.encode(item).unwrap();

     

       405
       405
       +
               }

     

       406
       406
       +
               encoder.finish().unwrap();

     

       407
       407
       +
       

     

       408
       408
       +
               // decode and verify

     

       409
       409
       +
               let cursor = Cursor::new(buffer);

     

       410
       410
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       411
       411
       +
       

     

       412
       412
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       413
       413
       +
               let decoded_items = decoded_items.unwrap();

     

       414
       414
       +
       

     

       415
       415
       +
               for (original, decoded) in items.iter().zip(decoded_items.iter()) {

     

       416
       416
       +
                   assert_eq!(original.timestamp, decoded.timestamp);

     

       417
       417
       +
                   assert_eq!(original.access().id, decoded.access().id);

     

       418
       418
       +
               }

     

       419
       419
       +
           }

     

       420
       420
       +
       

     

       421
       421
       +
           #[test]

     

       422
       422
       +
           fn test_empty_decode() {

     

       423
       423
       +
               let buffer = Vec::new();

     

       424
       424
       +
               let cursor = Cursor::new(buffer);

     

       425
       425
       +
               let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       426
       426
       +
       

     

       427
       427
       +
               let result = decoder.decode().unwrap();

     

       428
       428
       +
               assert!(result.is_none());

     

       429
       429
       +
           }

     

       430
       430
       +
       

     

       431
       431
       +
           #[test]

     

       432
       432
       +
           fn test_backwards_timestamp() {

     

       433
       433
       +
               let items = vec![

     

       434
       434
       +
                   Item::new(

     

       435
       435
       +
                       1000,

     

       436
       436
       +
                       &TestData {

     

       437
       437
       +
                           id: 1,

     

       438
       438
       +
                           value: "first".to_string(),

     

       439
       439
       +
                       },

     

       440
       440
       +
                   ),

     

       441
       441
       +
                   Item::new(

     

       442
       442
       +
                       900,

     

       443
       443
       +
                       &TestData {

     

       444
       444
       +
                           id: 2,

     

       445
       445
       +
                           value: "second".to_string(),

     

       446
       446
       +
                       },

     

       447
       447
       +
                   ),

     

       448
       448
       +
               ];

     

       449
       449
       +
       

     

       450
       450
       +
               let mut buffer = Vec::new();

     

       451
       451
       +
               let mut encoder = ItemEncoder::new(&mut buffer);

     

       452
       452
       +
       

     

       453
       453
       +
               for item in &items {

     

       454
       454
       +
                   encoder.encode(item).unwrap();

     

       455
       455
       +
               }

     

       456
       456
       +
               encoder.finish().unwrap();

     

       457
       457
       +
       

     

       458
       458
       +
               let cursor = Cursor::new(buffer);

     

       459
       459
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       460
       460
       +
       

     

       461
       461
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       462
       462
       +
               let decoded_items = decoded_items.unwrap();

     

       463
       463
       +
       

     

       464
       464
       +
               assert_eq!(decoded_items.len(), 2);

     

       465
       465
       +
               assert_eq!(decoded_items[0].timestamp, 1000);

     

       466
       466
       +
               assert_eq!(decoded_items[1].timestamp, 900);

     

       467
       467
       +
           }

     

       468
       468
       +
       

     

       469
       469
       +
           #[test]

     

       470
       470
       +
           fn test_different_data_sizes() {

     

       471
       471
       +
               let small_data = TestData {

     

       472
       472
       +
                   id: 1,

     

       473
       473
       +
                   value: "x".to_string(),

     

       474
       474
       +
               };

     

       475
       475
       +
               let large_data = TestData {

     

       476
       476
       +
                   id: 2,

     

       477
       477
       +
                   value: "a".repeat(1000),

     

       478
       478
       +
               };

     

       479
       479
       +
       

     

       480
       480
       +
               let items = vec![Item::new(1000, &small_data), Item::new(1001, &large_data)];

     

       481
       481
       +
       

     

       482
       482
       +
               let mut buffer = Vec::new();

     

       483
       483
       +
               let mut encoder = ItemEncoder::new(&mut buffer);

     

       484
       484
       +
       

     

       485
       485
       +
               for item in &items {

     

       486
       486
       +
                   encoder.encode(item).unwrap();

     

       487
       487
       +
               }

     

       488
       488
       +
               encoder.finish().unwrap();

     

       489
       489
       +
       

     

       490
       490
       +
               let cursor = Cursor::new(buffer);

     

       491
       491
       +
               let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();

     

       492
       492
       +
       

     

       493
       493
       +
               let decoded_items: Result<Vec<_>, _> = decoder.collect();

     

       494
       494
       +
               let decoded_items = decoded_items.unwrap();

     

       495
       495
       +
       

     

       496
       496
       +
               assert_eq!(decoded_items.len(), 2);

     

       497
       497
       +
               assert_eq!(decoded_items[0].access().value.as_str(), "x");

     

       498
       498
       +
               assert_eq!(decoded_items[1].access().value.len(), 1000);

     

       499
       499
       +
               assert_eq!(decoded_items[1].access().value.as_str(), "a".repeat(1000));

     

       500
       500
       +
           }

     

       501
       501
       +
       }

+424

server/src/db_old/mod.rs

···

       1
       1
       +
       use std::{

     

       2
       2
       +
           io::Cursor,

     

       3
       3
       +
           ops::{Bound, Deref, RangeBounds},

     

       4
       4
       +
           path::Path,

     

       5
       5
       +
           sync::{

     

       6
       6
       +
               Arc,

     

       7
       7
       +
               atomic::{AtomicU64, AtomicUsize, Ordering as AtomicOrdering},

     

       8
       8
       +
           },

     

       9
       9
       +
           time::{Duration, Instant},

     

       10
       10
       +
       };

     

       11
       11
       +
       

     

       12
       12
       +
       use fjall::{Config, Keyspace, Partition, PartitionCreateOptions, Slice};

     

       13
       13
       +
       use ordered_varint::Variable;

     

       14
       14
       +
       use rkyv::{Archive, Deserialize, Serialize, rancor::Error};

     

       15
       15
       +
       use smol_str::SmolStr;

     

       16
       16
       +
       use tokio::sync::broadcast;

     

       17
       17
       +
       

     

       18
       18
       +
       use crate::{

     

       19
       19
       +
           db_old::block::{ReadVariableExt, WriteVariableExt},

     

       20
       20
       +
           error::{AppError, AppResult},

     

       21
       21
       +
           jetstream::JetstreamEvent,

     

       22
       22
       +
           utils::{DefaultRateTracker, get_time},

     

       23
       23
       +
       };

     

       24
       24
       +
       

     

       25
       25
       +
       mod block;

     

       26
       26
       +
       

     

       27
       27
       +
       #[derive(Clone, Debug, Default, Archive, Deserialize, Serialize, PartialEq)]

     

       28
       28
       +
       #[rkyv(compare(PartialEq), derive(Debug))]

     

       29
       29
       +
       pub struct NsidCounts {

     

       30
       30
       +
           pub count: u128,

     

       31
       31
       +
           pub deleted_count: u128,

     

       32
       32
       +
           pub last_seen: u64,

     

       33
       33
       +
       }

     

       34
       34
       +
       

     

       35
       35
       +
       #[derive(Debug, Default, Archive, Deserialize, Serialize, PartialEq)]

     

       36
       36
       +
       #[rkyv(compare(PartialEq), derive(Debug))]

     

       37
       37
       +
       pub struct NsidHit {

     

       38
       38
       +
           pub deleted: bool,

     

       39
       39
       +
       }

     

       40
       40
       +
       

     

       41
       41
       +
       #[derive(Clone)]

     

       42
       42
       +
       pub struct EventRecord {

     

       43
       43
       +
           pub nsid: SmolStr,

     

       44
       44
       +
           pub timestamp: u64, // seconds

     

       45
       45
       +
           pub deleted: bool,

     

       46
       46
       +
       }

     

       47
       47
       +
       

     

       48
       48
       +
       impl EventRecord {

     

       49
       49
       +
           pub fn from_jetstream(event: JetstreamEvent) -> Option<Self> {

     

       50
       50
       +
               match event {

     

       51
       51
       +
                   JetstreamEvent::Commit {

     

       52
       52
       +
                       time_us, commit, ..

     

       53
       53
       +
                   } => Some(Self {

     

       54
       54
       +
                       nsid: commit.collection.into(),

     

       55
       55
       +
                       timestamp: time_us / 1_000_000,

     

       56
       56
       +
                       deleted: false,

     

       57
       57
       +
                   }),

     

       58
       58
       +
                   JetstreamEvent::Delete {

     

       59
       59
       +
                       time_us, commit, ..

     

       60
       60
       +
                   } => Some(Self {

     

       61
       61
       +
                       nsid: commit.collection.into(),

     

       62
       62
       +
                       timestamp: time_us / 1_000_000,

     

       63
       63
       +
                       deleted: true,

     

       64
       64
       +
                   }),

     

       65
       65
       +
                   _ => None,

     

       66
       66
       +
               }

     

       67
       67
       +
           }

     

       68
       68
       +
       }

     

       69
       69
       +
       

     

       70
       70
       +
       type ItemDecoder = block::ItemDecoder<Cursor<Slice>, NsidHit>;

     

       71
       71
       +
       type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;

     

       72
       72
       +
       type Item = block::Item<NsidHit>;

     

       73
       73
       +
       

     

       74
       74
       +
       pub struct LexiconHandle {

     

       75
       75
       +
           tree: Partition,

     

       76
       76
       +
           buf: Arc<scc::Queue<EventRecord>>,

     

       77
       77
       +
           buf_len: AtomicUsize,

     

       78
       78
       +
           last_insert: AtomicU64,

     

       79
       79
       +
           eps: DefaultRateTracker,

     

       80
       80
       +
           block_size: AtomicUsize,

     

       81
       81
       +
       }

     

       82
       82
       +
       

     

       83
       83
       +
       impl LexiconHandle {

     

       84
       84
       +
           fn new(keyspace: &Keyspace, nsid: &str) -> Self {

     

       85
       85
       +
               let opts = PartitionCreateOptions::default().compression(fjall::CompressionType::Miniz(9));

     

       86
       86
       +
               Self {

     

       87
       87
       +
                   tree: keyspace.open_partition(nsid, opts).unwrap(),

     

       88
       88
       +
                   buf: Default::default(),

     

       89
       89
       +
                   buf_len: AtomicUsize::new(0),

     

       90
       90
       +
                   last_insert: AtomicU64::new(0),

     

       91
       91
       +
                   eps: DefaultRateTracker::new(Duration::from_secs(5)),

     

       92
       92
       +
                   block_size: AtomicUsize::new(1000),

     

       93
       93
       +
               }

     

       94
       94
       +
           }

     

       95
       95
       +
       

     

       96
       96
       +
           fn item_count(&self) -> usize {

     

       97
       97
       +
               self.buf_len.load(AtomicOrdering::Acquire)

     

       98
       98
       +
           }

     

       99
       99
       +
       

     

       100
       100
       +
           fn last_insert(&self) -> u64 {

     

       101
       101
       +
               self.last_insert.load(AtomicOrdering::Acquire)

     

       102
       102
       +
           }

     

       103
       103
       +
       

     

       104
       104
       +
           fn suggested_block_size(&self) -> usize {

     

       105
       105
       +
               self.block_size.load(AtomicOrdering::Relaxed)

     

       106
       106
       +
           }

     

       107
       107
       +
       

     

       108
       108
       +
           fn insert(&self, event: EventRecord) {

     

       109
       109
       +
               self.buf.push(event);

     

       110
       110
       +
               self.buf_len.fetch_add(1, AtomicOrdering::Release);

     

       111
       111
       +
               self.last_insert

     

       112
       112
       +
                   .store(get_time().as_millis() as u64, AtomicOrdering::Release);

     

       113
       113
       +
               self.eps.observe(1);

     

       114
       114
       +
               let rate = self.eps.rate() as usize;

     

       115
       115
       +
               if rate != 0 {

     

       116
       116
       +
                   self.block_size.store(rate * 60, AtomicOrdering::Relaxed);

     

       117
       117
       +
               }

     

       118
       118
       +
           }

     

       119
       119
       +
       

     

       120
       120
       +
           fn sync(&self, max_block_size: usize) -> AppResult<usize> {

     

       121
       121
       +
               let mut writer = ItemEncoder::new(Vec::with_capacity(

     

       122
       122
       +
                   size_of::<u64>() + self.item_count().min(max_block_size) * size_of::<(u64, NsidHit)>(),

     

       123
       123
       +
               ));

     

       124
       124
       +
               let mut start_timestamp = None;

     

       125
       125
       +
               let mut end_timestamp = None;

     

       126
       126
       +
               let mut written = 0_usize;

     

       127
       127
       +
               while let Some(event) = self.buf.pop() {

     

       128
       128
       +
                   let item = Item::new(

     

       129
       129
       +
                       event.timestamp,

     

       130
       130
       +
                       &NsidHit {

     

       131
       131
       +
                           deleted: event.deleted,

     

       132
       132
       +
                       },

     

       133
       133
       +
                   );

     

       134
       134
       +
                   writer.encode(&item)?;

     

       135
       135
       +
                   if start_timestamp.is_none() {

     

       136
       136
       +
                       start_timestamp = Some(event.timestamp);

     

       137
       137
       +
                   }

     

       138
       138
       +
                   end_timestamp = Some(event.timestamp);

     

       139
       139
       +
                   if written >= max_block_size {

     

       140
       140
       +
                       break;

     

       141
       141
       +
                   }

     

       142
       142
       +
                   written += 1;

     

       143
       143
       +
               }

     

       144
       144
       +
               if let (Some(start_timestamp), Some(end_timestamp)) = (start_timestamp, end_timestamp) {

     

       145
       145
       +
                   self.buf_len.store(0, AtomicOrdering::Release);

     

       146
       146
       +
                   let value = writer.finish()?;

     

       147
       147
       +
                   let mut key = Vec::with_capacity(size_of::<u64>() * 2);

     

       148
       148
       +
                   key.write_varint(start_timestamp)?;

     

       149
       149
       +
                   key.write_varint(end_timestamp)?;

     

       150
       150
       +
                   self.tree.insert(key, value)?;

     

       151
       151
       +
               }

     

       152
       152
       +
               Ok(written)

     

       153
       153
       +
           }

     

       154
       154
       +
       }

     

       155
       155
       +
       

     

       156
       156
       +
       type BoxedIter<T> = Box<dyn Iterator<Item = T>>;

     

       157
       157
       +
       

     

       158
       158
       +
       // counts is nsid -> NsidCounts

     

       159
       159
       +
       // hits is tree per nsid: varint start time + varint end time -> block of hits

     

       160
       160
       +
       pub struct Db {

     

       161
       161
       +
           inner: Keyspace,

     

       162
       162
       +
           hits: scc::HashIndex<SmolStr, Arc<LexiconHandle>>,

     

       163
       163
       +
           counts: Partition,

     

       164
       164
       +
           event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,

     

       165
       165
       +
           eps: DefaultRateTracker,

     

       166
       166
       +
           min_block_size: usize,

     

       167
       167
       +
           max_block_size: usize,

     

       168
       168
       +
           max_last_activity: Duration,

     

       169
       169
       +
       }

     

       170
       170
       +
       

     

       171
       171
       +
       impl Db {

     

       172
       172
       +
           pub fn new(path: impl AsRef<Path>) -> AppResult<Self> {

     

       173
       173
       +
               tracing::info!("opening db...");

     

       174
       174
       +
               let ks = Config::new(path)

     

       175
       175
       +
                   .cache_size(8 * 1024 * 1024) // from talna

     

       176
       176
       +
                   .open()?;

     

       177
       177
       +
               Ok(Self {

     

       178
       178
       +
                   hits: Default::default(),

     

       179
       179
       +
                   counts: ks.open_partition(

     

       180
       180
       +
                       "_counts",

     

       181
       181
       +
                       PartitionCreateOptions::default().compression(fjall::CompressionType::None),

     

       182
       182
       +
                   )?,

     

       183
       183
       +
                   inner: ks,

     

       184
       184
       +
                   event_broadcaster: broadcast::channel(1000).0,

     

       185
       185
       +
                   eps: DefaultRateTracker::new(Duration::from_secs(1)),

     

       186
       186
       +
                   min_block_size: 512,

     

       187
       187
       +
                   max_block_size: 100_000,

     

       188
       188
       +
                   max_last_activity: Duration::from_secs(10),

     

       189
       189
       +
               })

     

       190
       190
       +
           }

     

       191
       191
       +
       

     

       192
       192
       +
           pub fn sync(&self, all: bool) -> AppResult<()> {

     

       193
       193
       +
               let _guard = scc::ebr::Guard::new();

     

       194
       194
       +
               for (nsid, tree) in self.hits.iter(&_guard) {

     

       195
       195
       +
                   let count = tree.item_count();

     

       196
       196
       +
                   let is_max_block_size = count > self.min_block_size.max(tree.suggested_block_size());

     

       197
       197
       +
                   let is_too_old = (get_time().as_millis() as u64 - tree.last_insert())

     

       198
       198
       +
                       > self.max_last_activity.as_millis() as u64;

     

       199
       199
       +
                   if count > 0 && (all || is_max_block_size || is_too_old) {

     

       200
       200
       +
                       loop {

     

       201
       201
       +
                           let synced = tree.sync(self.max_block_size)?;

     

       202
       202
       +
                           if synced == 0 {

     

       203
       203
       +
                               break;

     

       204
       204
       +
                           }

     

       205
       205
       +
                           tracing::info!("synced {synced} of {nsid} to db");

     

       206
       206
       +
                       }

     

       207
       207
       +
                   }

     

       208
       208
       +
               }

     

       209
       209
       +
               Ok(())

     

       210
       210
       +
           }

     

       211
       211
       +
       

     

       212
       212
       +
           #[inline(always)]

     

       213
       213
       +
           pub fn eps(&self) -> usize {

     

       214
       214
       +
               self.eps.rate() as usize

     

       215
       215
       +
           }

     

       216
       216
       +
       

     

       217
       217
       +
           #[inline(always)]

     

       218
       218
       +
           pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {

     

       219
       219
       +
               self.event_broadcaster.subscribe()

     

       220
       220
       +
           }

     

       221
       221
       +
       

     

       222
       222
       +
           #[inline(always)]

     

       223
       223
       +
           fn maybe_run_in_nsid_tree<T>(

     

       224
       224
       +
               &self,

     

       225
       225
       +
               nsid: &str,

     

       226
       226
       +
               f: impl FnOnce(&LexiconHandle) -> T,

     

       227
       227
       +
           ) -> Option<T> {

     

       228
       228
       +
               let _guard = scc::ebr::Guard::new();

     

       229
       229
       +
               let handle = match self.hits.peek(nsid, &_guard) {

     

       230
       230
       +
                   Some(handle) => handle.clone(),

     

       231
       231
       +
                   None => {

     

       232
       232
       +
                       if self.inner.partition_exists(nsid) {

     

       233
       233
       +
                           let handle = Arc::new(LexiconHandle::new(&self.inner, nsid));

     

       234
       234
       +
                           let _ = self.hits.insert(SmolStr::new(nsid), handle.clone());

     

       235
       235
       +
                           handle

     

       236
       236
       +
                       } else {

     

       237
       237
       +
                           return None;

     

       238
       238
       +
                       }

     

       239
       239
       +
                   }

     

       240
       240
       +
               };

     

       241
       241
       +
               Some(f(&handle))

     

       242
       242
       +
           }

     

       243
       243
       +
       

     

       244
       244
       +
           #[inline(always)]

     

       245
       245
       +
           fn run_in_nsid_tree<T>(

     

       246
       246
       +
               &self,

     

       247
       247
       +
               nsid: SmolStr,

     

       248
       248
       +
               f: impl FnOnce(&LexiconHandle) -> AppResult<T>,

     

       249
       249
       +
           ) -> AppResult<T> {

     

       250
       250
       +
               f(self

     

       251
       251
       +
                   .hits

     

       252
       252
       +
                   .entry(nsid.clone())

     

       253
       253
       +
                   .or_insert_with(move || Arc::new(LexiconHandle::new(&self.inner, &nsid)))

     

       254
       254
       +
                   .get())

     

       255
       255
       +
           }

     

       256
       256
       +
       

     

       257
       257
       +
           pub fn record_event(&self, e: EventRecord) -> AppResult<()> {

     

       258
       258
       +
               let EventRecord {

     

       259
       259
       +
                   nsid,

     

       260
       260
       +
                   timestamp,

     

       261
       261
       +
                   deleted,

     

       262
       262
       +
               } = e.clone();

     

       263
       263
       +
       

     

       264
       264
       +
               // insert event

     

       265
       265
       +
               self.run_in_nsid_tree(nsid.clone(), move |tree| Ok(tree.insert(e)))?;

     

       266
       266
       +
               // increment count

     

       267
       267
       +
               let mut counts = self.get_count(&nsid)?;

     

       268
       268
       +
               counts.last_seen = timestamp;

     

       269
       269
       +
               if deleted {

     

       270
       270
       +
                   counts.deleted_count += 1;

     

       271
       271
       +
               } else {

     

       272
       272
       +
                   counts.count += 1;

     

       273
       273
       +
               }

     

       274
       274
       +
               self.insert_count(&nsid, counts.clone())?;

     

       275
       275
       +
               if self.event_broadcaster.receiver_count() > 0 {

     

       276
       276
       +
                   let _ = self.event_broadcaster.send((SmolStr::new(&nsid), counts));

     

       277
       277
       +
               }

     

       278
       278
       +
               self.eps.observe(1);

     

       279
       279
       +
               Ok(())

     

       280
       280
       +
           }

     

       281
       281
       +
       

     

       282
       282
       +
           #[inline(always)]

     

       283
       283
       +
           fn insert_count(&self, nsid: &str, counts: NsidCounts) -> AppResult<()> {

     

       284
       284
       +
               self.counts

     

       285
       285
       +
                   .insert(

     

       286
       286
       +
                       nsid,

     

       287
       287
       +
                       unsafe { rkyv::to_bytes::<Error>(&counts).unwrap_unchecked() }.as_slice(),

     

       288
       288
       +
                   )

     

       289
       289
       +
                   .map_err(AppError::from)

     

       290
       290
       +
           }

     

       291
       291
       +
       

     

       292
       292
       +
           pub fn get_count(&self, nsid: &str) -> AppResult<NsidCounts> {

     

       293
       293
       +
               let Some(raw) = self.counts.get(nsid)? else {

     

       294
       294
       +
                   return Ok(NsidCounts::default());

     

       295
       295
       +
               };

     

       296
       296
       +
               Ok(unsafe { rkyv::from_bytes_unchecked::<_, Error>(&raw).unwrap_unchecked() })

     

       297
       297
       +
           }

     

       298
       298
       +
       

     

       299
       299
       +
           pub fn get_counts(&self) -> impl Iterator<Item = AppResult<(SmolStr, NsidCounts)>> {

     

       300
       300
       +
               self.counts.iter().map(|res| {

     

       301
       301
       +
                   res.map_err(AppError::from).map(|(key, val)| {

     

       302
       302
       +
                       (

     

       303
       303
       +
                           SmolStr::new(unsafe { str::from_utf8_unchecked(&key) }),

     

       304
       304
       +
                           unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },

     

       305
       305
       +
                       )

     

       306
       306
       +
                   })

     

       307
       307
       +
               })

     

       308
       308
       +
           }

     

       309
       309
       +
       

     

       310
       310
       +
           pub fn get_nsids(&self) -> impl Iterator<Item = impl Deref<Target = str> + 'static> {

     

       311
       311
       +
               self.inner

     

       312
       312
       +
                   .list_partitions()

     

       313
       313
       +
                   .into_iter()

     

       314
       314
       +
                   .filter(|k| k.deref() != "_counts")

     

       315
       315
       +
           }

     

       316
       316
       +
       

     

       317
       317
       +
           pub fn get_hits_debug(&self, nsid: &str) -> BoxedIter<AppResult<(Slice, Slice)>> {

     

       318
       318
       +
               self.maybe_run_in_nsid_tree(nsid, |handle| -> BoxedIter<AppResult<(Slice, Slice)>> {

     

       319
       319
       +
                   Box::new(

     

       320
       320
       +
                       handle

     

       321
       321
       +
                           .tree

     

       322
       322
       +
                           .iter()

     

       323
       323
       +
                           .rev()

     

       324
       324
       +
                           .map(|res| res.map_err(AppError::from)),

     

       325
       325
       +
                   )

     

       326
       326
       +
               })

     

       327
       327
       +
               .unwrap_or_else(|| Box::new(std::iter::empty()))

     

       328
       328
       +
           }

     

       329
       329
       +
       

     

       330
       330
       +
           pub fn get_hits(

     

       331
       331
       +
               &self,

     

       332
       332
       +
               nsid: &str,

     

       333
       333
       +
               range: impl RangeBounds<u64> + std::fmt::Debug,

     

       334
       334
       +
           ) -> BoxedIter<AppResult<Item>> {

     

       335
       335
       +
               let start = range

     

       336
       336
       +
                   .start_bound()

     

       337
       337
       +
                   .cloned()

     

       338
       338
       +
                   .map(|t| unsafe { t.to_variable_vec().unwrap_unchecked() });

     

       339
       339
       +
               let end = range

     

       340
       340
       +
                   .end_bound()

     

       341
       341
       +
                   .cloned()

     

       342
       342
       +
                   .map(|t| unsafe { t.to_variable_vec().unwrap_unchecked() });

     

       343
       343
       +
               let limit = match range.end_bound().cloned() {

     

       344
       344
       +
                   Bound::Included(end) => end,

     

       345
       345
       +
                   Bound::Excluded(end) => end.saturating_sub(1),

     

       346
       346
       +
                   Bound::Unbounded => u64::MAX,

     

       347
       347
       +
               };

     

       348
       348
       +
       

     

       349
       349
       +
               self.maybe_run_in_nsid_tree(nsid, move |handle| -> BoxedIter<AppResult<Item>> {

     

       350
       350
       +
                   let map_block = move |(key, val)| {

     

       351
       351
       +
                       let mut key_reader = Cursor::new(key);

     

       352
       352
       +
                       let start_timestamp = key_reader.read_varint::<u64>()?;

     

       353
       353
       +
                       let items =

     

       354
       354
       +
                           ItemDecoder::new(Cursor::new(val), start_timestamp)?.take_while(move |item| {

     

       355
       355
       +
                               item.as_ref().map_or(true, |item| item.timestamp <= limit)

     

       356
       356
       +
                           });

     

       357
       357
       +
                       Ok(items)

     

       358
       358
       +
                   };

     

       359
       359
       +
       

     

       360
       360
       +
                   Box::new(

     

       361
       361
       +
                       handle

     

       362
       362
       +
                           .tree

     

       363
       363
       +
                           .range(TimestampRange { start, end })

     

       364
       364
       +
                           .map(move |res| res.map_err(AppError::from).and_then(map_block))

     

       365
       365
       +
                           .flatten()

     

       366
       366
       +
                           .flatten(),

     

       367
       367
       +
                   )

     

       368
       368
       +
               })

     

       369
       369
       +
               .unwrap_or_else(|| Box::new(std::iter::empty()))

     

       370
       370
       +
           }

     

       371
       371
       +
       

     

       372
       372
       +
           pub fn tracking_since(&self) -> AppResult<u64> {

     

       373
       373
       +
               // HACK: we should actually store when we started tracking but im lazy

     

       374
       374
       +
               // should be accurate enough

     

       375
       375
       +
               self.maybe_run_in_nsid_tree("app.bsky.feed.like", |handle| {

     

       376
       376
       +
                   let Some((timestamps_raw, _)) = handle.tree.first_key_value()? else {

     

       377
       377
       +
                       return Ok(0);

     

       378
       378
       +
                   };

     

       379
       379
       +
                   let mut timestamp_reader = Cursor::new(timestamps_raw);

     

       380
       380
       +
                   timestamp_reader

     

       381
       381
       +
                       .read_varint::<u64>()

     

       382
       382
       +
                       .map_err(AppError::from)

     

       383
       383
       +
               })

     

       384
       384
       +
               .unwrap_or(Ok(0))

     

       385
       385
       +
           }

     

       386
       386
       +
       }

     

       387
       387
       +
       

     

       388
       388
       +
       type TimestampRepr = Vec<u8>;

     

       389
       389
       +
       

     

       390
       390
       +
       struct TimestampRange {

     

       391
       391
       +
           start: Bound<TimestampRepr>,

     

       392
       392
       +
           end: Bound<TimestampRepr>,

     

       393
       393
       +
       }

     

       394
       394
       +
       

     

       395
       395
       +
       impl RangeBounds<TimestampRepr> for TimestampRange {

     

       396
       396
       +
           #[inline(always)]

     

       397
       397
       +
           fn start_bound(&self) -> Bound<&TimestampRepr> {

     

       398
       398
       +
               self.start.as_ref()

     

       399
       399
       +
           }

     

       400
       400
       +
       

     

       401
       401
       +
           #[inline(always)]

     

       402
       402
       +
           fn end_bound(&self) -> Bound<&TimestampRepr> {

     

       403
       403
       +
               self.end.as_ref()

     

       404
       404
       +
           }

     

       405
       405
       +
       }

     

       406
       406
       +
       

     

       407
       407
       +
       type TimestampReprOld = [u8; 8];

     

       408
       408
       +
       

     

       409
       409
       +
       struct TimestampRangeOld {

     

       410
       410
       +
           start: Bound<TimestampReprOld>,

     

       411
       411
       +
           end: Bound<TimestampReprOld>,

     

       412
       412
       +
       }

     

       413
       413
       +
       

     

       414
       414
       +
       impl RangeBounds<TimestampReprOld> for TimestampRangeOld {

     

       415
       415
       +
           #[inline(always)]

     

       416
       416
       +
           fn start_bound(&self) -> Bound<&TimestampReprOld> {

     

       417
       417
       +
               self.start.as_ref()

     

       418
       418
       +
           }

     

       419
       419
       +
       

     

       420
       420
       +
           #[inline(always)]

     

       421
       421
       +
           fn end_bound(&self) -> Bound<&TimestampReprOld> {

     

       422
       422
       +
               self.end.as_ref()

     

       423
       423
       +
           }

     

       424
       424
       +
       }

+230 -40

server/src/main.rs

···

       1
       1
       -
       use std::{ops::Deref, sync::Arc};

     

       1
       1
       +
       use std::{ops::Deref, time::Duration, u64};

     

       2
       2
        
       

     

       3
       3
       +
       use itertools::Itertools;

     

       4
       4
       +
       use rclite::Arc;

     

       3
       5
        
       use smol_str::ToSmolStr;

     

       4
       4
       -
       #[cfg(not(target_env = "msvc"))]

     

       5
       5
       -
       use tikv_jemallocator::Jemalloc;

     

       6
       6
        
       use tokio_util::sync::CancellationToken;

     

       7
       7
        
       use tracing::Level;

     

       8
       8
        
       use tracing_subscriber::EnvFilter;

     

       9
       9
        
       

     

       10
       10
        
       use crate::{

     

       11
       11
        
           api::serve,

     

       12
       12
       -
           db::{Db, EventRecord},

     

       12
       12
       +
           db::{Db, DbConfig, EventRecord},

     

       13
       13
        
           error::AppError,

     

       14
       14
        
           jetstream::JetstreamClient,

     

       15
       15
       +
           utils::{CLOCK, RelativeDateTime, get_time},

     

       15
       16
        
       };

     

       16
       17
        
       

     

       17
       18
        
       mod api;

     

       18
       19
        
       mod db;

     

       20
       20
       +
       mod db_old;

     

       19
       21
        
       mod error;

     

       20
       22
        
       mod jetstream;

     

       23
       23
       +
       mod utils;

     

       21
       24
        
       

     

       22
       25
        
       #[cfg(not(target_env = "msvc"))]

     

       23
       26
        
       #[global_allocator]

     

       24
       24
       -
       static GLOBAL: Jemalloc = Jemalloc;

     

       27
       27
       +
       static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

     

       28
       28
       +
       

     

       29
       29
       +
       #[cfg(target_env = "msvc")]

     

       30
       30
       +
       #[global_allocator]

     

       31
       31
       +
       static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;

     

       25
       32
        
       

     

       26
       33
        
       #[tokio::main]

     

       27
       34
        
       async fn main() {

     
···

       34
       41
        
               .compact()

     

       35
       42
        
               .init();

     

       36
       43
        
       

     

       37
       37
       -
           if std::env::args()

     

       38
       38
       -
               .nth(1)

     

       39
       39
       -
               .map_or(false, |arg| arg == "migrate")

     

       40
       40
       -
           {

     

       41
       41
       -
               migrate_to_miniz();

     

       42
       42
       -
               return;

     

       44
       44
       +
           match std::env::args().nth(1).as_deref() {

     

       45
       45
       +
               Some("compact") => {

     

       46
       46
       +
                   compact();

     

       47
       47
       +
                   return;

     

       48
       48
       +
               }

     

       49
       49
       +
               Some("migrate") => {

     

       50
       50
       +
                   migrate();

     

       51
       51
       +
                   return;

     

       52
       52
       +
               }

     

       53
       53
       +
               Some("debug") => {

     

       54
       54
       +
                   debug();

     

       55
       55
       +
                   return;

     

       56
       56
       +
               }

     

       57
       57
       +
               Some(x) => {

     

       58
       58
       +
                   tracing::error!("unknown command: {}", x);

     

       59
       59
       +
                   return;

     

       60
       60
       +
               }

     

       61
       61
       +
               None => {}

     

       43
       62
        
           }

     

       44
       63
        
       

     

       45
       45
       -
           let db = Arc::new(Db::new(".fjall_data").expect("couldnt create db"));

     

       64
       64
       +
           let cancel_token = CancellationToken::new();

     

       65
       65
       +
       

     

       66
       66
       +
           let db = Arc::new(

     

       67
       67
       +
               Db::new(DbConfig::default(), cancel_token.child_token()).expect("couldnt create db"),

     

       68
       68
       +
           );

     

       46
       69
        
       

     

       47
       70
        
           rustls::crypto::ring::default_provider()

     

       48
       71
        
               .install_default()

     
···

       57
       80
        
                   }

     

       58
       81
        
               };

     

       59
       82
        
       

     

       60
       60
       -
           let cancel_token = CancellationToken::new();

     

       61
       83
        
           let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(1000);

     

       62
       62
       -
       

     

       63
       84
        
           let consume_events = tokio::spawn({

     

       64
       85
        
               let consume_cancel = cancel_token.child_token();

     

       65
       86
        
               async move {

     
···

       71
       92
        
                                   let Some(record) = EventRecord::from_jetstream(event) else {

     

       72
       93
        
                                       continue;

     

       73
       94
        
                                   };

     

       74
       74
       -
                                   let _ = event_tx.send(record).await;

     

       95
       95
       +
                                   event_tx.send(record).await?;

     

       75
       96
        
                               }

     

       76
       97
        
                               Err(err) => return Err(err),

     

       77
       98
        
                           },

     
···

       84
       105
        
           let ingest_events = std::thread::spawn({

     

       85
       106
        
               let db = db.clone();

     

       86
       107
        
               move || {

     

       87
       87
       -
                   tracing::info!("starting ingest events thread...");

     

       88
       88
       -
                   while let Some(e) = event_rx.blocking_recv() {

     

       89
       89
       -
                       if let Err(e) = db.record_event(e) {

     

       90
       90
       -
                           tracing::error!("failed to record event: {}", e);

     

       108
       108
       +
                   let mut buffer = Vec::new();

     

       109
       109
       +
                   loop {

     

       110
       110
       +
                       let read = event_rx.blocking_recv_many(&mut buffer, 100);

     

       111
       111
       +
                       if let Err(err) = db.ingest_events(buffer.drain(..)) {

     

       112
       112
       +
                           tracing::error!("failed to ingest events: {}", err);

     

       113
       113
       +
                       }

     

       114
       114
       +
                       if read == 0 || db.is_shutting_down() {

     

       115
       115
       +
                           break;

     

       116
       116
       +
                       }

     

       117
       117
       +
                   }

     

       118
       118
       +
               }

     

       119
       119
       +
           });

     

       120
       120
       +
       

     

       121
       121
       +
           let db_task = tokio::task::spawn({

     

       122
       122
       +
               let db = db.clone();

     

       123
       123
       +
               async move {

     

       124
       124
       +
                   let sync_period = Duration::from_secs(10);

     

       125
       125
       +
                   let mut sync_interval = tokio::time::interval(sync_period);

     

       126
       126
       +
                   sync_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);

     

       127
       127
       +
       

     

       128
       128
       +
                   let compact_period = std::time::Duration::from_secs(60 * 30); // 30 mins

     

       129
       129
       +
                   let mut compact_interval = tokio::time::interval(compact_period);

     

       130
       130
       +
                   compact_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);

     

       131
       131
       +
       

     

       132
       132
       +
                   loop {

     

       133
       133
       +
                       let sync_db = async || {

     

       134
       134
       +
                           tokio::task::spawn_blocking({

     

       135
       135
       +
                               let db = db.clone();

     

       136
       136
       +
                               move || {

     

       137
       137
       +
                                   if db.is_shutting_down() {

     

       138
       138
       +
                                       return;

     

       139
       139
       +
                                   }

     

       140
       140
       +
                                   match db.sync(false) {

     

       141
       141
       +
                                       Ok(_) => (),

     

       142
       142
       +
                                       Err(e) => tracing::error!("failed to sync db: {}", e),

     

       143
       143
       +
                                   }

     

       144
       144
       +
                               }

     

       145
       145
       +
                           })

     

       146
       146
       +
                           .await

     

       147
       147
       +
                           .unwrap();

     

       148
       148
       +
                       };

     

       149
       149
       +
                       let compact_db = async || {

     

       150
       150
       +
                           tokio::task::spawn_blocking({

     

       151
       151
       +
                               let db = db.clone();

     

       152
       152
       +
                               move || {

     

       153
       153
       +
                                   if db.is_shutting_down() {

     

       154
       154
       +
                                       return;

     

       155
       155
       +
                                   }

     

       156
       156
       +
                                   let end = get_time() - compact_period / 2;

     

       157
       157
       +
                                   let start = end - compact_period;

     

       158
       158
       +
                                   let range = start.as_secs()..end.as_secs();

     

       159
       159
       +
                                   tracing::info!(

     

       160
       160
       +
                                       {

     

       161
       161
       +
                                           start = %RelativeDateTime::from_now(start),

     

       162
       162
       +
                                           end = %RelativeDateTime::from_now(end),

     

       163
       163
       +
                                       },

     

       164
       164
       +
                                       "running compaction...",

     

       165
       165
       +
                                   );

     

       166
       166
       +
                                   match db.compact_all(db.cfg.max_block_size, range, false) {

     

       167
       167
       +
                                       Ok(_) => (),

     

       168
       168
       +
                                       Err(e) => tracing::error!("failed to compact db: {}", e),

     

       169
       169
       +
                                   }

     

       170
       170
       +
                               }

     

       171
       171
       +
                           })

     

       172
       172
       +
                           .await

     

       173
       173
       +
                           .unwrap();

     

       174
       174
       +
                       };

     

       175
       175
       +
                       tokio::select! {

     

       176
       176
       +
                           _ = sync_interval.tick() => sync_db().await,

     

       177
       177
       +
                           _ = compact_interval.tick() => compact_db().await,

     

       178
       178
       +
                           _ = db.shutting_down() => break,

     

       91
       179
        
                       }

     

       92
       180
        
                   }

     

       93
       181
        
               }

     

       94
       182
        
           });

     

       95
       183
        
       

     

       96
       184
        
           tokio::select! {

     

       97
       97
       -
               res = serve(db, cancel_token.child_token()) => {

     

       185
       185
       +
               res = serve(db.clone(), cancel_token.child_token()) => {

     

       98
       186
        
                   if let Err(e) = res {

     

       99
       187
        
                       tracing::error!("serve failed: {}", e);

     

       100
       188
        
                   }

     
···

       114
       202
        
           }

     

       115
       203
        
       

     

       116
       204
        
           tracing::info!("shutting down...");

     

       117
       117
       -
           ingest_events

     

       118
       118
       -
               .join()

     

       119
       119
       -
               .expect("failed to join ingest events thread");

     

       205
       205
       +
           cancel_token.cancel();

     

       206
       206
       +
           ingest_events.join().expect("failed to join ingest events");

     

       207
       207
       +
           db_task.await.expect("cant join db task");

     

       208
       208
       +
           db.sync(true).expect("cant sync db");

     

       120
       209
        
       }

     

       121
       210
        
       

     

       122
       122
       -
       fn migrate_to_miniz() {

     

       123
       123
       -
           let from = Db::new(".fjall_data").expect("couldnt create db");

     

       124
       124
       -
           let to = Db::new(".fjall_data_miniz").expect("couldnt create db");

     

       211
       211
       +
       fn debug() {

     

       212
       212
       +
           let db = Db::new(DbConfig::default(), CancellationToken::new()).expect("couldnt create db");

     

       213
       213
       +
           let info = db.info().expect("cant get db info");

     

       214
       214
       +
           println!("disk size: {}", info.disk_size);

     

       215
       215
       +
           for (nsid, blocks) in info.nsids {

     

       216
       216
       +
               print!("{nsid}:");

     

       217
       217
       +
               let mut last_size = 0;

     

       218
       218
       +
               let mut same_size_count = 0;

     

       219
       219
       +
               for item_count in blocks {

     

       220
       220
       +
                   if item_count == last_size {

     

       221
       221
       +
                       same_size_count += 1;

     

       222
       222
       +
                   } else {

     

       223
       223
       +
                       if same_size_count > 1 {

     

       224
       224
       +
                           print!("x{}", same_size_count);

     

       225
       225
       +
                       }

     

       226
       226
       +
                       print!(" {item_count}");

     

       227
       227
       +
                       same_size_count = 0;

     

       228
       228
       +
                   }

     

       229
       229
       +
                   last_size = item_count;

     

       230
       230
       +
               }

     

       231
       231
       +
               print!("\n");

     

       232
       232
       +
           }

     

       233
       233
       +
       }

     

       234
       234
       +
       

     

       235
       235
       +
       fn compact() {

     

       236
       236
       +
           let db = Db::new(

     

       237
       237
       +
               DbConfig::default().ks(|c| {

     

       238
       238
       +
                   c.max_journaling_size(u64::MAX)

     

       239
       239
       +
                       .max_write_buffer_size(u64::MAX)

     

       240
       240
       +
               }),

     

       241
       241
       +
               CancellationToken::new(),

     

       242
       242
       +
           )

     

       243
       243
       +
           .expect("couldnt create db");

     

       244
       244
       +
           let info = db.info().expect("cant get db info");

     

       245
       245
       +
           db.major_compact().expect("cant compact");

     

       246
       246
       +
           std::thread::sleep(Duration::from_secs(5));

     

       247
       247
       +
           let compacted_info = db.info().expect("cant get db info");

     

       248
       248
       +
           println!(

     

       249
       249
       +
               "disk size: {} -> {}",

     

       250
       250
       +
               info.disk_size, compacted_info.disk_size

     

       251
       251
       +
           );

     

       252
       252
       +
           for (nsid, blocks) in info.nsids {

     

       253
       253
       +
               println!(

     

       254
       254
       +
                   "{nsid}: {} -> {}",

     

       255
       255
       +
                   blocks.len(),

     

       256
       256
       +
                   compacted_info.nsids[&nsid].len()

     

       257
       257
       +
               )

     

       258
       258
       +
           }

     

       259
       259
       +
       }

     

       260
       260
       +
       

     

       261
       261
       +
       fn migrate() {

     

       262
       262
       +
           let cancel_token = CancellationToken::new();

     

       263
       263
       +
       

     

       264
       264
       +
           let from = Arc::new(db_old::Db::new(".fjall_data_from").expect("couldnt create db"));

     

       265
       265
       +
       

     

       266
       266
       +
           let to = Arc::new(

     

       267
       267
       +
               Db::new(

     

       268
       268
       +
                   DbConfig::default().path(".fjall_data_to").ks(|c| {

     

       269
       269
       +
                       c.max_journaling_size(u64::MAX)

     

       270
       270
       +
                           .max_write_buffer_size(u64::MAX)

     

       271
       271
       +
                           .compaction_workers(rayon::current_num_threads() * 4)

     

       272
       272
       +
                           .flush_workers(rayon::current_num_threads() * 4)

     

       273
       273
       +
                   }),

     

       274
       274
       +
                   cancel_token.child_token(),

     

       275
       275
       +
               )

     

       276
       276
       +
               .expect("couldnt create db"),

     

       277
       277
       +
           );

     

       125
       278
        
       

     

       279
       279
       +
           let nsids = from.get_nsids().collect::<Vec<_>>();

     

       280
       280
       +
           let eps_thread = std::thread::spawn({

     

       281
       281
       +
               let to = to.clone();

     

       282
       282
       +
               move || {

     

       283
       283
       +
                   loop {

     

       284
       284
       +
                       std::thread::sleep(Duration::from_secs(3));

     

       285
       285
       +
                       let eps = to.eps();

     

       286
       286
       +
                       if eps > 0 {

     

       287
       287
       +
                           tracing::info!("{} rps", eps);

     

       288
       288
       +
                       }

     

       289
       289
       +
                   }

     

       290
       290
       +
               }

     

       291
       291
       +
           });

     

       292
       292
       +
           let mut threads = Vec::with_capacity(nsids.len());

     

       293
       293
       +
           let start = CLOCK.now();

     

       294
       294
       +
           for nsid in nsids {

     

       295
       295
       +
               let from = from.clone();

     

       296
       296
       +
               let to = to.clone();

     

       297
       297
       +
               threads.push(std::thread::spawn(move || {

     

       298
       298
       +
                   tracing::info!("{}: migrating...", nsid.deref());

     

       299
       299
       +
                   let mut count = 0_u64;

     

       300
       300
       +
                   for hits in from.get_hits(&nsid, ..).chunks(100000).into_iter() {

     

       301
       301
       +
                       to.ingest_events(hits.map(|hit| {

     

       302
       302
       +
                           count += 1;

     

       303
       303
       +
                           let hit = hit.expect("cant decode hit");

     

       304
       304
       +
                           EventRecord {

     

       305
       305
       +
                               nsid: nsid.to_smolstr(),

     

       306
       306
       +
                               timestamp: hit.timestamp,

     

       307
       307
       +
                               deleted: hit.deser().unwrap().deleted,

     

       308
       308
       +
                           }

     

       309
       309
       +
                       }))

     

       310
       310
       +
                       .expect("cant record event");

     

       311
       311
       +
                   }

     

       312
       312
       +
                   tracing::info!("{}: ingested {} events...", nsid.deref(), count);

     

       313
       313
       +
                   count

     

       314
       314
       +
               }));

     

       315
       315
       +
           }

     

       126
       316
        
           let mut total_count = 0_u64;

     

       127
       127
       -
           for nsid in from.get_nsids() {

     

       128
       128
       -
               tracing::info!("migrating {} ...", nsid.deref());

     

       129
       129
       -
               for hit in from.get_hits(&nsid, ..).expect("cant read hits") {

     

       130
       130
       -
                   let (timestamp, data) = hit.expect("cant read event");

     

       131
       131
       -
                   to.record_event(EventRecord {

     

       132
       132
       -
                       nsid: nsid.to_smolstr(),

     

       133
       133
       -
                       timestamp,

     

       134
       134
       -
                       deleted: data.deleted,

     

       135
       135
       -
                   })

     

       136
       136
       -
                   .expect("cant record event");

     

       137
       137
       -
                   total_count += 1;

     

       138
       138
       -
               }

     

       317
       317
       +
           for thread in threads {

     

       318
       318
       +
               let count = thread.join().expect("thread panicked");

     

       319
       319
       +
               total_count += count;

     

       139
       320
        
           }

     

       140
       140
       -
       

     

       141
       141
       -
           tracing::info!("migrated {total_count} events!");

     

       321
       321
       +
           let read_time = start.elapsed();

     

       322
       322
       +
           let read_per_second = total_count as f64 / read_time.as_secs_f64();

     

       323
       323
       +
           drop(from);

     

       324
       324
       +
           tracing::info!("starting sync!!!");

     

       325
       325
       +
           to.sync(true).expect("cant sync");

     

       326
       326
       +
           tracing::info!("persisting...");

     

       327
       327
       +
           let total_time = start.elapsed();

     

       328
       328
       +
           let write_per_second = total_count as f64 / (total_time - read_time).as_secs_f64();

     

       329
       329
       +
           tracing::info!(

     

       330
       330
       +
               "migrated {total_count} events in {total_time:?} ({read_per_second:.2} rps, {write_per_second:.2} wps)"

     

       331
       331
       +
           );

     

       142
       332
        
       }

+322

server/src/utils.rs

···

       1
       1
       +
       use std::io::{self, Read, Write};

     

       2
       2
       +
       use std::sync::atomic::{AtomicU64, Ordering};

     

       3
       3
       +
       use std::time::Duration;

     

       4
       4
       +
       

     

       5
       5
       +
       use byteview::ByteView;

     

       6
       6
       +
       use ordered_varint::Variable;

     

       7
       7
       +
       

     

       8
       8
       +
       pub fn get_time() -> Duration {

     

       9
       9
       +
           std::time::SystemTime::now()

     

       10
       10
       +
               .duration_since(std::time::UNIX_EPOCH)

     

       11
       11
       +
               .unwrap()

     

       12
       12
       +
       }

     

       13
       13
       +
       

     

       14
       14
       +
       pub trait WriteVariableExt: Write {

     

       15
       15
       +
           fn write_varint(&mut self, value: impl Variable) -> io::Result<usize> {

     

       16
       16
       +
               value.encode_variable(self)

     

       17
       17
       +
           }

     

       18
       18
       +
       }

     

       19
       19
       +
       impl<W: Write> WriteVariableExt for W {}

     

       20
       20
       +
       

     

       21
       21
       +
       pub trait ReadVariableExt: Read {

     

       22
       22
       +
           fn read_varint<T: Variable>(&mut self) -> io::Result<T> {

     

       23
       23
       +
               T::decode_variable(self)

     

       24
       24
       +
           }

     

       25
       25
       +
       }

     

       26
       26
       +
       impl<R: Read> ReadVariableExt for R {}

     

       27
       27
       +
       

     

       28
       28
       +
       pub struct WritableByteView {

     

       29
       29
       +
           view: ByteView,

     

       30
       30
       +
           written: usize,

     

       31
       31
       +
       }

     

       32
       32
       +
       

     

       33
       33
       +
       impl WritableByteView {

     

       34
       34
       +
           // returns None if the view already has a reference to it

     

       35
       35
       +
           pub fn with_size(capacity: usize) -> Self {

     

       36
       36
       +
               Self {

     

       37
       37
       +
                   view: ByteView::with_size(capacity),

     

       38
       38
       +
                   written: 0,

     

       39
       39
       +
               }

     

       40
       40
       +
           }

     

       41
       41
       +
       

     

       42
       42
       +
           #[inline(always)]

     

       43
       43
       +
           pub fn into_inner(self) -> ByteView {

     

       44
       44
       +
               self.view

     

       45
       45
       +
           }

     

       46
       46
       +
       }

     

       47
       47
       +
       

     

       48
       48
       +
       impl Write for WritableByteView {

     

       49
       49
       +
           fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {

     

       50
       50
       +
               let len = buf.len();

     

       51
       51
       +
               if len > self.view.len() - self.written {

     

       52
       52
       +
                   return Err(std::io::Error::new(

     

       53
       53
       +
                       std::io::ErrorKind::StorageFull,

     

       54
       54
       +
                       "buffer full",

     

       55
       55
       +
                   ));

     

       56
       56
       +
               }

     

       57
       57
       +
               // SAFETY: this is safe because we have checked that the buffer is not full

     

       58
       58
       +
               // SAFETY: we own the mutator so no other references to the view exist

     

       59
       59
       +
               unsafe {

     

       60
       60
       +
                   std::ptr::copy_nonoverlapping(

     

       61
       61
       +
                       buf.as_ptr(),

     

       62
       62
       +
                       self.view

     

       63
       63
       +
                           .get_mut()

     

       64
       64
       +
                           .unwrap_unchecked()

     

       65
       65
       +
                           .as_mut_ptr()

     

       66
       66
       +
                           .add(self.written),

     

       67
       67
       +
                       len,

     

       68
       68
       +
                   );

     

       69
       69
       +
                   self.written += len;

     

       70
       70
       +
               }

     

       71
       71
       +
               Ok(len)

     

       72
       72
       +
           }

     

       73
       73
       +
       

     

       74
       74
       +
           #[inline(always)]

     

       75
       75
       +
           fn flush(&mut self) -> std::io::Result<()> {

     

       76
       76
       +
               Ok(())

     

       77
       77
       +
           }

     

       78
       78
       +
       }

     

       79
       79
       +
       

     

       80
       80
       +
       pub fn varints_unsigned_encoded<const N: usize>(values: [u64; N]) -> ByteView {

     

       81
       81
       +
           let mut buf =

     

       82
       82
       +
               WritableByteView::with_size(values.into_iter().map(varint_unsigned_encoded_len).sum());

     

       83
       83
       +
           for value in values {

     

       84
       84
       +
               // cant fail

     

       85
       85
       +
               let _ = buf.write_varint(value);

     

       86
       86
       +
           }

     

       87
       87
       +
           buf.into_inner()

     

       88
       88
       +
       }

     

       89
       89
       +
       

     

       90
       90
       +
       // gets the encoded length of a varint-encoded unsigned integer

     

       91
       91
       +
       // see ordered_varint

     

       92
       92
       +
       pub fn varint_unsigned_encoded_len(value: u64) -> usize {

     

       93
       93
       +
           let value = value.to_be_bytes();

     

       94
       94
       +
           value

     

       95
       95
       +
               .iter()

     

       96
       96
       +
               .enumerate()

     

       97
       97
       +
               .find_map(|(index, &byte)| {

     

       98
       98
       +
                   (byte > 0).then(|| {

     

       99
       99
       +
                       let extra_bytes = 7 - index;

     

       100
       100
       +
                       (byte < 16)

     

       101
       101
       +
                           .then(|| extra_bytes + 1)

     

       102
       102
       +
                           .unwrap_or_else(|| extra_bytes + 2)

     

       103
       103
       +
                   })

     

       104
       104
       +
               })

     

       105
       105
       +
               .unwrap_or(0)

     

       106
       106
       +
               .max(1)

     

       107
       107
       +
       }

     

       108
       108
       +
       

     

       109
       109
       +
       pub static CLOCK: std::sync::LazyLock<quanta::Clock> =

     

       110
       110
       +
           std::sync::LazyLock::new(|| quanta::Clock::new());

     

       111
       111
       +
       

     

       112
       112
       +
       /// simple thread-safe rate tracker using time buckets

     

       113
       113
       +
       /// divides time into fixed buckets and rotates through them

     

       114
       114
       +
       #[derive(Debug)]

     

       115
       115
       +
       pub struct RateTracker<const BUCKET_WINDOW: u64> {

     

       116
       116
       +
           buckets: Vec<AtomicU64>,

     

       117
       117
       +
           last_bucket_time: AtomicU64,

     

       118
       118
       +
           bucket_duration_nanos: u64,

     

       119
       119
       +
           window_duration: Duration,

     

       120
       120
       +
           start_time: u64, // raw time when tracker was created

     

       121
       121
       +
       }

     

       122
       122
       +
       

     

       123
       123
       +
       pub type DefaultRateTracker = RateTracker<1000>;

     

       124
       124
       +
       

     

       125
       125
       +
       impl<const BUCKET_WINDOW: u64> RateTracker<BUCKET_WINDOW> {

     

       126
       126
       +
           /// create a new rate tracker with the specified time window

     

       127
       127
       +
           pub fn new(window_duration: Duration) -> Self {

     

       128
       128
       +
               let bucket_duration_nanos = Duration::from_millis(BUCKET_WINDOW).as_nanos() as u64;

     

       129
       129
       +
               let num_buckets =

     

       130
       130
       +
                   (window_duration.as_nanos() as u64 / bucket_duration_nanos).max(1) as usize;

     

       131
       131
       +
       

     

       132
       132
       +
               let mut buckets = Vec::with_capacity(num_buckets);

     

       133
       133
       +
               for _ in 0..num_buckets {

     

       134
       134
       +
                   buckets.push(AtomicU64::new(0));

     

       135
       135
       +
               }

     

       136
       136
       +
       

     

       137
       137
       +
               let start_time = CLOCK.raw();

     

       138
       138
       +
               Self {

     

       139
       139
       +
                   buckets,

     

       140
       140
       +
                   bucket_duration_nanos,

     

       141
       141
       +
                   window_duration,

     

       142
       142
       +
                   last_bucket_time: AtomicU64::new(0),

     

       143
       143
       +
                   start_time,

     

       144
       144
       +
               }

     

       145
       145
       +
           }

     

       146
       146
       +
       

     

       147
       147
       +
           #[inline(always)]

     

       148
       148
       +
           fn elapsed(&self) -> u64 {

     

       149
       149
       +
               CLOCK.delta_as_nanos(self.start_time, CLOCK.raw())

     

       150
       150
       +
           }

     

       151
       151
       +
       

     

       152
       152
       +
           /// record an event

     

       153
       153
       +
           pub fn observe(&self, count: u64) {

     

       154
       154
       +
               self.maybe_advance_buckets();

     

       155
       155
       +
       

     

       156
       156
       +
               let bucket_index = self.get_current_bucket_index();

     

       157
       157
       +
               self.buckets[bucket_index].fetch_add(count, Ordering::Relaxed);

     

       158
       158
       +
           }

     

       159
       159
       +
       

     

       160
       160
       +
           /// get the current rate in events per second

     

       161
       161
       +
           pub fn rate(&self) -> f64 {

     

       162
       162
       +
               self.maybe_advance_buckets();

     

       163
       163
       +
       

     

       164
       164
       +
               let total_events: u64 = self

     

       165
       165
       +
                   .buckets

     

       166
       166
       +
                   .iter()

     

       167
       167
       +
                   .map(|bucket| bucket.load(Ordering::Relaxed))

     

       168
       168
       +
                   .sum();

     

       169
       169
       +
       

     

       170
       170
       +
               total_events as f64 / self.window_duration.as_secs_f64()

     

       171
       171
       +
           }

     

       172
       172
       +
       

     

       173
       173
       +
           fn get_current_bucket_index(&self) -> usize {

     

       174
       174
       +
               let bucket_number = self.elapsed() / self.bucket_duration_nanos;

     

       175
       175
       +
               (bucket_number as usize) % self.buckets.len()

     

       176
       176
       +
           }

     

       177
       177
       +
       

     

       178
       178
       +
           fn maybe_advance_buckets(&self) {

     

       179
       179
       +
               let current_bucket_time =

     

       180
       180
       +
                   (self.elapsed() / self.bucket_duration_nanos) * self.bucket_duration_nanos;

     

       181
       181
       +
               let last_bucket_time = self.last_bucket_time.load(Ordering::Relaxed);

     

       182
       182
       +
       

     

       183
       183
       +
               if current_bucket_time > last_bucket_time {

     

       184
       184
       +
                   // try to update the last bucket time

     

       185
       185
       +
                   if self

     

       186
       186
       +
                       .last_bucket_time

     

       187
       187
       +
                       .compare_exchange_weak(

     

       188
       188
       +
                           last_bucket_time,

     

       189
       189
       +
                           current_bucket_time,

     

       190
       190
       +
                           Ordering::Relaxed,

     

       191
       191
       +
                           Ordering::Relaxed,

     

       192
       192
       +
                       )

     

       193
       193
       +
                       .is_ok()

     

       194
       194
       +
                   {

     

       195
       195
       +
                       // clear buckets that are now too old

     

       196
       196
       +
                       let buckets_to_advance = ((current_bucket_time - last_bucket_time)

     

       197
       197
       +
                           / self.bucket_duration_nanos)

     

       198
       198
       +
                           .min(self.buckets.len() as u64);

     

       199
       199
       +
       

     

       200
       200
       +
                       for i in 0..buckets_to_advance {

     

       201
       201
       +
                           let bucket_time = last_bucket_time + (i + 1) * self.bucket_duration_nanos;

     

       202
       202
       +
                           let bucket_index =

     

       203
       203
       +
                               (bucket_time / self.bucket_duration_nanos) as usize % self.buckets.len();

     

       204
       204
       +
                           self.buckets[bucket_index].store(0, Ordering::Relaxed);

     

       205
       205
       +
                       }

     

       206
       206
       +
                   }

     

       207
       207
       +
               }

     

       208
       208
       +
           }

     

       209
       209
       +
       }

     

       210
       210
       +
       

     

       211
       211
       +
       #[cfg(test)]

     

       212
       212
       +
       mod tests {

     

       213
       213
       +
           use super::*;

     

       214
       214
       +
           use std::sync::Arc;

     

       215
       215
       +
           use std::thread;

     

       216
       216
       +
       

     

       217
       217
       +
           #[test]

     

       218
       218
       +
           fn test_rate_tracker_basic() {

     

       219
       219
       +
               let tracker = DefaultRateTracker::new(Duration::from_secs(2));

     

       220
       220
       +
       

     

       221
       221
       +
               // record some events

     

       222
       222
       +
               tracker.observe(3);

     

       223
       223
       +
       

     

       224
       224
       +
               let rate = tracker.rate();

     

       225
       225
       +
               assert_eq!(rate, 1.5); // 3 events over 2 seconds = 1.5 events/sec

     

       226
       226
       +
           }

     

       227
       227
       +
       

     

       228
       228
       +
           #[test]

     

       229
       229
       +
           fn test_rate_tracker_burst() {

     

       230
       230
       +
               let tracker = DefaultRateTracker::new(Duration::from_secs(1));

     

       231
       231
       +
       

     

       232
       232
       +
               // record a lot of events

     

       233
       233
       +
               tracker.observe(1000);

     

       234
       234
       +
       

     

       235
       235
       +
               let rate = tracker.rate();

     

       236
       236
       +
               assert_eq!(rate, 1000.0); // 1000 events in 1 second

     

       237
       237
       +
           }

     

       238
       238
       +
       

     

       239
       239
       +
           #[test]

     

       240
       240
       +
           fn test_rate_tracker_threading() {

     

       241
       241
       +
               let tracker = Arc::new(DefaultRateTracker::new(Duration::from_secs(1)));

     

       242
       242
       +
               let mut handles = vec![];

     

       243
       243
       +
       

     

       244
       244
       +
               for _ in 0..4 {

     

       245
       245
       +
                   let tracker_clone = Arc::clone(&tracker);

     

       246
       246
       +
                   let handle = thread::spawn(move || {

     

       247
       247
       +
                       tracker_clone.observe(10);

     

       248
       248
       +
                   });

     

       249
       249
       +
                   handles.push(handle);

     

       250
       250
       +
               }

     

       251
       251
       +
       

     

       252
       252
       +
               for handle in handles {

     

       253
       253
       +
                   handle.join().unwrap();

     

       254
       254
       +
               }

     

       255
       255
       +
       

     

       256
       256
       +
               let rate = tracker.rate();

     

       257
       257
       +
               assert_eq!(rate, 40.0); // 40 events in 1 second

     

       258
       258
       +
           }

     

       259
       259
       +
       }

     

       260
       260
       +
       

     

       261
       261
       +
       #[derive(Debug, Clone, Copy, PartialEq, Eq)]

     

       262
       262
       +
       pub enum TimeDirection {

     

       263
       263
       +
           Backwards, // Past (default)

     

       264
       264
       +
           Forwards,  // Future

     

       265
       265
       +
       }

     

       266
       266
       +
       

     

       267
       267
       +
       impl Default for TimeDirection {

     

       268
       268
       +
           fn default() -> Self {

     

       269
       269
       +
               TimeDirection::Backwards

     

       270
       270
       +
           }

     

       271
       271
       +
       }

     

       272
       272
       +
       

     

       273
       273
       +
       #[derive(Debug, Clone, PartialEq, Eq)]

     

       274
       274
       +
       pub struct RelativeDateTime {

     

       275
       275
       +
           duration: Duration,

     

       276
       276
       +
           direction: TimeDirection,

     

       277
       277
       +
       }

     

       278
       278
       +
       

     

       279
       279
       +
       impl RelativeDateTime {

     

       280
       280
       +
           pub fn new(duration: Duration, direction: TimeDirection) -> Self {

     

       281
       281
       +
               Self {

     

       282
       282
       +
                   duration,

     

       283
       283
       +
                   direction,

     

       284
       284
       +
               }

     

       285
       285
       +
           }

     

       286
       286
       +
       

     

       287
       287
       +
           pub fn from_now(duration: Duration) -> Self {

     

       288
       288
       +
               let cur = get_time();

     

       289
       289
       +
               if duration > cur {

     

       290
       290
       +
                   Self::new(duration - cur, TimeDirection::Forwards)

     

       291
       291
       +
               } else {

     

       292
       292
       +
                   Self::new(cur - duration, TimeDirection::Backwards)

     

       293
       293
       +
               }

     

       294
       294
       +
           }

     

       295
       295
       +
       }

     

       296
       296
       +
       

     

       297
       297
       +
       impl std::fmt::Display for RelativeDateTime {

     

       298
       298
       +
           fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

     

       299
       299
       +
               let secs = self.duration.as_secs();

     

       300
       300
       +
       

     

       301
       301
       +
               if secs == 0 {

     

       302
       302
       +
                   return write!(f, "now");

     

       303
       303
       +
               }

     

       304
       304
       +
       

     

       305
       305
       +
               let (amount, unit) = match secs {

     

       306
       306
       +
                   0 => unreachable!(), // handled above

     

       307
       307
       +
                   1..=59 => (secs, "second"),

     

       308
       308
       +
                   60..=3599 => (secs / 60, "minute"),

     

       309
       309
       +
                   3600..=86399 => (secs / 3600, "hour"),

     

       310
       310
       +
                   86400..=2591999 => (secs / 86400, "day"), // up to 29 days

     

       311
       311
       +
                   2592000..=31535999 => (secs / 2592000, "month"), // 30 days to 364 days

     

       312
       312
       +
                   _ => (secs / 31536000, "year"),           // 365 days+

     

       313
       313
       +
               };

     

       314
       314
       +
       

     

       315
       315
       +
               let plural = if amount != 1 { "s" } else { "" };

     

       316
       316
       +
       

     

       317
       317
       +
               match self.direction {

     

       318
       318
       +
                   TimeDirection::Forwards => write!(f, "in {} {}{}", amount, unit, plural),

     

       319
       319
       +
                   TimeDirection::Backwards => write!(f, "{} {}{} ago", amount, unit, plural),

     

       320
       320
       +
               }

     

       321
       321
       +
           }

     

       322
       322
       +
       }

Compare changes