ptr.pet / nsid-tracker
tracks lexicons and how many times they appeared on the jetstream
fork atom

refactor(server): use mimalloc

ptr.pet 9581c553 c2522740

verified
1/1
build.yml
options
unified split
Changed files
+29 -39
.tangled
workflows
build.yml
server
Cargo.lock
Cargo.toml
src
db
mod.rs
main.rs
-2
.tangled/workflows/build.yml 
···

       
8

       
 

       
    - bun


     

       
9

       
 

       
    - curl


     

       
10

       
 

       
    - gcc


     

       
11

       
-

       
    - cmake


     

       
12

       
-

       
    - gnumake


     

       
13

       
 

       



     

       
14

       
 

       
steps:


     

       
15

       
 

       
  - name: test server


     
···

       
8

       
 

       
    - bun


     

       
9

       
 

       
    - curl


     

       
10

       
 

       
    - gcc


     

       

       

       
     

       

       

       
     

       
11

       
 

       



     

       
12

       
 

       
steps:


     

       
13

       
 

       
  - name: test server
+20 -28
server/Cargo.lock 
···

       
329

       
 

       
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"


     

       
330

       
 

       



     

       
331

       
 

       
[[package]]


     

       
332

       
-

       
name = "cmake"


     

       
333

       
-

       
version = "0.1.54"


     

       
334

       
-

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
335

       
-

       
checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"


     

       
336

       
-

       
dependencies = [


     

       
337

       
-

       
 "cc",


     

       
338

       
-

       
]


     

       
339

       
-

       



     

       
340

       
-

       
[[package]]


     

       
341

       
 

       
name = "combine"


     

       
342

       
 

       
version = "4.6.7"


     

       
343

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     
···

       
861

       
 

       
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"


     

       
862

       
 

       



     

       
863

       
 

       
[[package]]


     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       
864

       
 

       
name = "linux-raw-sys"


     

       
865

       
 

       
version = "0.9.4"


     

       
866

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     
···

       
935

       
 

       
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"


     

       
936

       
 

       



     

       
937

       
 

       
[[package]]


     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       
938

       
 

       
name = "mime"


     

       
939

       
 

       
version = "0.3.17"


     

       
940

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     
···

       
1582

       
 

       
 "fjall",


     

       
1583

       
 

       
 "futures-util",


     

       
1584

       
 

       
 "itertools",


     

       

       

       
     

       
1585

       
 

       
 "ordered-varint",


     

       
1586

       
 

       
 "parking_lot",


     

       
1587

       
 

       
 "quanta",


     
···

       
1593

       
 

       
 "serde",


     

       
1594

       
 

       
 "serde_json",


     

       
1595

       
 

       
 "smol_str",


     

       
1596

       
-

       
 "snmalloc-rs",


     

       
1597

       
 

       
 "threadpool",


     

       
1598

       
 

       
 "tokio",


     

       
1599

       
 

       
 "tokio-util",


     
···

       
1664

       
 

       
dependencies = [


     

       
1665

       
 

       
 "borsh",


     

       
1666

       
 

       
 "serde",


     

       
1667

       
-

       
]


     

       
1668

       
-

       



     

       
1669

       
-

       
[[package]]


     

       
1670

       
-

       
name = "snmalloc-rs"


     

       
1671

       
-

       
version = "0.3.8"


     

       
1672

       
-

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
1673

       
-

       
checksum = "eb317153089fdfa4d8a2eec059d40a5a23c3bde43995ea23b19121c3f621e74a"


     

       
1674

       
-

       
dependencies = [


     

       
1675

       
-

       
 "snmalloc-sys",


     

       
1676

       
-

       
]


     

       
1677

       
-

       



     

       
1678

       
-

       
[[package]]


     

       
1679

       
-

       
name = "snmalloc-sys"


     

       
1680

       
-

       
version = "0.3.8"


     

       
1681

       
-

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
1682

       
-

       
checksum = "065fea53d32bb77bc36cca466cb191f2e5216ebfd0ed360b1d64889ee6e559ea"


     

       
1683

       
-

       
dependencies = [


     

       
1684

       
-

       
 "cmake",


     

       
1685

       
 

       
]


     

       
1686

       
 

       



     

       
1687

       
 

       
[[package]]


     
···

       
329

       
 

       
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"


     

       
330

       
 

       



     

       
331

       
 

       
[[package]]


     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       
332

       
 

       
name = "combine"


     

       
333

       
 

       
version = "4.6.7"


     

       
334

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     
···

       
852

       
 

       
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"


     

       
853

       
 

       



     

       
854

       
 

       
[[package]]


     

       
855

       
+

       
name = "libmimalloc-sys"


     

       
856

       
+

       
version = "0.1.43"


     

       
857

       
+

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
858

       
+

       
checksum = "bf88cd67e9de251c1781dbe2f641a1a3ad66eaae831b8a2c38fbdc5ddae16d4d"


     

       
859

       
+

       
dependencies = [


     

       
860

       
+

       
 "cc",


     

       
861

       
+

       
 "libc",


     

       
862

       
+

       
]


     

       
863

       
+

       



     

       
864

       
+

       
[[package]]


     

       
865

       
 

       
name = "linux-raw-sys"


     

       
866

       
 

       
version = "0.9.4"


     

       
867

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     
···

       
936

       
 

       
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"


     

       
937

       
 

       



     

       
938

       
 

       
[[package]]


     

       
939

       
+

       
name = "mimalloc"


     

       
940

       
+

       
version = "0.1.47"


     

       
941

       
+

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
942

       
+

       
checksum = "b1791cbe101e95af5764f06f20f6760521f7158f69dbf9d6baf941ee1bf6bc40"


     

       
943

       
+

       
dependencies = [


     

       
944

       
+

       
 "libmimalloc-sys",


     

       
945

       
+

       
]


     

       
946

       
+

       



     

       
947

       
+

       
[[package]]


     

       
948

       
 

       
name = "mime"


     

       
949

       
 

       
version = "0.3.17"


     

       
950

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     
···

       
1592

       
 

       
 "fjall",


     

       
1593

       
 

       
 "futures-util",


     

       
1594

       
 

       
 "itertools",


     

       
1595

       
+

       
 "mimalloc",


     

       
1596

       
 

       
 "ordered-varint",


     

       
1597

       
 

       
 "parking_lot",


     

       
1598

       
 

       
 "quanta",


     
···

       
1604

       
 

       
 "serde",


     

       
1605

       
 

       
 "serde_json",


     

       
1606

       
 

       
 "smol_str",


     

       

       

       
     

       
1607

       
 

       
 "threadpool",


     

       
1608

       
 

       
 "tokio",


     

       
1609

       
 

       
 "tokio-util",


     
···

       
1674

       
 

       
dependencies = [


     

       
1675

       
 

       
 "borsh",


     

       
1676

       
 

       
 "serde",


     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       

       

       
     

       
1677

       
 

       
]


     

       
1678

       
 

       



     

       
1679

       
 

       
[[package]]
+1 -1
server/Cargo.toml 
···

       
30

       
 

       
rayon = "1.10.0"


     

       
31

       
 

       
parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] }


     

       
32

       
 

       
rclite = "0.2.7"


     

       
33

       
-

       
snmalloc-rs = "0.3.8"


     

       
34

       
 

       
arc-swap = "1.7.1"


     

       
35

       
 

       
ahash = { version = "0.8.12", features = ["serde"] }


     

       

       

       
     
···

       
30

       
 

       
rayon = "1.10.0"


     

       
31

       
 

       
parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] }


     

       
32

       
 

       
rclite = "0.2.7"


     

       

       

       
     

       
33

       
 

       
arc-swap = "1.7.1"


     

       
34

       
 

       
ahash = { version = "0.8.12", features = ["serde"] }


     

       
35

       
+

       
mimalloc = "*"
+7 -7
server/src/db/mod.rs 
···

       
171

       
 

       
        let _guard = scc::ebr::Guard::new();


     

       
172

       
 

       
        for (nsid, handle) in self.hits.iter(&_guard) {


     

       
173

       
 

       
            let mut nsid_data = Vec::with_capacity(2);


     

       
174

       
-

       
            let mut total_count = 0;


     

       
175

       
 

       
            let is_too_old = handle.since_last_activity() > self.cfg.max_last_activity;


     

       
176

       
 

       
            // if we disconnect for a long time, we want to sync all of what we


     

       
177

       
 

       
            // have to avoid having many small blocks (even if we run compaction


     
···

       
188

       
 

       
            if count > 0 && (all || data_count > 0 || is_too_old) {


     

       
189

       
 

       
                for _ in 0..data_count {


     

       
190

       
 

       
                    nsid_data.push((handle.clone(), block_size));


     

       
191

       
-

       
                    total_count += block_size;


     

       
192

       
 

       
                }


     

       
193

       
 

       
                // only sync remainder if we haven't met block size


     

       
194

       
 

       
                let remainder = count % block_size;


     

       
195

       
 

       
                if (all || data_count == 0) && remainder > 0 {


     

       
196

       
 

       
                    nsid_data.push((handle.clone(), remainder));


     

       
197

       
-

       
                    total_count += remainder;


     

       
198

       
 

       
                }


     

       
199

       
 

       
            }


     

       
200

       
 

       
            let _span = handle.span().entered();


     

       
201

       
 

       
            if nsid_data.len() > 0 {


     

       
202

       
-

       
                tracing::info!(


     

       
203

       
-

       
                    {blocks = %nsid_data.len(), count = %total_count},


     

       
204

       
-

       
                    "will encode & sync",


     

       
205

       
-

       
                );


     

       
206

       
 

       
                nsids.insert(nsid.clone());


     

       
207

       
 

       
                data.push(nsid_data);


     

       
208

       
 

       
            }


     
···

       
171

       
 

       
        let _guard = scc::ebr::Guard::new();


     

       
172

       
 

       
        for (nsid, handle) in self.hits.iter(&_guard) {


     

       
173

       
 

       
            let mut nsid_data = Vec::with_capacity(2);


     

       
174

       
+

       
            // let mut total_count = 0;


     

       
175

       
 

       
            let is_too_old = handle.since_last_activity() > self.cfg.max_last_activity;


     

       
176

       
 

       
            // if we disconnect for a long time, we want to sync all of what we


     

       
177

       
 

       
            // have to avoid having many small blocks (even if we run compaction


     
···

       
188

       
 

       
            if count > 0 && (all || data_count > 0 || is_too_old) {


     

       
189

       
 

       
                for _ in 0..data_count {


     

       
190

       
 

       
                    nsid_data.push((handle.clone(), block_size));


     

       
191

       
+

       
                    // total_count += block_size;


     

       
192

       
 

       
                }


     

       
193

       
 

       
                // only sync remainder if we haven't met block size


     

       
194

       
 

       
                let remainder = count % block_size;


     

       
195

       
 

       
                if (all || data_count == 0) && remainder > 0 {


     

       
196

       
 

       
                    nsid_data.push((handle.clone(), remainder));


     

       
197

       
+

       
                    // total_count += remainder;


     

       
198

       
 

       
                }


     

       
199

       
 

       
            }


     

       
200

       
 

       
            let _span = handle.span().entered();


     

       
201

       
 

       
            if nsid_data.len() > 0 {


     

       
202

       
+

       
                // tracing::info!(


     

       
203

       
+

       
                //     {blocks = %nsid_data.len(), count = %total_count},


     

       
204

       
+

       
                //     "will encode & sync",


     

       
205

       
+

       
                // );


     

       
206

       
 

       
                nsids.insert(nsid.clone());


     

       
207

       
 

       
                data.push(nsid_data);


     

       
208

       
 

       
            }
+1 -1
server/src/main.rs 
···

       
22

       
 

       
mod utils;


     

       
23

       
 

       



     

       
24

       
 

       
#[global_allocator]


     

       
25

       
-

       
static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;


     

       
26

       
 

       



     

       
27

       
 

       
#[tokio::main]


     

       
28

       
 

       
async fn main() {


     
···

       
22

       
 

       
mod utils;


     

       
23

       
 

       



     

       
24

       
 

       
#[global_allocator]


     

       
25

       
+

       
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;


     

       
26

       
 

       



     

       
27

       
 

       
#[tokio::main]


     

       
28

       
 

       
async fn main() {