nekomimi.pet / microcosm-rs
forked from microcosm.blue/microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
fork atom

use git-main fjall

for weak delete

bad-example.com 59028924 fbb2f92d

options
unified split
Changed files
+22 -18
Cargo.lock
ufos
Cargo.toml
src
storage_fjall.rs
+15 -16
Cargo.lock 
···

       
1220

       
1220

       
 

       
checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f"


     

       
1221

       
1221

       
 

       
dependencies = [


     

       
1222

       
1222

       
 

       
 "data-encoding",


     

       
1223

       

       
-

       
 "syn 2.0.103",


     

       

       
1223

       
+

       
 "syn 1.0.109",


     

       
1224

       
1224

       
 

       
]


     

       
1225

       
1225

       
 

       



     

       
1226

       
1226

       
 

       
[[package]]


     
···

       
1569

       
1569

       
 

       
checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"


     

       
1570

       
1570

       
 

       
dependencies = [


     

       
1571

       
1571

       
 

       
 "libc",


     

       
1572

       

       
-

       
 "windows-sys 0.59.0",


     

       

       
1572

       
+

       
 "windows-sys 0.52.0",


     

       
1573

       
1573

       
 

       
]


     

       
1574

       
1574

       
 

       



     

       
1575

       
1575

       
 

       
[[package]]


     
···

       
1611

       
1611

       
 

       



     

       
1612

       
1612

       
 

       
[[package]]


     

       
1613

       
1613

       
 

       
name = "fjall"


     

       
1614

       

       
-

       
version = "2.8.0"


     

       
1615

       

       
-

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
1616

       

       
-

       
checksum = "26b2ced3483989a62b3533c9f99054d73b527c6c0045cf22b00fe87956f1a46f"


     

       

       
1614

       
+

       
version = "2.11.2"


     

       

       
1615

       
+

       
source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d"


     

       
1617

       
1616

       
 

       
dependencies = [


     

       
1618

       
1617

       
 

       
 "byteorder",


     

       
1619

       
1618

       
 

       
 "byteview",


     
···

       
2595

       
2594

       
 

       
dependencies = [


     

       
2596

       
2595

       
 

       
 "hermit-abi",


     

       
2597

       
2596

       
 

       
 "libc",


     

       
2598

       

       
-

       
 "windows-sys 0.59.0",


     

       

       
2597

       
+

       
 "windows-sys 0.52.0",


     

       
2599

       
2598

       
 

       
]


     

       
2600

       
2599

       
 

       



     

       
2601

       
2600

       
 

       
[[package]]


     
···

       
2800

       
2799

       
 

       
checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"


     

       
2801

       
2800

       
 

       
dependencies = [


     

       
2802

       
2801

       
 

       
 "cfg-if",


     

       
2803

       

       
-

       
 "windows-targets 0.52.6",


     

       

       
2802

       
+

       
 "windows-targets 0.48.5",


     

       
2804

       
2803

       
 

       
]


     

       
2805

       
2804

       
 

       



     

       
2806

       
2805

       
 

       
[[package]]


     
···

       
2931

       
2930

       
 

       



     

       
2932

       
2931

       
 

       
[[package]]


     

       
2933

       
2932

       
 

       
name = "lsm-tree"


     

       
2934

       

       
-

       
version = "2.8.0"


     

       

       
2933

       
+

       
version = "2.10.2"


     

       
2935

       
2934

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
2936

       

       
-

       
checksum = "d0a63a5e98a38b51765274137d8aedfbd848da5f4d016867e186b673fcc06a8c"


     

       

       
2935

       
+

       
checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c"


     

       
2937

       
2936

       
 

       
dependencies = [


     

       
2938

       
2937

       
 

       
 "byteorder",


     

       
2939

       
2938

       
 

       
 "crossbeam-skiplist",


     
···

       
4054

       
4053

       
 

       
 "once_cell",


     

       
4055

       
4054

       
 

       
 "socket2 0.5.9",


     

       
4056

       
4055

       
 

       
 "tracing",


     

       
4057

       

       
-

       
 "windows-sys 0.59.0",


     

       

       
4056

       
+

       
 "windows-sys 0.52.0",


     

       
4058

       
4057

       
 

       
]


     

       
4059

       
4058

       
 

       



     

       
4060

       
4059

       
 

       
[[package]]


     
···

       
4436

       
4435

       
 

       
 "errno",


     

       
4437

       
4436

       
 

       
 "libc",


     

       
4438

       
4437

       
 

       
 "linux-raw-sys 0.4.15",


     

       
4439

       

       
-

       
 "windows-sys 0.59.0",


     

       

       
4438

       
+

       
 "windows-sys 0.52.0",


     

       
4440

       
4439

       
 

       
]


     

       
4441

       
4440

       
 

       



     

       
4442

       
4441

       
 

       
[[package]]


     
···

       
4449

       
4448

       
 

       
 "errno",


     

       
4450

       
4449

       
 

       
 "libc",


     

       
4451

       
4450

       
 

       
 "linux-raw-sys 0.9.4",


     

       
4452

       

       
-

       
 "windows-sys 0.59.0",


     

       

       
4451

       
+

       
 "windows-sys 0.52.0",


     

       
4453

       
4452

       
 

       
]


     

       
4454

       
4453

       
 

       



     

       
4455

       
4454

       
 

       
[[package]]


     
···

       
5188

       
5187

       
 

       
 "getrandom 0.3.3",


     

       
5189

       
5188

       
 

       
 "once_cell",


     

       
5190

       
5189

       
 

       
 "rustix 1.0.5",


     

       
5191

       

       
-

       
 "windows-sys 0.59.0",


     

       

       
5190

       
+

       
 "windows-sys 0.52.0",


     

       
5192

       
5191

       
 

       
]


     

       
5193

       
5192

       
 

       



     

       
5194

       
5193

       
 

       
[[package]]


     
···

       
5821

       
5820

       
 

       



     

       
5822

       
5821

       
 

       
[[package]]


     

       
5823

       
5822

       
 

       
name = "value-log"


     

       
5824

       

       
-

       
version = "1.8.0"


     

       

       
5823

       
+

       
version = "1.9.0"


     

       
5825

       
5824

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
5826

       

       
-

       
checksum = "fd29b17c041f94e0885179637289815cd038f0c9fc19c4549d5a97017404fb7d"


     

       

       
5825

       
+

       
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"


     

       
5827

       
5826

       
 

       
dependencies = [


     

       
5828

       
5827

       
 

       
 "byteorder",


     

       
5829

       
5828

       
 

       
 "byteview",


     
···

       
6073

       
6072

       
 

       
source = "registry+https://github.com/rust-lang/crates.io-index"


     

       
6074

       
6073

       
 

       
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"


     

       
6075

       
6074

       
 

       
dependencies = [


     

       
6076

       

       
-

       
 "windows-sys 0.59.0",


     

       

       
6075

       
+

       
 "windows-sys 0.48.0",


     

       
6077

       
6076

       
 

       
]


     

       
6078

       
6077

       
 

       



     

       
6079

       
6078

       
 

       
[[package]]
+1 -1
ufos/Cargo.toml 
···

       
13

       
13

       
 

       
clap = { version = "4.5.31", features = ["derive"] }


     

       
14

       
14

       
 

       
dropshot = "0.16.0"


     

       
15

       
15

       
 

       
env_logger = "0.11.7"


     

       
16

       

       
-

       
fjall = { version = "2.8.0", features = ["lz4"] }


     

       

       
16

       
+

       
fjall = { git = "https://github.com/fjall-rs/fjall.git", features = ["lz4"] }


     

       
17

       
17

       
 

       
getrandom = "0.3.3"


     

       
18

       
18

       
 

       
http = "1.3.1"


     

       
19

       
19

       
 

       
jetstream = { path = "../jetstream", features = ["metrics"] }
+6 -1
ufos/src/storage_fjall.rs 
···

       
1619

       
1619

       
 

       
                    histogram!("storage_trim_dirty_nsids").record(completed.len() as f64);


     

       
1620

       
1620

       
 

       
                    histogram!("storage_trim_duration").record(dt.as_micros() as f64);


     

       
1621

       
1621

       
 

       
                    counter!("storage_trim_removed", "dangling" => "true").increment(total_danglers as u64);


     

       
1622

       

       
-

       
                    counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64);


     

       

       
1622

       
+

       
                    if total_deleted >= total_danglers {


     

       

       
1623

       
+

       
                        counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64);


     

       

       
1624

       
+

       
                    } else {


     

       

       
1625

       
+

       
                        // TODO: probably think through what's happening here


     

       

       
1626

       
+

       
                        log::warn!("weird trim case: more danglers than deleted? metric will be missing for dangling=false. deleted={total_deleted} danglers={total_danglers}");


     

       

       
1627

       
+

       
                    }


     

       
1623

       
1628

       
 

       
                    for c in completed {


     

       
1624

       
1629

       
 

       
                        dirty_nsids.remove(&c);


     

       
1625

       
1630

       
 

       
                    }