···

       
4
       
 
       


     

       
5
       
 
       
pub async fn firehose_cache(

     

       
6
       
 
       
    cache_dir: impl AsRef<Path>,

     

       
0
       
 
       

     

       
0
       
 
       

     

       
7
       
 
       
) -> Result<HybridCache<String, CachedRecord>, String> {

     

       
8
       
 
       
    let cache = HybridCacheBuilder::new()

     

       
9
       
 
       
        .with_name("firehose")

     

       
10
       
-
       
        .memory(64 * 2_usize.pow(20))

     

       
11
       
 
       
        .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v))

     

       
12
       
 
       
        .storage(Engine::large())

     

       
13
       
 
       
        .with_device_options(

     

       
14
       
 
       
            DirectFsDeviceOptions::new(cache_dir)

     

       
15
       
-
       
                .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something)

     

       
16
       
 
       
                .with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records

     

       
17
       
 
       
        )

     

       
18
       
 
       
        .build()

     

···

       
4
       
 
       


     

       
5
       
 
       
pub async fn firehose_cache(

     

       
6
       
 
       
    cache_dir: impl AsRef<Path>,

     

       
7
       
+
       
    memory_mb: usize,

     

       
8
       
+
       
    disk_gb: usize,

     

       
9
       
 
       
) -> Result<HybridCache<String, CachedRecord>, String> {

     

       
10
       
 
       
    let cache = HybridCacheBuilder::new()

     

       
11
       
 
       
        .with_name("firehose")

     

       
12
       
+
       
        .memory(memory_mb * 2_usize.pow(20))

     

       
13
       
 
       
        .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v))

     

       
14
       
 
       
        .storage(Engine::large())

     

       
15
       
 
       
        .with_device_options(

     

       
16
       
 
       
            DirectFsDeviceOptions::new(cache_dir)

     

       
17
       
+
       
                .with_capacity(disk_gb * 2_usize.pow(30))

     

       
18
       
 
       
                .with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records

     

       
19
       
 
       
        )

     

       
20
       
 
       
        .build()

     

···

       
25
       
 
       
    /// where to keep disk caches

     

       
26
       
 
       
    #[arg(long)]

     

       
27
       
 
       
    cache_dir: PathBuf,

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
28
       
 
       
    /// the domain pointing to this server

     

       
29
       
 
       
    ///

     

       
30
       
 
       
    /// if present:

     
···

       
83
       
 
       
    log::info!("cache dir ready at at {cache_dir:?}.");

     

       
84
       
 
       


     

       
85
       
 
       
    log::info!("setting up firehose cache...");

     

       
86
       
-
       
    let cache = firehose_cache(cache_dir.join("./firehose")).await?;

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
0
       
 
       

     

       
87
       
 
       
    log::info!("firehose cache ready.");

     

       
88
       
 
       


     

       
89
       
 
       
    let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new();

     

···

       
25
       
 
       
    /// where to keep disk caches

     

       
26
       
 
       
    #[arg(long)]

     

       
27
       
 
       
    cache_dir: PathBuf,

     

       
28
       
+
       
    /// memory cache size in MB

     

       
29
       
+
       
    #[arg(long, default_value_t = 64)]

     

       
30
       
+
       
    cache_memory_mb: usize,

     

       
31
       
+
       
    /// disk cache size in GB

     

       
32
       
+
       
    #[arg(long, default_value_t = 1)]

     

       
33
       
+
       
    cache_disk_gb: usize,

     

       
34
       
 
       
    /// the domain pointing to this server

     

       
35
       
 
       
    ///

     

       
36
       
 
       
    /// if present:

     
···

       
89
       
 
       
    log::info!("cache dir ready at at {cache_dir:?}.");

     

       
90
       
 
       


     

       
91
       
 
       
    log::info!("setting up firehose cache...");

     

       
92
       
+
       
    let cache = firehose_cache(

     

       
93
       
+
       
        cache_dir.join("./firehose"),

     

       
94
       
+
       
        args.cache_memory_mb,

     

       
95
       
+
       
        args.cache_disk_gb,

     

       
96
       
+
       
    )

     

       
97
       
+
       
    .await?;

     

       
98
       
 
       
    log::info!("firehose cache ready.");

     

       
99
       
 
       


     

       
100
       
 
       
    let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new();