commit 7ebaa1097ab6de057354bb8a45b867d6a8dee18b · anil.recoil.org/slop

cacheio/.gitignore stack/cacheio/.gitignore

cacheio/README.md stack/cacheio/README.md

cacheio/bin/dune stack/cacheio/bin/dune

+3 -3

cacheio/bin/example.ml stack/cacheio/bin/example.ml

···

       30
        
           ] in

     

       31
        
       

     

       32
        
           (* Cache for 1 hour *)

     

       33
       -
           Cacheio.put_json cache ~key:url ~data:response_data ~ttl:3600.0;

     

       34
        
           Logs.info (fun m -> m "Cached response for %s" url)

     

       35
        
         ) urls;

     

       36
        
       

     
···

       67
        
         List.iter (fun components ->

     

       68
        
           let key = Cacheio.make_key components in

     

       69
        
           let data = Printf.sprintf "Data for %s" key in

     

       70
       -
           Cacheio.put cache ~key ~data ~ttl:1800.0;

     

       71
        
           Logs.info (fun m -> m "Stored data with key: %s" key)

     

       72
        
         ) repos;

     

       73
        
       

     
···

       108
        
         for i = 1 to 15 do

     

       109
        
           let key = Printf.sprintf "key_%d" i in

     

       110
        
           let data = Printf.sprintf "value_%d" i in

     

       111
       -
           Cacheio.put mem_cache ~key ~data ~ttl:60.0

     

       112
        
         done;

     

       113
        
       

     

       114
        
         (* Check that eviction worked (max 10 entries) *)

···

       30
        
           ] in

     

       31
        
       

     

       32
        
           (* Cache for 1 hour *)

     

       33
       +
           Cacheio.put_json cache ~key:url ~data:response_data ~ttl:(Some 3600.0);

     

       34
        
           Logs.info (fun m -> m "Cached response for %s" url)

     

       35
        
         ) urls;

     

       36
        
       

     
···

       67
        
         List.iter (fun components ->

     

       68
        
           let key = Cacheio.make_key components in

     

       69
        
           let data = Printf.sprintf "Data for %s" key in

     

       70
       +
           Cacheio.put cache ~key ~data ~ttl:(Some 1800.0);

     

       71
        
           Logs.info (fun m -> m "Stored data with key: %s" key)

     

       72
        
         ) repos;

     

       73
        
       

     
···

       108
        
         for i = 1 to 15 do

     

       109
        
           let key = Printf.sprintf "key_%d" i in

     

       110
        
           let data = Printf.sprintf "value_%d" i in

     

       111
       +
           Cacheio.put mem_cache ~key ~data ~ttl:(Some 60.0)

     

       112
        
         done;

     

       113
        
       

     

       114
        
         (* Check that eviction worked (max 10 entries) *)

+1

cacheio/cacheio.opam stack/cacheio/cacheio.opam

···

       19
        
         "ptime"

     

       20
        
         "logs"

     

       21
        
         "fmt"

     

       0
        
       
     

       22
        
         "alcotest" {with-test}

     

       23
        
         "odoc" {with-doc}

     

       24
        
       ]

···

       19
        
         "ptime"

     

       20
        
         "logs"

     

       21
        
         "fmt"

     

       22
       +
         "xdge"

     

       23
        
         "alcotest" {with-test}

     

       24
        
         "odoc" {with-doc}

     

       25
        
       ]

+1

cacheio/dune-project stack/cacheio/dune-project

···

       28
        
         ptime

     

       29
        
         logs

     

       30
        
         fmt

     

       0
        
       
     

       31
        
         (alcotest :with-test))

     

       32
        
        (tags

     

       33
        
         (cache filesystem xdg cmdliner eio)))

···

       28
        
         ptime

     

       29
        
         logs

     

       30
        
         fmt

     

       31
       +
         xdge

     

       32
        
         (alcotest :with-test))

     

       33
        
        (tags

     

       34
        
         (cache filesystem xdg cmdliner eio)))

+42 -24

cacheio/lib/cacheio.ml stack/cacheio/lib/cacheio.ml

···

       14
        
       module type STORAGE = sig

     

       15
        
         type t

     

       16
        
         val get : t -> key:string -> string option

     

       17
       -
         val put : t -> key:string -> data:string -> ttl:float -> unit

     

       18
        
         val delete : t -> key:string -> unit

     

       19
        
         val clear : t -> unit

     

       20
        
         val exists : t -> key:string -> bool

     
···

       27
        
       module FileStorage = struct

     

       28
        
         type t = {

     

       29
        
           base_dir : Fs.dir_ty Path.t;

     

       30
       -
           max_size : int64 option;

     

       31
        
           app_name : string;

     

       32
        
           mutable hits : int;

     

       33
        
           mutable misses : int;

     

       34
        
           mutex : Eio.Mutex.t;

     

       35
        
         }

     

       36
        
       

     

       37
       -
         let create ~base_dir ?max_size ~app_name () =

     

       38
       -
           { base_dir; max_size; app_name; hits = 0; misses = 0;

     

       39
        
             mutex = Eio.Mutex.create () }

     

       40
        
       

     

       41
        
         (** Ensure directory structure exists. TODO simplify with create_dirs? *)

     
···

       91
        
         (** Cache entry metadata *)

     

       92
        
         type entry_metadata = {

     

       93
        
           created_at : Ptime.t;

     

       94
       -
           ttl : float;

     

       95
        
           size : int;

     

       96
        
         }

     

       97
        
       

     

       98
        
         let metadata_to_json metadata =

     

       99
       -
           `Assoc [

     

       100
        
             ("created_at", `String (Ptime.to_rfc3339 metadata.created_at));

     

       101
       -
             ("ttl", `Float metadata.ttl);

     

       102
        
             ("size", `Int metadata.size);

     

       103
       -
           ]

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       104
        
       

     

       105
        
         let metadata_from_json json =

     

       106
        
           let open Yojson.Safe.Util in

     
···

       109
        
             | Ok (t, _, _) -> t

     

       110
        
             | Error _ -> Ptime.epoch

     

       111
        
           in

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       112
        
           {

     

       113
        
             created_at;

     

       114
       -
             ttl = json |> member "ttl" |> to_float;

     

       115
        
             size = json |> member "size" |> to_int;

     

       116
        
           }

     

       117
        
       

     

       118
        
         let is_expired metadata =

     

       119
       -
           let now = Ptime_clock.now () in

     

       120
       -
           let age_span = Ptime.diff now metadata.created_at in

     

       121
       -
           let age_seconds = Ptime.Span.to_float_s age_span in

     

       122
       -
           age_seconds > metadata.ttl

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       123
        
       

     

       124
        
         let get t ~key =

     

       125
        
           Mutex.use_rw ~protect:true t.mutex @@ fun () ->

     
···

       259
        
         type entry = {

     

       260
        
           data : string;

     

       261
        
           created_at : Ptime.t;

     

       262
       -
           ttl : float;

     

       263
        
         }

     

       264
        
       

     

       265
        
         type t = {

     
···

       278
        
             mutex = Eio.Mutex.create () }

     

       279
        
       

     

       280
        
         let is_expired entry =

     

       281
       -
           let now = Ptime_clock.now () in

     

       282
       -
           let age_span = Ptime.diff now entry.created_at in

     

       283
       -
           let age_seconds = Ptime.Span.to_float_s age_span in

     

       284
       -
           age_seconds > entry.ttl

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       285
        
       

     

       286
        
         let get t ~key =

     

       287
        
           Mutex.use_rw ~protect:true t.mutex @@ fun () ->

     
···

       464
        
             let cache_dir_path = match config.cache_dir with

     

       465
        
               | Some dir -> dir

     

       466
        
               | None ->

     

       467
       -
                   let xdg = Xdg.create ~env:Sys.getenv_opt () in

     

       468
       -
                   Filename.concat (Xdg.cache_dir xdg) app_name

     

       0
        
       
     

       469
        
             in

     

       470
        
       

     

       471
        
             let base_dir = Eio.Path.(Eio.Stdenv.fs env / cache_dir_path) in

     
···

       511
        
         float_of_int (hours * 3600)

     

       512
        
       

     

       513
        
       let is_expired ~created_at ~ttl =

     

       514
       -
         let now = Ptime_clock.now () in

     

       515
       -
         let age_span = Ptime.diff now created_at in

     

       516
       -
         let age_seconds = Ptime.Span.to_float_s age_span in

     

       517
       -
         age_seconds > ttl

     

       0
        
       
     

       0
        
       
     

       0
        
       
     

       518
        
       

     

       519
        
       let format_size bytes =

     

       520
        
         let open Int64 in

···

       14
        
       module type STORAGE = sig

     

       15
        
         type t

     

       16
        
         val get : t -> key:string -> string option

     

       17
       +
         val put : t -> key:string -> data:string -> ttl:float option -> unit

     

       18
        
         val delete : t -> key:string -> unit

     

       19
        
         val clear : t -> unit

     

       20
        
         val exists : t -> key:string -> bool

     
···

       27
        
       module FileStorage = struct

     

       28
        
         type t = {

     

       29
        
           base_dir : Fs.dir_ty Path.t;

     

       30
       +
           (* max_size : int64 option;  TODO: implement max size enforcement *)

     

       31
        
           app_name : string;

     

       32
        
           mutable hits : int;

     

       33
        
           mutable misses : int;

     

       34
        
           mutex : Eio.Mutex.t;

     

       35
        
         }

     

       36
        
       

     

       37
       +
         let create ~base_dir ?max_size:_ ~app_name () =

     

       38
       +
           { base_dir; app_name; hits = 0; misses = 0;

     

       39
        
             mutex = Eio.Mutex.create () }

     

       40
        
       

     

       41
        
         (** Ensure directory structure exists. TODO simplify with create_dirs? *)

     
···

       91
        
         (** Cache entry metadata *)

     

       92
        
         type entry_metadata = {

     

       93
        
           created_at : Ptime.t;

     

       94
       +
           ttl : float option;  (* None means no expiration *)

     

       95
        
           size : int;

     

       96
        
         }

     

       97
        
       

     

       98
        
         let metadata_to_json metadata =

     

       99
       +
           let fields = [

     

       100
        
             ("created_at", `String (Ptime.to_rfc3339 metadata.created_at));

     

       0
        
       
     

       101
        
             ("size", `Int metadata.size);

     

       102
       +
           ] in

     

       103
       +
           let fields = match metadata.ttl with

     

       104
       +
             | Some ttl -> ("ttl", `Float ttl) :: fields

     

       105
       +
             | None -> fields

     

       106
       +
           in

     

       107
       +
           `Assoc fields

     

       108
        
       

     

       109
        
         let metadata_from_json json =

     

       110
        
           let open Yojson.Safe.Util in

     
···

       113
        
             | Ok (t, _, _) -> t

     

       114
        
             | Error _ -> Ptime.epoch

     

       115
        
           in

     

       116
       +
           let ttl =

     

       117
       +
             try Some (json |> member "ttl" |> to_float)

     

       118
       +
             with _ -> None

     

       119
       +
           in

     

       120
        
           {

     

       121
        
             created_at;

     

       122
       +
             ttl;

     

       123
        
             size = json |> member "size" |> to_int;

     

       124
        
           }

     

       125
        
       

     

       126
        
         let is_expired metadata =

     

       127
       +
           match metadata.ttl with

     

       128
       +
           | None -> false  (* No TTL means never expires *)

     

       129
       +
           | Some ttl ->

     

       130
       +
               let now = Ptime_clock.now () in

     

       131
       +
               let age_span = Ptime.diff now metadata.created_at in

     

       132
       +
               let age_seconds = Ptime.Span.to_float_s age_span in

     

       133
       +
               age_seconds > ttl

     

       134
        
       

     

       135
        
         let get t ~key =

     

       136
        
           Mutex.use_rw ~protect:true t.mutex @@ fun () ->

     
···

       270
        
         type entry = {

     

       271
        
           data : string;

     

       272
        
           created_at : Ptime.t;

     

       273
       +
           ttl : float option;  (* None means no expiration *)

     

       274
        
         }

     

       275
        
       

     

       276
        
         type t = {

     
···

       289
        
             mutex = Eio.Mutex.create () }

     

       290
        
       

     

       291
        
         let is_expired entry =

     

       292
       +
           match entry.ttl with

     

       293
       +
           | None -> false  (* No TTL means never expires *)

     

       294
       +
           | Some ttl ->

     

       295
       +
               let now = Ptime_clock.now () in

     

       296
       +
               let age_span = Ptime.diff now entry.created_at in

     

       297
       +
               let age_seconds = Ptime.Span.to_float_s age_span in

     

       298
       +
               age_seconds > ttl

     

       299
        
       

     

       300
        
         let get t ~key =

     

       301
        
           Mutex.use_rw ~protect:true t.mutex @@ fun () ->

     
···

       478
        
             let cache_dir_path = match config.cache_dir with

     

       479
        
               | Some dir -> dir

     

       480
        
               | None ->

     

       481
       +
                   (* Use xdge library for XDG paths *)

     

       482
       +
                   let xdg = Xdge.create (Eio.Stdenv.fs env) app_name in

     

       483
       +
                   Eio.Path.native_exn (Xdge.cache_dir xdg)

     

       484
        
             in

     

       485
        
       

     

       486
        
             let base_dir = Eio.Path.(Eio.Stdenv.fs env / cache_dir_path) in

     
···

       526
        
         float_of_int (hours * 3600)

     

       527
        
       

     

       528
        
       let is_expired ~created_at ~ttl =

     

       529
       +
         match ttl with

     

       530
       +
         | None -> false  (* No TTL means never expires *)

     

       531
       +
         | Some ttl_val ->

     

       532
       +
             let now = Ptime_clock.now () in

     

       533
       +
             let age_span = Ptime.diff now created_at in

     

       534
       +
             let age_seconds = Ptime.Span.to_float_s age_span in

     

       535
       +
             age_seconds > ttl_val

     

       536
        
       

     

       537
        
       let format_size bytes =

     

       538
        
         let open Int64 in

+9 -7

cacheio/lib/cacheio.mli stack/cacheio/lib/cacheio.mli

···

       15
        
         type t

     

       16
        
       

     

       17
        
         val get : t -> key:string -> string option

     

       18
       -
         val put : t -> key:string -> data:string -> ttl:float -> unit

     

       19
        
         val delete : t -> key:string -> unit

     

       20
        
         val clear : t -> unit

     

       21
        
         val exists : t -> key:string -> bool

     
···

       78
        
       (** Get a value from the cache *)

     

       79
        
       val get : t -> key:string -> string option

     

       80
        
       

     

       81
       -
       (** Put a value into the cache with TTL in seconds *)

     

       82
       -
       val put : t -> key:string -> data:string -> ttl:float -> unit

     

       0
        
       
     

       83
        
       

     

       84
        
       (** Delete a key from the cache *)

     

       85
        
       val delete : t -> key:string -> unit

     
···

       101
        
       (** Get and parse JSON from cache *)

     

       102
        
       val get_json : t -> key:string -> Yojson.Safe.t option

     

       103
        
       

     

       104
       -
       (** Store JSON in cache *)

     

       105
       -
       val put_json : t -> key:string -> data:Yojson.Safe.t -> ttl:float -> unit

     

       106
        
       

     

       107
        
       (** {2 Hierarchical Keys} *)

     

       108
        
       

     
···

       168
        
       (** Convert TTL in hours to seconds *)

     

       169
        
       val hours_to_ttl : int -> float

     

       170
        
       

     

       171
       -
       (** Check if a timestamp is expired given a TTL *)

     

       172
       -
       val is_expired : created_at:Ptime.t -> ttl:float -> bool

     

       0
        
       
     

       173
        
       

     

       174
        
       (** Format cache size for display *)

     

       175
        
       val format_size : int64 -> string

···

       15
        
         type t

     

       16
        
       

     

       17
        
         val get : t -> key:string -> string option

     

       18
       +
         val put : t -> key:string -> data:string -> ttl:float option -> unit

     

       19
        
         val delete : t -> key:string -> unit

     

       20
        
         val clear : t -> unit

     

       21
        
         val exists : t -> key:string -> bool

     
···

       78
        
       (** Get a value from the cache *)

     

       79
        
       val get : t -> key:string -> string option

     

       80
        
       

     

       81
       +
       (** Put a value into the cache with optional TTL in seconds.

     

       82
       +
           If TTL is None, the entry never expires. *)

     

       83
       +
       val put : t -> key:string -> data:string -> ttl:float option -> unit

     

       84
        
       

     

       85
        
       (** Delete a key from the cache *)

     

       86
        
       val delete : t -> key:string -> unit

     
···

       102
        
       (** Get and parse JSON from cache *)

     

       103
        
       val get_json : t -> key:string -> Yojson.Safe.t option

     

       104
        
       

     

       105
       +
       (** Store JSON in cache with optional TTL *)

     

       106
       +
       val put_json : t -> key:string -> data:Yojson.Safe.t -> ttl:float option -> unit

     

       107
        
       

     

       108
        
       (** {2 Hierarchical Keys} *)

     

       109
        
       

     
···

       169
        
       (** Convert TTL in hours to seconds *)

     

       170
        
       val hours_to_ttl : int -> float

     

       171
        
       

     

       172
       +
       (** Check if a timestamp is expired given an optional TTL.

     

       173
       +
           Returns false if TTL is None (never expires). *)

     

       174
       +
       val is_expired : created_at:Ptime.t -> ttl:float option -> bool

     

       175
        
       

     

       176
        
       (** Format cache size for display *)

     

       177
        
       val format_size : int64 -> string

+1 -1

cacheio/lib/dune stack/cacheio/lib/dune

···

       1
        
       (library

     

       2
        
        (public_name cacheio)

     

       3
        
        (name cacheio)

     

       4
       -
        (libraries eio cmdliner yojson ptime ptime.clock.os logs fmt xdg))

···

       1
        
       (library

     

       2
        
        (public_name cacheio)

     

       3
        
        (name cacheio)

     

       4
       +
        (libraries eio cmdliner yojson ptime ptime.clock.os logs fmt xdge))

cacheio/xdg-eio stack/cacheio/xdg-eio

requests/.gitignore stack/requests/.gitignore

requests/CLAUDE.md stack/requests/CLAUDE.md

requests/DESIGN.md stack/requests/DESIGN.md

requests/bin/dune stack/requests/bin/dune

requests/bin/ocurl.ml stack/requests/bin/ocurl.ml

requests/dune-project stack/requests/dune-project

requests/lib/dune stack/requests/lib/dune

requests/lib/requests.ml stack/requests/lib/requests.ml

requests/lib/requests.mli stack/requests/lib/requests.mli

requests/requests.opam stack/requests/requests.opam

requests/test/dune stack/requests/test/dune

requests/test/test_requests.ml stack/requests/test/test_requests.ml

+266

stack/toru/lib/toru/cache.ml

···

       1
       +
       (** Cache module for managing local file storage

     

       2
       +
           Now uses cacheio for the underlying cache implementation *)

     

       3
       +
       

     

       4
       +
       (** File info: size in bytes and modification time *)

     

       5
       +
       type file_info = {

     

       6
       +
         size: int64;

     

       7
       +
         mtime: float;

     

       8
       +
       }

     

       9
       +
       

     

       10
       +
       (** Cache usage statistics *)

     

       11
       +
       type usage_stats = {

     

       12
       +
         total_size: int64;

     

       13
       +
         file_count: int;

     

       14
       +
         oldest: float;

     

       15
       +
         newest: float;

     

       16
       +
       }

     

       17
       +
       

     

       18
       +
       type t = {

     

       19
       +
         cacheio : Cacheio.t;

     

       20
       +
         base_path : Eio.Fs.dir_ty Eio.Path.t;

     

       21
       +
         version : string option;

     

       22
       +
         sw : Eio.Switch.t;

     

       23
       +
         app_name : string;

     

       24
       +
       }

     

       25
       +
       

     

       26
       +
       let create ~sw ~fs ?version path_str =

     

       27
       +
         let base_path = Eio.Path.(fs / path_str) in

     

       28
       +
         let app_name = match version with

     

       29
       +
           | Some v -> Printf.sprintf "toru-%s" v

     

       30
       +
           | None -> "toru"

     

       31
       +
         in

     

       32
       +
       

     

       33
       +
         (* Create cacheio FileStorage backend *)

     

       34
       +
         let storage = Cacheio.FileStorage.create ~base_dir:base_path ~app_name () in

     

       35
       +
         let cacheio = Cacheio.create ~storage:(`File storage) in

     

       36
       +
       

     

       37
       +
         { cacheio; base_path; version; sw; app_name }

     

       38
       +
       

     

       39
       +
       let default_cache_path ?app_name () =

     

       40
       +
         let app_name = Option.value app_name ~default:"toru" in

     

       41
       +
         (* Using xdge library for XDG paths - but this needs env at runtime *)

     

       42
       +
         (* For now, use a simple default *)

     

       43
       +
         Filename.concat (Filename.concat (Sys.getenv "HOME") ".cache") app_name

     

       44
       +
       

     

       45
       +
       let base_path t = t.base_path

     

       46
       +
       let version t = t.version

     

       47
       +
       

     

       48
       +
       let cache_path t =

     

       49
       +
         Option.fold t.version ~none:t.base_path

     

       50
       +
           ~some:(fun v -> Eio.Path.(t.base_path / v))

     

       51
       +
       

     

       52
       +
       let file_path t filename =

     

       53
       +
         Option.fold t.version ~none:Eio.Path.(t.base_path / filename)

     

       54
       +
           ~some:(fun v -> Eio.Path.(t.base_path / v / filename))

     

       55
       +
       

     

       56
       +
       let exists t filename =

     

       57
       +
         (* Check if the file exists in cacheio *)

     

       58
       +
         Cacheio.exists t.cacheio ~key:filename

     

       59
       +
       

     

       60
       +
       let exists_path path =

     

       61
       +
         try

     

       62
       +
           let _stat = Eio.Path.stat ~follow:false path in

     

       63
       +
           true

     

       64
       +
         with

     

       65
       +
         | _ -> false

     

       66
       +
       

     

       67
       +
       let ensure_dir t =

     

       68
       +
         let create_dir_recursive path =

     

       69
       +
           if not (exists_path path) then

     

       70
       +
             try

     

       71
       +
               Eio.Path.mkdir path ~perm:0o755

     

       72
       +
             with

     

       73
       +
             | _ -> () (* Directory may already exist or creation failed *)

     

       74
       +
         in

     

       75
       +
         (* Create base directory first *)

     

       76
       +
         create_dir_recursive t.base_path;

     

       77
       +
         (* If version is specified, create version subdirectory *)

     

       78
       +
         Option.iter (fun v ->

     

       79
       +
           let version_path = Eio.Path.(t.base_path / v) in

     

       80
       +
           create_dir_recursive version_path) t.version

     

       81
       +
       

     

       82
       +
       let clear t =

     

       83
       +
         (* Clear all entries from cacheio *)

     

       84
       +
         Cacheio.clear t.cacheio

     

       85
       +
       

     

       86
       +
       let size_bytes t =

     

       87
       +
         let stats = Cacheio.stats t.cacheio in

     

       88
       +
         stats.size

     

       89
       +
       

     

       90
       +
       let list_files t =

     

       91
       +
         Cacheio.list_keys t.cacheio

     

       92
       +
       

     

       93
       +
       (** Get file info (size and mtime) *)

     

       94
       +
       let file_info t filename =

     

       95
       +
         let path = file_path t filename in

     

       96
       +
         try

     

       97
       +
           let stat = Eio.Path.stat ~follow:false path in

     

       98
       +
           match stat.kind with

     

       99
       +
           | `Regular_file ->

     

       100
       +
               let size = Optint.Int63.to_int64 stat.size in

     

       101
       +
               let mtime = stat.mtime in

     

       102
       +
               Some { size; mtime }

     

       103
       +
           | _ -> None

     

       104
       +
         with

     

       105
       +
         | _ -> None

     

       106
       +
       

     

       107
       +
       (** Get cache usage statistics *)

     

       108
       +
       let usage_stats t =

     

       109
       +
         let stats = Cacheio.stats t.cacheio in

     

       110
       +
         let cache_dir = cache_path t in

     

       111
       +
       

     

       112
       +
         (* For oldest/newest, we need to scan the actual files *)

     

       113
       +
         let rec collect_timestamps path acc_oldest acc_newest =

     

       114
       +
           if not (exists_path path) then (acc_oldest, acc_newest)

     

       115
       +
           else

     

       116
       +
             match Eio.Path.read_dir path with

     

       117
       +
             | [] -> (acc_oldest, acc_newest)

     

       118
       +
             | entries ->

     

       119
       +
                 List.fold_left (fun (oldest, newest) entry ->

     

       120
       +
                   let entry_path = Eio.Path.(path / entry) in

     

       121
       +
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       122
       +
                   match stat.kind with

     

       123
       +
                   | `Regular_file ->

     

       124
       +
                       let mtime = stat.mtime in

     

       125
       +
                       let new_oldest = if oldest = 0.0 || mtime < oldest then mtime else oldest in

     

       126
       +
                       let new_newest = if newest = 0.0 || mtime > newest then mtime else newest in

     

       127
       +
                       (new_oldest, new_newest)

     

       128
       +
                   | `Directory ->

     

       129
       +
                       collect_timestamps entry_path oldest newest

     

       130
       +
                   | _ -> (oldest, newest)

     

       131
       +
                 ) (acc_oldest, acc_newest) entries

     

       132
       +
         in

     

       133
       +
       

     

       134
       +
         let (oldest, newest) = collect_timestamps cache_dir 0.0 0.0 in

     

       135
       +
       

     

       136
       +
         {

     

       137
       +
           total_size = stats.size;

     

       138
       +
           file_count = stats.entries;

     

       139
       +
           oldest;

     

       140
       +
           newest;

     

       141
       +
         }

     

       142
       +
       

     

       143
       +
       (** Remove oldest files to fit within size limit *)

     

       144
       +
       let trim_to_size t max_size =

     

       145
       +
         let cache_dir = cache_path t in

     

       146
       +
         let rec collect_files_with_stats path prefix acc =

     

       147
       +
           if not (exists_path path) then acc

     

       148
       +
           else

     

       149
       +
             match Eio.Path.read_dir path with

     

       150
       +
             | [] -> acc

     

       151
       +
             | entries ->

     

       152
       +
                 List.fold_left (fun files entry ->

     

       153
       +
                   let entry_path = Eio.Path.(path / entry) in

     

       154
       +
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       155
       +
                   match stat.kind with

     

       156
       +
                   | `Regular_file ->

     

       157
       +
                       let full_name = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       158
       +
                       let size = Optint.Int63.to_int64 stat.size in

     

       159
       +
                       let mtime = stat.mtime in

     

       160
       +
                       (full_name, entry_path, size, mtime) :: files

     

       161
       +
                   | `Directory ->

     

       162
       +
                       let new_prefix = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       163
       +
                       collect_files_with_stats entry_path new_prefix files

     

       164
       +
                   | _ -> files

     

       165
       +
                 ) acc entries

     

       166
       +
         in

     

       167
       +
       

     

       168
       +
         let files = collect_files_with_stats cache_dir "" [] in

     

       169
       +
       

     

       170
       +
         (* Sort by mtime (oldest first) *)

     

       171
       +
         let sorted_files = List.sort (fun (_, _, _, a) (_, _, _, b) -> Float.compare a b) files in

     

       172
       +
       

     

       173
       +
         (* Calculate total size and remove oldest until under limit *)

     

       174
       +
         let rec remove_until_fit current_size files_to_check =

     

       175
       +
           if current_size <= max_size then ()

     

       176
       +
           else

     

       177
       +
             match files_to_check with

     

       178
       +
             | [] -> ()

     

       179
       +
             | (name, path, size, _) :: rest ->

     

       180
       +
                 Eio.Path.unlink path;

     

       181
       +
                 (* Also remove from cacheio *)

     

       182
       +
                 Cacheio.delete t.cacheio ~key:name;

     

       183
       +
                 remove_until_fit (Int64.sub current_size size) rest

     

       184
       +
         in

     

       185
       +
       

     

       186
       +
         let total = List.fold_left (fun acc (_, _, size, _) -> Int64.add acc size) 0L files in

     

       187
       +
         remove_until_fit total sorted_files

     

       188
       +
       

     

       189
       +
       (** Remove files older than N days *)

     

       190
       +
       let trim_by_age t max_age_days =

     

       191
       +
         let max_age_seconds = max_age_days *. 86400.0 in

     

       192
       +
         let now = Unix.time () in

     

       193
       +
       

     

       194
       +
         let cache_dir = cache_path t in

     

       195
       +
         let rec remove_old_files path prefix =

     

       196
       +
           if not (exists_path path) then ()

     

       197
       +
           else

     

       198
       +
             match Eio.Path.read_dir path with

     

       199
       +
             | [] -> ()

     

       200
       +
             | entries ->

     

       201
       +
                 List.iter (fun entry ->

     

       202
       +
                   let entry_path = Eio.Path.(path / entry) in

     

       203
       +
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       204
       +
                   match stat.kind with

     

       205
       +
                   | `Regular_file ->

     

       206
       +
                       let age = now -. stat.mtime in

     

       207
       +
                       if age > max_age_seconds then begin

     

       208
       +
                         Eio.Path.unlink entry_path;

     

       209
       +
                         let full_name = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       210
       +
                         Cacheio.delete t.cacheio ~key:full_name

     

       211
       +
                       end

     

       212
       +
                   | `Directory ->

     

       213
       +
                       let new_prefix = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       214
       +
                       remove_old_files entry_path new_prefix

     

       215
       +
                   | _ -> ()

     

       216
       +
                 ) entries

     

       217
       +
         in

     

       218
       +
       

     

       219
       +
         remove_old_files cache_dir ""

     

       220
       +
       

     

       221
       +
       (** Remove empty directories and broken links *)

     

       222
       +
       let vacuum t =

     

       223
       +
         let cache_dir = cache_path t in

     

       224
       +
         let rec clean_dir path =

     

       225
       +
           if not (exists_path path) then false

     

       226
       +
           else

     

       227
       +
             match Eio.Path.read_dir path with

     

       228
       +
             | [] ->

     

       229
       +
                 (* Empty directory, can be removed *)

     

       230
       +
                 (try Eio.Path.rmdir path; true with _ -> false)

     

       231
       +
             | entries ->

     

       232
       +
                 let all_removed = List.fold_left (fun acc entry ->

     

       233
       +
                   let entry_path = Eio.Path.(path / entry) in

     

       234
       +
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       235
       +
                   match stat.kind with

     

       236
       +
                   | `Directory -> clean_dir entry_path && acc

     

       237
       +
                   | `Symbolic_link ->

     

       238
       +
                       (* Check if link is broken *)

     

       239
       +
                       (try

     

       240
       +
                          let _ = Eio.Path.stat ~follow:true entry_path in

     

       241
       +
                          false

     

       242
       +
                        with _ ->

     

       243
       +
                          (* Broken link, remove it *)

     

       244
       +
                          Eio.Path.unlink entry_path;

     

       245
       +
                          acc)

     

       246
       +
                   | _ -> false

     

       247
       +
                 ) true entries in

     

       248
       +
       

     

       249
       +
                 if all_removed && path <> cache_dir then

     

       250
       +
                   (* All entries removed and not the root cache dir *)

     

       251
       +
                   (try Eio.Path.rmdir path; true with _ -> false)

     

       252
       +
                 else

     

       253
       +
                   false

     

       254
       +
         in

     

       255
       +
       

     

       256
       +
         let _ = clean_dir cache_dir in

     

       257
       +
         ()

     

       258
       +
       

     

       259
       +
       (** Pretty printer for cache *)

     

       260
       +
       let pp fmt t =

     

       261
       +
         let stats = usage_stats t in

     

       262
       +
         Format.fprintf fmt "Cache(path=%s, version=%s, files=%d, size=%s)"

     

       263
       +
           (Eio.Path.native_exn t.base_path)

     

       264
       +
           (Option.value t.version ~default:"none")

     

       265
       +
           stats.file_count

     

       266
       +
           (Cacheio.format_size stats.total_size)

+346

stack/toru/lib/toru/downloader.ml

···

       1
       +
       (** Downloader module - now using requests library for HTTP operations *)

     

       2
       +
       

     

       3
       +
       module Progress_reporter = struct

     

       4
       +
         type t = {

     

       5
       +
           name : string;

     

       6
       +
           total_bytes : int64 option;

     

       7
       +
           mutable current_bytes : int64;

     

       8
       +
           mutable last_percent : int;

     

       9
       +
         }

     

       10
       +
       

     

       11
       +
         let create ?total_bytes name =

     

       12
       +
           Printf.printf "Starting download: %s\n%!" name;

     

       13
       +
           { name; total_bytes; current_bytes = 0L; last_percent = -1 }

     

       14
       +
       

     

       15
       +
         let update t bytes =

     

       16
       +
           t.current_bytes <- bytes;

     

       17
       +
           match t.total_bytes with

     

       18
       +
           | Some total when total > 0L ->

     

       19
       +
               let percent = Int64.(to_int (div (mul bytes 100L) total)) in

     

       20
       +
               if percent > t.last_percent && percent mod 10 = 0 then (

     

       21
       +
                 t.last_percent <- percent;

     

       22
       +
                 Printf.printf "\r%s: %d%% (%Ld/%Ld bytes)%!"

     

       23
       +
                   t.name percent bytes total

     

       24
       +
               )

     

       25
       +
           | _ ->

     

       26
       +
               (* Unknown size, just show bytes downloaded *)

     

       27
       +
               if Int64.rem bytes 1048576L = 0L then (* Every MB *)

     

       28
       +
                 Printf.printf "\r%s: %Ld bytes downloaded%!" t.name bytes

     

       29
       +
       

     

       30
       +
         let finish t =

     

       31
       +
           Printf.printf "\n%s: Complete (%Ld bytes)\n%!" t.name t.current_bytes

     

       32
       +
       end

     

       33
       +
       

     

       34
       +
       module Config = struct

     

       35
       +
         type auth = {

     

       36
       +
           username : string option;

     

       37
       +
           password : string option;

     

       38
       +
         }

     

       39
       +
       end

     

       40
       +
       

     

       41
       +
       module type DOWNLOADER = sig

     

       42
       +
         type t

     

       43
       +
       

     

       44
       +
         val create : sw:Eio.Switch.t -> env:Eio_unix.Stdenv.base ->

     

       45
       +
                      ?auth:Config.auth -> unit -> t

     

       46
       +
       

     

       47
       +
         val download : t ->

     

       48
       +
           url:string ->

     

       49
       +
           dest:Eio.Fs.dir_ty Eio.Path.t ->

     

       50
       +
           ?hash:Hash.t ->

     

       51
       +
           ?progress:Progress_reporter.t ->

     

       52
       +
           ?resume:bool ->

     

       53
       +
           unit -> (unit, string) result

     

       54
       +
       

     

       55
       +
         val supports_resume : t -> bool

     

       56
       +
         val name : t -> string

     

       57
       +
       end

     

       58
       +
       

     

       59
       +
       (** HTTP downloader using requests library *)

     

       60
       +
       module Http_downloader = struct

     

       61
       +
         type t = {

     

       62
       +
           sw : Eio.Switch.t;

     

       63
       +
           env : Eio_unix.Stdenv.base;

     

       64
       +
           net : Eio_unix.Stdenv.base;

     

       65
       +
           auth : Requests.Auth.t option;

     

       66
       +
         }

     

       67
       +
       

     

       68
       +
         let create ~sw ~env ?auth () =

     

       69
       +
           (* Convert auth config to Requests.Auth.t *)

     

       70
       +
           let auth = Option.map (fun a ->

     

       71
       +
             match a.Config.username, a.Config.password with

     

       72
       +
             | Some u, Some p -> Requests.Auth.basic ~username:u ~password:p

     

       73
       +
             | _ -> Requests.Auth.none

     

       74
       +
           ) auth in

     

       75
       +
       

     

       76
       +
           { sw; env; net = env; auth }

     

       77
       +
       

     

       78
       +
         let download t ~url ~dest ?hash ?progress ?(resume=false) () =

     

       79
       +
           let _ = resume in  (* TODO: implement resume support with range requests *)

     

       80
       +
           try

     

       81
       +
             let uri = Uri.of_string url in

     

       82
       +
             let dest_path = Eio.Path.native_exn dest in

     

       83
       +
       

     

       84
       +
             (* Create a client for this request *)

     

       85
       +
             let client = Requests.create

     

       86
       +
               ~tls_config:(Requests.Tls.default ())

     

       87
       +
               ~clock:t.env#clock

     

       88
       +
               t.env#net

     

       89
       +
             in

     

       90
       +
       

     

       91
       +
             (* Configure request *)

     

       92
       +
             let config =

     

       93
       +
               let base_config = Requests.Config.create

     

       94
       +
                 ~timeout:300.0

     

       95
       +
                 ~follow_redirects:true

     

       96
       +
                 ~max_redirects:10

     

       97
       +
                 () in

     

       98
       +
               match t.auth with

     

       99
       +
               | Some auth -> Requests.Config.create ~auth ~timeout:300.0 ~follow_redirects:true ()

     

       100
       +
               | None -> base_config

     

       101
       +
             in

     

       102
       +
       

     

       103
       +
             (* Download the file *)

     

       104
       +
             Requests.download_file ~sw:t.sw client ~config uri ~path:dest_path;

     

       105
       +
       

     

       106
       +
             (* Update progress if provided *)

     

       107
       +
             Option.iter (fun p ->

     

       108
       +
               (* Get file size for final progress update *)

     

       109
       +
               let stat = Eio.Path.stat ~follow:false dest in

     

       110
       +
               let size = Optint.Int63.to_int64 stat.size in

     

       111
       +
               Progress_reporter.update p size;

     

       112
       +
               Progress_reporter.finish p

     

       113
       +
             ) progress;

     

       114
       +
       

     

       115
       +
             (* Verify hash if provided *)

     

       116
       +
             match hash with

     

       117
       +
             | Some h ->

     

       118
       +
                 if Hash.verify dest h then

     

       119
       +
                   Ok ()

     

       120
       +
                 else

     

       121
       +
                   Error ("Hash verification failed for " ^ dest_path)

     

       122
       +
             | None -> Ok ()

     

       123
       +
       

     

       124
       +
           with

     

       125
       +
           | Requests.Request_error err ->

     

       126
       +
               Error (Format.asprintf "HTTP download failed: %a" Requests.pp_error err)

     

       127
       +
           | exn ->

     

       128
       +
               Error ("Download failed: " ^ Printexc.to_string exn)

     

       129
       +
       

     

       130
       +
         let supports_resume _ = false  (* Requests library doesn't support resume yet *)

     

       131
       +
         let name _ = "http"

     

       132
       +
       end

     

       133
       +
       

     

       134
       +
       (** Legacy wget downloader for systems that prefer external tools *)

     

       135
       +
       module Wget_downloader = struct

     

       136
       +
         type t = {

     

       137
       +
           sw : Eio.Switch.t;

     

       138
       +
           env : Eio_unix.Stdenv.base;

     

       139
       +
           auth : Config.auth option;

     

       140
       +
           timeout : float;

     

       141
       +
         }

     

       142
       +
       

     

       143
       +
         let create ~sw ~env ?auth () = { sw; env; auth; timeout = 300.0 }

     

       144
       +
       

     

       145
       +
         let download t ~url ~dest ?hash ?progress:_ ?(resume=true) () =

     

       146
       +
           let dest_path = Eio.Path.native_exn dest in

     

       147
       +
       

     

       148
       +
           (* Build wget arguments *)

     

       149
       +
           let args = [

     

       150
       +
             "--quiet";

     

       151
       +
             "--show-progress";

     

       152
       +
             "--timeout=300";

     

       153
       +
             "--tries=3";

     

       154
       +
             "--output-document=" ^ dest_path;

     

       155
       +
           ] in

     

       156
       +
       

     

       157
       +
           (* Add authentication if provided *)

     

       158
       +
           let args = Option.fold t.auth ~none:args ~some:(fun auth ->

     

       159
       +
             let user_arg = Option.map (fun u -> "--user=" ^ u) auth.Config.username

     

       160
       +
               |> Option.to_list in

     

       161
       +
             let pass_arg = Option.map (fun p -> "--password=" ^ p) auth.Config.password

     

       162
       +
               |> Option.to_list in

     

       163
       +
             user_arg @ pass_arg @ args) in

     

       164
       +
       

     

       165
       +
           (* Add resume support if enabled *)

     

       166
       +
           let args = if resume then "--continue" :: args else args in

     

       167
       +
       

     

       168
       +
           (* Add URL as last argument *)

     

       169
       +
           let args = args @ [url] in

     

       170
       +
       

     

       171
       +
           (* Build command line with wget command *)

     

       172
       +
           let cmd_args = "wget" :: args in

     

       173
       +
       

     

       174
       +
           try

     

       175
       +
             (* Run wget using Eio process manager *)

     

       176
       +
             let cmd_line = String.concat " " (List.map Filename.quote cmd_args) in

     

       177
       +
             let process = Eio.Process.spawn t.env#process_mgr ~sw:t.sw

     

       178
       +
               ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd_line] in

     

       179
       +
             let exit_status = Eio.Process.await process in

     

       180
       +
             if exit_status <> `Exited 0 then

     

       181
       +
               let error_msg = match exit_status with

     

       182
       +
                 | `Exited n -> Printf.sprintf "wget exited with code %d" n

     

       183
       +
                 | `Signaled n -> Printf.sprintf "wget killed by signal %d" n in

     

       184
       +
               Error ("Download failed: " ^ error_msg)

     

       185
       +
             else (

     

       186
       +
               (* Verify hash if provided *)

     

       187
       +
               match hash with

     

       188
       +
               | Some h ->

     

       189
       +
                   if Hash.verify dest h then

     

       190
       +
                     Ok ()

     

       191
       +
                   else

     

       192
       +
                     Error ("Hash verification failed for " ^ dest_path)

     

       193
       +
               | None -> Ok ()

     

       194
       +
             )

     

       195
       +
           with

     

       196
       +
           | exn -> Error ("wget failed: " ^ (Printexc.to_string exn))

     

       197
       +
       

     

       198
       +
         let supports_resume _ = true

     

       199
       +
         let name _ = "wget"

     

       200
       +
       end

     

       201
       +
       

     

       202
       +
       (** Legacy curl downloader for systems that prefer external tools *)

     

       203
       +
       module Curl_downloader = struct

     

       204
       +
         type t = {

     

       205
       +
           sw : Eio.Switch.t;

     

       206
       +
           env : Eio_unix.Stdenv.base;

     

       207
       +
           auth : Config.auth option;

     

       208
       +
           timeout : float;

     

       209
       +
         }

     

       210
       +
       

     

       211
       +
         let create ~sw ~env ?auth () = { sw; env; auth; timeout = 300.0 }

     

       212
       +
       

     

       213
       +
         let download t ~url ~dest ?hash ?progress:_ ?(resume=true) () =

     

       214
       +
           let dest_path = Eio.Path.native_exn dest in

     

       215
       +
       

     

       216
       +
           (* Build curl arguments *)

     

       217
       +
           let args = [

     

       218
       +
             "--silent";

     

       219
       +
             "--show-error";

     

       220
       +
             "--location";

     

       221
       +
             "--max-time"; "300";

     

       222
       +
             "--retry"; "3";

     

       223
       +
             "--output"; dest_path;

     

       224
       +
           ] in

     

       225
       +
       

     

       226
       +
           (* Add authentication if provided *)

     

       227
       +
           let args = Option.fold t.auth ~none:args ~some:(fun auth ->

     

       228
       +
             let auth_str = match auth.Config.username, auth.Config.password with

     

       229
       +
               | Some user, Some pass -> Some (user ^ ":" ^ pass)

     

       230
       +
               | Some user, None -> Some user

     

       231
       +
               | None, _ -> None in

     

       232
       +
             match auth_str with

     

       233
       +
             | Some s -> "--user" :: s :: args

     

       234
       +
             | None -> args) in

     

       235
       +
       

     

       236
       +
           (* Add resume support if enabled *)

     

       237
       +
           let args = if resume then "--continue-at" :: "-" :: args else args in

     

       238
       +
       

     

       239
       +
           (* Add URL as last argument *)

     

       240
       +
           let args = args @ [url] in

     

       241
       +
       

     

       242
       +
           (* Build command line with curl command *)

     

       243
       +
           let cmd_args = "curl" :: args in

     

       244
       +
       

     

       245
       +
           try

     

       246
       +
             (* Run curl using Eio process manager *)

     

       247
       +
             let cmd_line = String.concat " " (List.map Filename.quote cmd_args) in

     

       248
       +
             let process = Eio.Process.spawn t.env#process_mgr ~sw:t.sw

     

       249
       +
               ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd_line] in

     

       250
       +
             let exit_status = Eio.Process.await process in

     

       251
       +
             if exit_status <> `Exited 0 then

     

       252
       +
               let error_msg = match exit_status with

     

       253
       +
                 | `Exited n -> Printf.sprintf "curl exited with code %d" n

     

       254
       +
                 | `Signaled n -> Printf.sprintf "curl killed by signal %d" n in

     

       255
       +
               Error ("Download failed: " ^ error_msg)

     

       256
       +
             else (

     

       257
       +
               (* Verify hash if provided *)

     

       258
       +
               match hash with

     

       259
       +
               | Some h ->

     

       260
       +
                   if Hash.verify dest h then

     

       261
       +
                     Ok ()

     

       262
       +
                   else

     

       263
       +
                     Error ("Hash verification failed for " ^ dest_path)

     

       264
       +
               | None -> Ok ()

     

       265
       +
             )

     

       266
       +
           with

     

       267
       +
           | exn -> Error ("curl failed: " ^ (Printexc.to_string exn))

     

       268
       +
       

     

       269
       +
         let supports_resume _ = true

     

       270
       +
         let name _ = "curl"

     

       271
       +
       end

     

       272
       +
       

     

       273
       +
       (** Cohttp_downloader is now an alias for Http_downloader *)

     

       274
       +
       module Cohttp_downloader = Http_downloader

     

       275
       +
       

     

       276
       +
       module Downloaders = struct

     

       277
       +
         let wget () = (module Wget_downloader : DOWNLOADER)

     

       278
       +
         let curl () = (module Curl_downloader : DOWNLOADER)

     

       279
       +
         let cohttp () = (module Http_downloader : DOWNLOADER)

     

       280
       +
         let http () = (module Http_downloader : DOWNLOADER)

     

       281
       +
       

     

       282
       +
         let detect_available ~env =

     

       283
       +
           let check_command cmd =

     

       284
       +
             try

     

       285
       +
               let sw = Eio.Switch.run @@ fun sw ->

     

       286
       +
                 let process = Eio.Process.spawn env#process_mgr ~sw

     

       287
       +
                   ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd ^ " --version > /dev/null 2>&1"] in

     

       288
       +
                 let exit_status = Eio.Process.await process in

     

       289
       +
                 exit_status = `Exited 0

     

       290
       +
               in sw

     

       291
       +
             with _ -> false

     

       292
       +
           in

     

       293
       +
       

     

       294
       +
           let available = ref [] in

     

       295
       +
       

     

       296
       +
           (* Always available - uses requests library *)

     

       297
       +
           available := ("http", http ()) :: !available;

     

       298
       +
       

     

       299
       +
           (* Check for external tools *)

     

       300
       +
           if check_command "wget" then

     

       301
       +
             available := ("wget", wget ()) :: !available;

     

       302
       +
       

     

       303
       +
           if check_command "curl" then

     

       304
       +
             available := ("curl", curl ()) :: !available;

     

       305
       +
       

     

       306
       +
           List.rev !available

     

       307
       +
       

     

       308
       +
         let create_default ~env =

     

       309
       +
           (* Default to http (requests library), fallback to wget/curl if needed *)

     

       310
       +
           match detect_available ~env with

     

       311
       +
           | [] -> failwith "No downloaders available"

     

       312
       +
           | (_, d) :: _ -> d

     

       313
       +
       

     

       314
       +
         let of_string name =

     

       315
       +
           match name with

     

       316
       +
           | "wget" -> Some (wget ())

     

       317
       +
           | "curl" -> Some (curl ())

     

       318
       +
           | "cohttp" | "http" -> Some (http ())

     

       319
       +
           | _ -> None

     

       320
       +
       end

     

       321
       +
       

     

       322
       +
       module Cli = struct

     

       323
       +
         type downloader_choice = [ `Wget | `Curl | `Cohttp | `Http | `Auto ]

     

       324
       +
       

     

       325
       +
         open Cmdliner

     

       326
       +
       

     

       327
       +
         let downloader_term =

     

       328
       +
           let choices = [

     

       329
       +
             ("wget", `Wget);

     

       330
       +
             ("curl", `Curl);

     

       331
       +
             ("cohttp", `Cohttp);

     

       332
       +
             ("http", `Http);

     

       333
       +
             ("auto", `Auto)

     

       334
       +
           ] in

     

       335
       +
           let doc = "Download tool to use. 'auto' detects available tools, 'http' uses built-in HTTP client." in

     

       336
       +
           let docv = "TOOL" in

     

       337
       +
           Arg.(value & opt (enum choices) `Auto & info ["downloader"; "d"] ~doc ~docv)

     

       338
       +
       

     

       339
       +
         let downloader_info = Arg.info ["downloader"; "d"]

     

       340
       +
       

     

       341
       +
         let create_downloader ~env = function

     

       342
       +
           | `Wget -> Downloaders.wget ()

     

       343
       +
           | `Curl -> Downloaders.curl ()

     

       344
       +
           | `Cohttp | `Http -> Downloaders.http ()

     

       345
       +
           | `Auto -> Downloaders.create_default ~env

     

       346
       +
       end

toru/.gitignore stack/toru/.gitignore

toru/CLAUDE.md stack/toru/CLAUDE.md

toru/README.md stack/toru/README.md

toru/TODO.md stack/toru/TODO.md

toru/bin/dune stack/toru/bin/dune

toru/bin/tessera_loader.ml stack/toru/bin/tessera_loader.ml

toru/bin/toru_cache.ml stack/toru/bin/toru_cache.ml

toru/bin/toru_main.ml stack/toru/bin/toru_main.ml

toru/bin/toru_make_registry.ml stack/toru/bin/toru_make_registry.ml

toru/bin/toru_make_registry_simple.ml stack/toru/bin/toru_make_registry_simple.ml

+2

toru/dune-project stack/toru/dune-project

···

       18
        
         fmt

     

       19
        
         ptime

     

       20
        
         xdge

     

       0
        
       
     

       0
        
       
     

       21
        
         (logs (>= 0.7.0))

     

       22
        
         unix)

     

       23
        
        (authors "Toru Development Team")

···

       18
        
         fmt

     

       19
        
         ptime

     

       20
        
         xdge

     

       21
       +
         cacheio

     

       22
       +
         requests

     

       23
        
         (logs (>= 0.7.0))

     

       24
        
         unix)

     

       25
        
        (authors "Toru Development Team")

-289

toru/lib/toru/cache.ml

···

       1
       -
       (** File info: size in bytes and modification time *)

     

       2
       -
       type file_info = {

     

       3
       -
         size: int64;

     

       4
       -
         mtime: float;

     

       5
       -
       }

     

       6
       -
       

     

       7
       -
       (** Cache usage statistics *)

     

       8
       -
       type usage_stats = {

     

       9
       -
         total_size: int64;

     

       10
       -
         file_count: int;

     

       11
       -
         oldest: float;

     

       12
       -
         newest: float;

     

       13
       -
       }

     

       14
       -
       

     

       15
       -
       type t = {

     

       16
       -
         base_path : Eio.Fs.dir_ty Eio.Path.t;

     

       17
       -
         version : string option;

     

       18
       -
         sw : Eio.Switch.t;

     

       19
       -
       }

     

       20
       -
       

     

       21
       -
       let create ~sw ~fs ?version path_str =

     

       22
       -
         let base_path = Eio.Path.(fs / path_str) in

     

       23
       -
         { base_path; version; sw }

     

       24
       -
       

     

       25
       -
       let default_cache_path ?app_name () =

     

       26
       -
         let app_name = Option.value app_name ~default:"toru" in

     

       27
       -
         let xdg_dirs = Xdg.create ~env:Sys.getenv_opt () in

     

       28
       -
         let cache_dir = Xdg.cache_dir xdg_dirs in

     

       29
       -
         Filename.concat cache_dir app_name

     

       30
       -
       

     

       31
       -
       let base_path t = t.base_path

     

       32
       -
       let version t = t.version

     

       33
       -
       

     

       34
       -
       let cache_path t =

     

       35
       -
         Option.fold t.version ~none:t.base_path

     

       36
       -
           ~some:(fun v -> Eio.Path.(t.base_path / v))

     

       37
       -
       

     

       38
       -
       let file_path t filename =

     

       39
       -
         Option.fold t.version ~none:Eio.Path.(t.base_path / filename)

     

       40
       -
           ~some:(fun v -> Eio.Path.(t.base_path / v / filename))

     

       41
       -
       

     

       42
       -
       let exists t filename =

     

       43
       -
         let path = file_path t filename in

     

       44
       -
         (* TODO: Use Eio.Path.exists when available *)

     

       45
       -
         try

     

       46
       -
           let _stat = Eio.Path.stat ~follow:false path in

     

       47
       -
           true

     

       48
       -
         with

     

       49
       -
         | _ -> false

     

       50
       -
       

     

       51
       -
       let exists_path path =

     

       52
       -
         try

     

       53
       -
           let _stat = Eio.Path.stat ~follow:false path in

     

       54
       -
           true

     

       55
       -
         with

     

       56
       -
         | _ -> false

     

       57
       -
       

     

       58
       -
       let ensure_dir t =

     

       59
       -
         let create_dir_recursive path =

     

       60
       -
           if not (exists_path path) then

     

       61
       -
             try

     

       62
       -
               (* Try to create parent directory first *)

     

       63
       -
               (* Skip parent creation for now, rely on mkdir -p behavior if available *)

     

       64
       -
               Eio.Path.mkdir path ~perm:0o755

     

       65
       -
             with

     

       66
       -
             | _ -> () (* Directory may already exist or creation failed *)

     

       67
       -
         in

     

       68
       -
         (* Create base directory first *)

     

       69
       -
         create_dir_recursive t.base_path;

     

       70
       -
         (* If version is specified, create version subdirectory *)

     

       71
       -
         Option.iter (fun v -> 

     

       72
       -
           let version_path = Eio.Path.(t.base_path / v) in

     

       73
       -
           create_dir_recursive version_path) t.version

     

       74
       -
       

     

       75
       -
       let clear t =

     

       76
       -
         let cache_dir = cache_path t in

     

       77
       -
         let rec remove_contents path =

     

       78
       -
           match Eio.Path.read_dir path with

     

       79
       -
           | [] -> ()

     

       80
       -
           | entries ->

     

       81
       -
               List.iter (fun entry ->

     

       82
       -
                 let entry_path = Eio.Path.(path / entry) in

     

       83
       -
                 let stat = Eio.Path.stat ~follow:false entry_path in

     

       84
       -
                 match stat.kind with

     

       85
       -
                 | `Directory -> 

     

       86
       -
                     remove_contents entry_path;

     

       87
       -
                     Eio.Path.rmdir entry_path

     

       88
       -
                 | `Regular_file | `Symbolic_link -> 

     

       89
       -
                     Eio.Path.unlink entry_path

     

       90
       -
                 | _ -> () (* Skip other file types *)

     

       91
       -
               ) entries

     

       92
       -
         in

     

       93
       -
         if exists_path cache_dir then

     

       94
       -
           remove_contents cache_dir

     

       95
       -
       

     

       96
       -
       let size_bytes t =

     

       97
       -
         let cache_dir = cache_path t in

     

       98
       -
         let rec calculate_size path acc =

     

       99
       -
           if not (exists_path path) then acc

     

       100
       -
           else

     

       101
       -
             match Eio.Path.read_dir path with

     

       102
       -
             | [] -> acc

     

       103
       -
             | entries ->

     

       104
       -
                 List.fold_left (fun total entry ->

     

       105
       -
                   let entry_path = Eio.Path.(path / entry) in

     

       106
       -
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       107
       -
                   match stat.kind with

     

       108
       -
                   | `Regular_file -> Int64.add total (Optint.Int63.to_int64 stat.size)

     

       109
       -
                   | `Directory -> calculate_size entry_path total

     

       110
       -
                   | _ -> total

     

       111
       -
                 ) acc entries

     

       112
       -
         in

     

       113
       -
         calculate_size cache_dir 0L

     

       114
       -
       

     

       115
       -
       let list_files t =

     

       116
       -
         let cache_dir = cache_path t in

     

       117
       -
         let rec collect_files path prefix acc =

     

       118
       -
           if not (exists_path path) then acc

     

       119
       -
           else

     

       120
       -
             match Eio.Path.read_dir path with

     

       121
       -
             | [] -> acc

     

       122
       -
             | entries ->

     

       123
       -
                 List.fold_left (fun files entry ->

     

       124
       -
                   let entry_path = Eio.Path.(path / entry) in

     

       125
       -
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       126
       -
                   match stat.kind with

     

       127
       -
                   | `Regular_file -> 

     

       128
       -
                       let full_name = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       129
       -
                       full_name :: files

     

       130
       -
                   | `Directory ->

     

       131
       -
                       let new_prefix = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       132
       -
                       collect_files entry_path new_prefix files

     

       133
       -
                   | _ -> files

     

       134
       -
                 ) acc entries

     

       135
       -
         in

     

       136
       -
         List.rev (collect_files cache_dir "" [])

     

       137
       -
       

     

       138
       -
       (** Get file info (size and mtime) *)

     

       139
       -
       let file_info t filename =

     

       140
       -
         let path = file_path t filename in

     

       141
       -
         try

     

       142
       -
           let stat = Eio.Path.stat ~follow:false path in

     

       143
       -
           match stat.kind with

     

       144
       -
           | `Regular_file -> 

     

       145
       -
               let size = Optint.Int63.to_int64 stat.size in

     

       146
       -
               let mtime = stat.mtime in

     

       147
       -
               Some { size; mtime }

     

       148
       -
           | _ -> None

     

       149
       -
         with

     

       150
       -
         | _ -> None

     

       151
       -
       

     

       152
       -
       (** Get cache usage statistics *)

     

       153
       -
       let usage_stats t =

     

       154
       -
         let cache_dir = cache_path t in

     

       155
       -
         let rec collect_stats path acc_size acc_count acc_oldest acc_newest =

     

       156
       -
           if not (exists_path path) then (acc_size, acc_count, acc_oldest, acc_newest)

     

       157
       -
           else

     

       158
       -
             match Eio.Path.read_dir path with

     

       159
       -
             | [] -> (acc_size, acc_count, acc_oldest, acc_newest)

     

       160
       -
             | entries ->

     

       161
       -
                 List.fold_left (fun (total_size, file_count, oldest, newest) entry ->

     

       162
       -
                   let entry_path = Eio.Path.(path / entry) in

     

       163
       -
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       164
       -
                   match stat.kind with

     

       165
       -
                   | `Regular_file -> 

     

       166
       -
                       let size = Optint.Int63.to_int64 stat.size in

     

       167
       -
                       let mtime = stat.mtime in

     

       168
       -
                       let new_oldest = if oldest = 0.0 || mtime < oldest then mtime else oldest in

     

       169
       -
                       let new_newest = if newest = 0.0 || mtime > newest then mtime else newest in

     

       170
       -
                       (Int64.add total_size size, file_count + 1, new_oldest, new_newest)

     

       171
       -
                   | `Directory ->

     

       172
       -
                       collect_stats entry_path total_size file_count oldest newest

     

       173
       -
                   | _ -> (total_size, file_count, oldest, newest)

     

       174
       -
                 ) (acc_size, acc_count, acc_oldest, acc_newest) entries

     

       175
       -
         in

     

       176
       -
         let (total_size, file_count, oldest, newest) = collect_stats cache_dir 0L 0 0.0 0.0 in

     

       177
       -
         { total_size; file_count; oldest; newest }

     

       178
       -
       

     

       179
       -
       (** Remove oldest files to fit within size limit *)

     

       180
       -
       let trim_to_size t max_size =

     

       181
       -
         let cache_dir = cache_path t in

     

       182
       -
         let rec collect_files_with_stats path prefix acc =

     

       183
       -
           if not (exists_path path) then acc

     

       184
       -
           else

     

       185
       -
             match Eio.Path.read_dir path with

     

       186
       -
             | [] -> acc

     

       187
       -
             | entries ->

     

       188
       -
                 List.fold_left (fun files entry ->

     

       189
       -
                   let entry_path = Eio.Path.(path / entry) in

     

       190
       -
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       191
       -
                   match stat.kind with

     

       192
       -
                   | `Regular_file -> 

     

       193
       -
                       let full_name = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       194
       -
                       let size = Optint.Int63.to_int64 stat.size in

     

       195
       -
                       let mtime = stat.mtime in

     

       196
       -
                       (full_name, entry_path, size, mtime) :: files

     

       197
       -
                   | `Directory ->

     

       198
       -
                       let new_prefix = if prefix = "" then entry else prefix ^ "/" ^ entry in

     

       199
       -
                       collect_files_with_stats entry_path new_prefix files

     

       200
       -
                   | _ -> files

     

       201
       -
                 ) acc entries

     

       202
       -
         in

     

       203
       -
         let files = collect_files_with_stats cache_dir "" [] in

     

       204
       -
         let total_size = List.fold_left (fun acc (_, _, size, _) -> Int64.add acc size) 0L files in

     

       205
       -
         if Int64.compare total_size max_size > 0 then (

     

       206
       -
           (* Sort by modification time (oldest first) *)

     

       207
       -
           let sorted_files = List.sort (fun (_, _, _, mtime1) (_, _, _, mtime2) -> 

     

       208
       -
             Float.compare mtime1 mtime2) files in

     

       209
       -
           let rec remove_files remaining_files current_size =

     

       210
       -
             if Int64.compare current_size max_size <= 0 then ()

     

       211
       -
             else

     

       212
       -
               match remaining_files with

     

       213
       -
               | [] -> ()

     

       214
       -
               | (_, path, size, _) :: rest ->

     

       215
       -
                   (try Eio.Path.unlink path with _ -> ());

     

       216
       -
                   remove_files rest (Int64.sub current_size size)

     

       217
       -
           in

     

       218
       -
           remove_files sorted_files total_size

     

       219
       -
         )

     

       220
       -
       

     

       221
       -
       (** Remove files older than N days *)

     

       222
       -
       let trim_by_age t max_age_days =

     

       223
       -
         let cache_dir = cache_path t in

     

       224
       -
         let current_time = Unix.time () in

     

       225
       -
         let max_age_seconds = max_age_days *. 86400.0 in (* days to seconds *)

     

       226
       -
         let rec remove_old_files path =

     

       227
       -
           if exists_path path then

     

       228
       -
             match Eio.Path.read_dir path with

     

       229
       -
             | [] -> ()

     

       230
       -
             | entries ->

     

       231
       -
                 List.iter (fun entry ->

     

       232
       -
                   let entry_path = Eio.Path.(path / entry) in

     

       233
       -
                   let stat = Eio.Path.stat ~follow:false entry_path in

     

       234
       -
                   match stat.kind with

     

       235
       -
                   | `Regular_file -> 

     

       236
       -
                       let file_age = current_time -. stat.mtime in

     

       237
       -
                       if file_age > max_age_seconds then (

     

       238
       -
                         try Eio.Path.unlink entry_path with _ -> ()

     

       239
       -
                       )

     

       240
       -
                   | `Directory ->

     

       241
       -
                       remove_old_files entry_path

     

       242
       -
                   | _ -> ()

     

       243
       -
                 ) entries

     

       244
       -
         in

     

       245
       -
         remove_old_files cache_dir

     

       246
       -
       

     

       247
       -
       (** Remove empty directories and broken links *)

     

       248
       -
       let vacuum t =

     

       249
       -
         let cache_dir = cache_path t in

     

       250
       -
         let rec vacuum_directory path =

     

       251
       -
           if exists_path path then

     

       252
       -
             match Eio.Path.read_dir path with

     

       253
       -
             | [] -> 

     

       254
       -
                 (* Try to remove empty directory if it's not the base cache path *)

     

       255
       -
                 if path <> cache_dir then (

     

       256
       -
                   try Eio.Path.rmdir path with _ -> ()

     

       257
       -
                 )

     

       258
       -
             | entries ->

     

       259
       -
                 List.iter (fun entry ->

     

       260
       -
                   let entry_path = Eio.Path.(path / entry) in

     

       261
       -
                   try

     

       262
       -
                     let stat = Eio.Path.stat ~follow:false entry_path in

     

       263
       -
                     match stat.kind with

     

       264
       -
                     | `Directory -> vacuum_directory entry_path

     

       265
       -
                     | `Symbolic_link ->

     

       266
       -
                         (* Check if symlink is broken *)

     

       267
       -
                         (try

     

       268
       -
                            let _ = Eio.Path.stat ~follow:true entry_path in ()

     

       269
       -
                          with

     

       270
       -
                          | _ -> Eio.Path.unlink entry_path)

     

       271
       -
                     | _ -> ()

     

       272
       -
                   with

     

       273
       -
                   | _ -> 

     

       274
       -
                       (* If we can't stat it, it might be broken - try to remove *)

     

       275
       -
                       (try Eio.Path.unlink entry_path with _ -> ())

     

       276
       -
                 ) entries;

     

       277
       -
                 (* Check again if directory is now empty *)

     

       278
       -
                 (match Eio.Path.read_dir path with

     

       279
       -
                  | [] when path <> cache_dir -> 

     

       280
       -
                      (try Eio.Path.rmdir path with _ -> ())

     

       281
       -
                  | _ -> ())

     

       282
       -
         in

     

       283
       -
         vacuum_directory cache_dir

     

       284
       -
       

     

       285
       -
       let pp fmt t =

     

       286
       -
         let version_str = Option.fold t.version ~none:"no version" 

     

       287
       -
           ~some:(fun v -> "version " ^ v) in

     

       288
       -
         Format.fprintf fmt "Cache at %s (%s)" 

     

       289
       -
           (Eio.Path.native_exn t.base_path) version_str

toru/lib/toru/cache.mli stack/toru/lib/toru/cache.mli

-268

toru/lib/toru/downloader.ml

···

       1
       -
       module Progress_reporter = struct

     

       2
       -
         type t = {

     

       3
       -
           name : string;

     

       4
       -
           total_bytes : int64 option;

     

       5
       -
           mutable current_bytes : int64;

     

       6
       -
           mutable last_percent : int;

     

       7
       -
         }

     

       8
       -
         

     

       9
       -
         let create ?total_bytes name = 

     

       10
       -
           Printf.printf "Starting download: %s\n%!" name;

     

       11
       -
           { name; total_bytes; current_bytes = 0L; last_percent = -1 }

     

       12
       -
         

     

       13
       -
         let update t bytes = 

     

       14
       -
           t.current_bytes <- bytes;

     

       15
       -
           match t.total_bytes with

     

       16
       -
           | Some total when total > 0L ->

     

       17
       -
               let percent = Int64.(to_int (div (mul bytes 100L) total)) in

     

       18
       -
               if percent > t.last_percent && percent mod 10 = 0 then (

     

       19
       -
                 t.last_percent <- percent;

     

       20
       -
                 Printf.printf "\r%s: %d%% (%Ld/%Ld bytes)%!" 

     

       21
       -
                   t.name percent bytes total

     

       22
       -
               )

     

       23
       -
           | _ ->

     

       24
       -
               (* Unknown size, just show bytes downloaded *)

     

       25
       -
               if Int64.rem bytes 1048576L = 0L then (* Every MB *)

     

       26
       -
                 Printf.printf "\r%s: %Ld bytes downloaded%!" t.name bytes

     

       27
       -
         

     

       28
       -
         let finish t = 

     

       29
       -
           Printf.printf "\n%s: Complete (%Ld bytes)\n%!" t.name t.current_bytes

     

       30
       -
       end

     

       31
       -
       

     

       32
       -
       module Config = struct

     

       33
       -
         type auth = {

     

       34
       -
           username : string option;

     

       35
       -
           password : string option;

     

       36
       -
         }

     

       37
       -
       end

     

       38
       -
       

     

       39
       -
       module type DOWNLOADER = sig

     

       40
       -
         type t

     

       41
       -
         

     

       42
       -
         val create : sw:Eio.Switch.t -> env:Eio_unix.Stdenv.base -> 

     

       43
       -
                      ?auth:Config.auth -> unit -> t

     

       44
       -
         

     

       45
       -
         val download : t ->

     

       46
       -
           url:string ->

     

       47
       -
           dest:Eio.Fs.dir_ty Eio.Path.t ->

     

       48
       -
           ?hash:Hash.t ->

     

       49
       -
           ?progress:Progress_reporter.t ->

     

       50
       -
           ?resume:bool ->

     

       51
       -
           unit -> (unit, string) result

     

       52
       -
           

     

       53
       -
         val supports_resume : t -> bool

     

       54
       -
         val name : t -> string

     

       55
       -
       end

     

       56
       -
       

     

       57
       -
       module Wget_downloader = struct

     

       58
       -
         type t = {

     

       59
       -
           sw : Eio.Switch.t;

     

       60
       -
           env : Eio_unix.Stdenv.base;

     

       61
       -
           auth : Config.auth option;

     

       62
       -
           timeout : float;

     

       63
       -
         }

     

       64
       -
         

     

       65
       -
         let create ~sw ~env ?auth () = { sw; env; auth; timeout = 300.0 }

     

       66
       -
         

     

       67
       -
         let download t ~url ~dest ?hash ?progress:_ ?(resume=true) () =

     

       68
       -
           let dest_path = Eio.Path.native_exn dest in

     

       69
       -
           

     

       70
       -
           (* Build wget arguments (excluding command name) *)

     

       71
       -
           let args = [

     

       72
       -
             "--quiet";                    (* Reduce output noise *)

     

       73
       -
             "--show-progress";            (* Show progress bar *)

     

       74
       -
             "--timeout=300";              (* 5 minute timeout *)

     

       75
       -
             "--tries=3";                  (* Retry 3 times *)

     

       76
       -
             "--output-document=" ^ dest_path;  (* Output file *)

     

       77
       -
           ] in

     

       78
       -
           

     

       79
       -
           (* Add authentication if provided *)

     

       80
       -
           let args = Option.fold t.auth ~none:args ~some:(fun auth ->

     

       81
       -
             let user_arg = Option.map (fun u -> "--user=" ^ u) auth.Config.username

     

       82
       -
               |> Option.to_list in

     

       83
       -
             let pass_arg = Option.map (fun p -> "--password=" ^ p) auth.Config.password

     

       84
       -
               |> Option.to_list in

     

       85
       -
             user_arg @ pass_arg @ args) in

     

       86
       -
           

     

       87
       -
           (* Add resume support if enabled *)

     

       88
       -
           let args = if resume then "--continue" :: args else args in

     

       89
       -
           

     

       90
       -
           (* Add URL as last argument *)

     

       91
       -
           let args = args @ [url] in

     

       92
       -
           

     

       93
       -
           (* Build command line with wget command *)

     

       94
       -
           let cmd_args = "wget" :: args in

     

       95
       -
           

     

       96
       -
           try

     

       97
       -
             (* Run wget using Eio process manager - use shell to handle PATH *)

     

       98
       -
             let cmd_line = String.concat " " (List.map Filename.quote cmd_args) in

     

       99
       -
             let process = Eio.Process.spawn t.env#process_mgr ~sw:t.sw

     

       100
       -
               ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd_line] in

     

       101
       -
             let exit_status = Eio.Process.await process in

     

       102
       -
             if exit_status <> `Exited 0 then

     

       103
       -
               let error_msg = match exit_status with 

     

       104
       -
                 | `Exited n -> Printf.sprintf "wget exited with code %d" n

     

       105
       -
                 | `Signaled n -> Printf.sprintf "wget killed by signal %d" n in

     

       106
       -
               Error ("Download failed: " ^ error_msg)

     

       107
       -
             else (

     

       108
       -
               (* Verify hash if provided *)

     

       109
       -
               match hash with

     

       110
       -
               | Some h -> 

     

       111
       -
                   if Hash.verify dest h then 

     

       112
       -
                     Ok () 

     

       113
       -
                   else 

     

       114
       -
                     Error ("Hash verification failed for " ^ dest_path)

     

       115
       -
               | None -> Ok ()

     

       116
       -
             )

     

       117
       -
           with

     

       118
       -
           | exn -> Error ("wget failed: " ^ (Printexc.to_string exn))

     

       119
       -
           

     

       120
       -
         let supports_resume _ = true

     

       121
       -
         let name _ = "wget"

     

       122
       -
       end

     

       123
       -
       

     

       124
       -
       module Curl_downloader = struct

     

       125
       -
         type t = {

     

       126
       -
           sw : Eio.Switch.t;

     

       127
       -
           env : Eio_unix.Stdenv.base;

     

       128
       -
           auth : Config.auth option;

     

       129
       -
           timeout : float;

     

       130
       -
         }

     

       131
       -
         

     

       132
       -
         let create ~sw ~env ?auth () = { sw; env; auth; timeout = 300.0 }

     

       133
       -
         

     

       134
       -
         let download t ~url ~dest ?hash ?progress:_ ?(resume=true) () =

     

       135
       -
           let dest_path = Eio.Path.native_exn dest in

     

       136
       -
           

     

       137
       -
           (* Build curl arguments (excluding command name) *)

     

       138
       -
           let args = [

     

       139
       -
             "--silent";                   (* Reduce output noise *)

     

       140
       -
             "--show-error";               (* Show error messages *)

     

       141
       -
             "--location";                 (* Follow redirects *)

     

       142
       -
             "--max-time"; "300";          (* 5 minute timeout *)

     

       143
       -
             "--retry"; "3";               (* Retry 3 times *)

     

       144
       -
             "--output"; dest_path;        (* Output file *)

     

       145
       -
           ] in

     

       146
       -
           

     

       147
       -
           (* Add authentication if provided *)

     

       148
       -
           let args = Option.fold t.auth ~none:args ~some:(fun auth ->

     

       149
       -
             let auth_str = match auth.Config.username, auth.Config.password with

     

       150
       -
               | Some user, Some pass -> Some (user ^ ":" ^ pass)

     

       151
       -
               | Some user, None -> Some user

     

       152
       -
               | None, _ -> None in

     

       153
       -
             Option.fold auth_str ~none:args ~some:(fun str -> "--user" :: str :: args)) in

     

       154
       -
           

     

       155
       -
           (* Add resume support if enabled *)

     

       156
       -
           let args = if resume then args @ ["--continue-at"; "-"] else args in

     

       157
       -
           

     

       158
       -
           (* Add URL as last argument *)

     

       159
       -
           let args = args @ [url] in

     

       160
       -
           

     

       161
       -
           (* Build command line with curl command *)

     

       162
       -
           let cmd_args = "curl" :: args in

     

       163
       -
           

     

       164
       -
           try

     

       165
       -
             (* Run curl using Eio process manager - use shell to handle PATH *)

     

       166
       -
             let cmd_line = String.concat " " (List.map Filename.quote cmd_args) in

     

       167
       -
             let process = Eio.Process.spawn t.env#process_mgr ~sw:t.sw

     

       168
       -
               ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd_line] in

     

       169
       -
             let exit_status = Eio.Process.await process in

     

       170
       -
             if exit_status <> `Exited 0 then

     

       171
       -
               let error_msg = match exit_status with 

     

       172
       -
                 | `Exited n -> Printf.sprintf "curl exited with code %d" n

     

       173
       -
                 | `Signaled n -> Printf.sprintf "curl killed by signal %d" n in

     

       174
       -
               Error ("Download failed: " ^ error_msg)

     

       175
       -
             else (

     

       176
       -
               (* Verify hash if provided *)

     

       177
       -
               match hash with

     

       178
       -
               | Some h -> 

     

       179
       -
                   if Hash.verify dest h then 

     

       180
       -
                     Ok () 

     

       181
       -
                   else 

     

       182
       -
                     Error ("Hash verification failed for " ^ dest_path)

     

       183
       -
               | None -> Ok ()

     

       184
       -
             )

     

       185
       -
           with

     

       186
       -
           | exn -> Error ("curl failed: " ^ (Printexc.to_string exn))

     

       187
       -
           

     

       188
       -
         let supports_resume _ = true  

     

       189
       -
         let name _ = "curl"

     

       190
       -
       end

     

       191
       -
       

     

       192
       -
       module Cohttp_downloader = struct

     

       193
       -
         type t = {

     

       194
       -
           sw : Eio.Switch.t;

     

       195
       -
           env : Eio_unix.Stdenv.base;

     

       196
       -
           auth : Config.auth option;

     

       197
       -
           timeout : float;

     

       198
       -
         }

     

       199
       -
         

     

       200
       -
         let create ~sw ~env ?auth () = { sw; env; auth; timeout = 300.0 }

     

       201
       -
         

     

       202
       -
         let download _t ~url:_ ~dest:_ ?hash:_ ?progress:_ ?resume:_ () =

     

       203
       -
           Error "Cohttp_downloader.download not yet implemented"

     

       204
       -
           

     

       205
       -
         let supports_resume _ = false

     

       206
       -
         let name _ = "cohttp-eio"

     

       207
       -
       end

     

       208
       -
       

     

       209
       -
       module Downloaders = struct

     

       210
       -
         let wget () = (module Wget_downloader : DOWNLOADER)

     

       211
       -
         

     

       212
       -
         let curl () = (module Curl_downloader : DOWNLOADER)

     

       213
       -
         

     

       214
       -
         let cohttp () = (module Cohttp_downloader : DOWNLOADER)

     

       215
       -
         

     

       216
       -
         let detect_available ~env =

     

       217
       -
           let test_command cmd =

     

       218
       -
             try

     

       219
       -
               Eio.Switch.run @@ fun sw ->

     

       220
       -
                 (* Use 'which' command to check if command exists on PATH *)

     

       221
       -
                 let process = Eio.Process.spawn env#process_mgr ~sw 

     

       222
       -
                   ~executable:"/bin/sh" ["/bin/sh"; "-c"; "which " ^ cmd ^ " >/dev/null 2>&1"] in

     

       223
       -
                 let exit_status = Eio.Process.await process in

     

       224
       -
                 exit_status = `Exited 0

     

       225
       -
             with

     

       226
       -
             | _ -> false

     

       227
       -
           in

     

       228
       -
           [("wget", (module Wget_downloader : DOWNLOADER)); 

     

       229
       -
            ("curl", (module Curl_downloader : DOWNLOADER))]

     

       230
       -
           |> List.filter (fun (cmd, _) -> test_command cmd)

     

       231
       -
         

     

       232
       -
         let create_default ~env =

     

       233
       -
           let available = detect_available ~env in

     

       234
       -
           match available with

     

       235
       -
           | (name, downloader) :: _ -> 

     

       236
       -
               Printf.eprintf "Using %s downloader\n" name;

     

       237
       -
               downloader

     

       238
       -
           | [] -> 

     

       239
       -
               failwith "No downloaders available (wget or curl required)"

     

       240
       -
         

     

       241
       -
         let of_string name =

     

       242
       -
           [("wget", (module Wget_downloader : DOWNLOADER));

     

       243
       -
            ("curl", (module Curl_downloader : DOWNLOADER))]

     

       244
       -
           |> List.assoc_opt name

     

       245
       -
       end

     

       246
       -
       

     

       247
       -
       module Cli = struct

     

       248
       -
         type downloader_choice = [ `Wget | `Curl | `Cohttp | `Auto ]

     

       249
       -
         

     

       250
       -
         let downloader_term =

     

       251
       -
           let open Cmdliner in

     

       252
       -
           let doc = "Download tool to use. 'auto' detects available tools." in

     

       253
       -
           let docv = "TOOL" in

     

       254
       -
           Arg.(value & opt (enum [

     

       255
       -
             ("wget", `Wget); ("curl", `Curl); 

     

       256
       -
             ("cohttp", `Cohttp); ("auto", `Auto)

     

       257
       -
           ]) `Auto & info ["downloader"; "d"] ~doc ~docv)

     

       258
       -
       

     

       259
       -
         let downloader_info = 

     

       260
       -
           Cmdliner.Arg.info ["downloader"; "d"] 

     

       261
       -
             ~doc:"Download tool to use"

     

       262
       -
       

     

       263
       -
         let create_downloader ~env = function

     

       264
       -
           | `Wget -> Downloaders.wget ()

     

       265
       -
           | `Curl -> Downloaders.curl ()

     

       266
       -
           | `Cohttp -> Downloaders.cohttp ()

     

       267
       -
           | `Auto -> Downloaders.create_default ~env

     

       268
       -
       end

+2 -1

toru/lib/toru/downloader.mli stack/toru/lib/toru/downloader.mli

···

       54
        
         val wget : unit -> (module DOWNLOADER)

     

       55
        
         val curl : unit -> (module DOWNLOADER)

     

       56
        
         val cohttp : unit -> (module DOWNLOADER)

     

       0
        
       
     

       57
        
         

     

       58
        
         val detect_available : env:Eio_unix.Stdenv.base ->

     

       59
        
                               (string * (module DOWNLOADER)) list

     
···

       64
        
       

     

       65
        
       (** CLI integration *)

     

       66
        
       module Cli : sig

     

       67
       -
         type downloader_choice = [ `Wget | `Curl | `Cohttp | `Auto ]

     

       68
        
         

     

       69
        
         val downloader_term : downloader_choice Cmdliner.Term.t

     

       70
        
         val downloader_info : Cmdliner.Arg.info

···

       54
        
         val wget : unit -> (module DOWNLOADER)

     

       55
        
         val curl : unit -> (module DOWNLOADER)

     

       56
        
         val cohttp : unit -> (module DOWNLOADER)

     

       57
       +
         val http : unit -> (module DOWNLOADER)

     

       58
        
         

     

       59
        
         val detect_available : env:Eio_unix.Stdenv.base ->

     

       60
        
                               (string * (module DOWNLOADER)) list

     
···

       65
        
       

     

       66
        
       (** CLI integration *)

     

       67
        
       module Cli : sig

     

       68
       +
         type downloader_choice = [ `Wget | `Curl | `Cohttp | `Http | `Auto ]

     

       69
        
         

     

       70
        
         val downloader_term : downloader_choice Cmdliner.Term.t

     

       71
        
         val downloader_info : Cmdliner.Arg.info

+4 -2

toru/lib/toru/dune stack/toru/lib/toru/dune

···

       2
        
        (public_name toru)

     

       3
        
        (name toru)

     

       4
        
        (modules hash registry cache processors downloader make_registry toru logging)

     

       5
       -
        (libraries 

     

       6
        
          eio

     

       7
        
          eio.unix

     

       8
        
          digestif

     
···

       10
        
          cmdliner

     

       11
        
          re

     

       12
        
          ptime

     

       13
       -
          xdg

     

       0
        
       
     

       0
        
       
     

       14
        
          unix

     

       15
        
          logs

     

       16
        
          logs.fmt

···

       2
        
        (public_name toru)

     

       3
        
        (name toru)

     

       4
        
        (modules hash registry cache processors downloader make_registry toru logging)

     

       5
       +
        (libraries

     

       6
        
          eio

     

       7
        
          eio.unix

     

       8
        
          digestif

     
···

       10
        
          cmdliner

     

       11
        
          re

     

       12
        
          ptime

     

       13
       +
          xdge

     

       14
       +
          cacheio

     

       15
       +
          requests

     

       16
        
          unix

     

       17
        
          logs

     

       18
        
          logs.fmt

toru/lib/toru/hash.ml stack/toru/lib/toru/hash.ml

toru/lib/toru/hash.mli stack/toru/lib/toru/hash.mli

toru/lib/toru/logging.ml stack/toru/lib/toru/logging.ml

toru/lib/toru/logging.mli stack/toru/lib/toru/logging.mli

toru/lib/toru/make_registry.ml stack/toru/lib/toru/make_registry.ml

toru/lib/toru/make_registry.mli stack/toru/lib/toru/make_registry.mli

toru/lib/toru/processors.ml stack/toru/lib/toru/processors.ml

toru/lib/toru/processors.mli stack/toru/lib/toru/processors.mli

toru/lib/toru/registry.ml stack/toru/lib/toru/registry.ml

toru/lib/toru/registry.mli stack/toru/lib/toru/registry.mli

toru/lib/toru/toru.ml stack/toru/lib/toru/toru.ml

toru/lib/toru/toru.mli stack/toru/lib/toru/toru.mli

toru/test/CACHE_IMPLEMENTATION_REPORT.md stack/toru/test/CACHE_IMPLEMENTATION_REPORT.md

toru/test/basic.t stack/toru/test/basic.t

toru/test/cli.t stack/toru/test/cli.t

toru/test/downloader_demo.ml stack/toru/test/downloader_demo.ml

toru/test/dune stack/toru/test/dune

toru/test/python/generate_pooch_registry.py stack/toru/test/python/generate_pooch_registry.py

toru/test/python/pyproject.toml stack/toru/test/python/pyproject.toml

toru/test/python/test_data/.hidden/secret.txt stack/toru/test/python/test_data/.hidden/secret.txt

toru/test/python/test_data/config.json stack/toru/test/python/test_data/config.json

toru/test/python/test_data/data/numbers.csv stack/toru/test/python/test_data/data/numbers.csv

toru/test/python/test_data/data/simple.txt stack/toru/test/python/test_data/data/simple.txt

toru/test/python/test_data/data/unicode.txt stack/toru/test/python/test_data/data/unicode.txt

toru/test/python/test_data/docs/readme.md stack/toru/test/python/test_data/docs/readme.md

toru/test/python/test_data/empty.txt stack/toru/test/python/test_data/empty.txt

toru/test/python/test_data/models/small.bin stack/toru/test/python/test_data/models/small.bin

toru/test/python/test_metadata.json stack/toru/test/python/test_metadata.json

toru/test/python/test_registry_md5.txt stack/toru/test/python/test_registry_md5.txt

toru/test/python/test_registry_mixed.txt stack/toru/test/python/test_registry_mixed.txt

toru/test/python/test_registry_sha1.txt stack/toru/test/python/test_registry_sha1.txt

toru/test/python/test_registry_sha256.txt stack/toru/test/python/test_registry_sha256.txt

toru/test/test_cache.ml stack/toru/test/test_cache.ml

toru/test/test_cache_xdg.ml stack/toru/test/test_cache_xdg.ml

toru/test/test_curl_download.ml stack/toru/test/test_curl_download.ml

toru/test/test_downloader.ml stack/toru/test/test_downloader.ml

toru/test/test_downloader_comprehensive.ml stack/toru/test/test_downloader_comprehensive.ml

toru/test/test_hash.ml stack/toru/test/test_hash.ml

toru/test/test_hash_manual.ml stack/toru/test/test_hash_manual.ml

toru/test/test_make_registry.ml stack/toru/test/test_make_registry.ml

toru/test/test_python_cross_validation.ml stack/toru/test/test_python_cross_validation.ml

toru/test/test_registry.ml stack/toru/test/test_registry.ml

toru/test/test_registry_real.ml stack/toru/test/test_registry_real.ml

toru/test/test_tessera_integration.ml stack/toru/test/test_tessera_integration.ml

toru/test/test_tessera_load.ml stack/toru/test/test_tessera_load.ml

toru/test/test_toru.ml stack/toru/test/test_toru.ml

toru/test/test_xdg_integration.ml stack/toru/test/test_xdg_integration.ml

xdg-eio/.gitignore stack/xdge/.gitignore

xdg-eio/.ocamlformat stack/xdge/.ocamlformat

xdg-eio/CLAUDE.md stack/xdge/CLAUDE.md

xdg-eio/dune-project stack/xdge/dune-project

xdg-eio/example/dune stack/xdge/example/dune

xdg-eio/example/minimal_test.cmi stack/xdge/example/minimal_test.cmi

xdg-eio/example/minimal_test.cmo stack/xdge/example/minimal_test.cmo

xdg-eio/example/xdg_example.ml stack/xdge/example/xdg_example.ml

xdg-eio/lib/dune stack/xdge/lib/dune

xdg-eio/lib/xdge.ml stack/xdge/lib/xdge.ml

xdg-eio/lib/xdge.mli stack/xdge/lib/xdge.mli

xdg-eio/test/dune stack/xdge/test/dune

xdg-eio/test/test_paths.ml stack/xdge/test/test_paths.ml

xdg-eio/test/xdg.t stack/xdge/test/xdg.t

xdg-eio/xdge.opam stack/xdge/xdge.opam