commit 925f7402396ef17c80848f955e0619874aeca2ab · anil.recoil.org/slop

stack/dune-project

···

       1
       1
       +
       (lang dune 3.20)

+12 -1

stack/toru/bin/dune

···

       1
       1
       +
       (executable

     

       2
       2
       +
        (public_name toru)

     

       3
       3
       +
        (name toru_cli)

     

       4
       4
       +
        (modules toru_cli)

     

       5
       5
       +
        (libraries toru.cmd))

     

       6
       6
       +
       

     

       1
       7
        
       (executable

     

       2
       8
        
        (public_name tessera-loader)

     

       3
       9
        
        (name tessera_loader)

     

       10
       10
       +
        (modules tessera_loader)

     

       4
       11
        
        (libraries toru cmdliner unix))

     

       5
       12
        
       

     

       6
       13
        
       (executable

     

       7
       14
        
        (public_name toru-cache)

     

       8
       15
        
        (name toru_cache)

     

       16
       16
       +
        (modules toru_cache)

     

       9
       17
        
        (libraries toru cmdliner yojson fmt fmt.tty ptime ptime.clock.os unix eio_main))

     

       10
       18
        
       

     

       11
       19
        
       (executable

     

       12
       20
        
        (public_name toru-make-registry-simple)

     

       13
       21
        
        (name toru_make_registry_simple)

     

       22
       22
       +
        (modules toru_make_registry_simple)

     

       14
       23
        
        (libraries toru cmdliner ptime ptime.clock.os eio_main))

     

       15
       24
        
       

     

       16
       25
        
       (executable

     

       17
       17
       -
        (public_name toru)

     

       26
       26
       +
        (public_name toru-main)

     

       18
       27
        
        (name toru_main)

     

       28
       28
       +
        (modules toru_main)

     

       19
       29
        
        (libraries toru cmdliner unix eio_main logs fmt xdge))

     

       20
       30
        
       

     

       21
       31
        
       ;; Complex version with enhanced features (disabled until field access is resolved)

     

       22
       32
        
       ;; (executable

     

       23
       33
        
       ;;  (public_name toru-make-registry)

     

       24
       34
        
       ;;  (name toru_make_registry)

     

       35
       35
       +
       ;;  (modules toru_make_registry)

     

       25
       36
        
       ;;  (libraries toru cmdliner yojson ptime ptime.clock.os unix eio_main))

stack/toru/bin/toru_cli.ml

···

       1
       1
       +
       (** Toru CLI - Command-line interface for Toru data management *)

     

       2
       2
       +
       

     

       3
       3
       +
       module Cmd = Toru_cmd.Cmd

     

       4
       4
       +
       

     

       5
       5
       +
       let () =

     

       6
       6
       +
         exit (Cmdliner.Cmd.eval' Cmd.main_cmd)

+90 -109

stack/toru/lib/toru/cache.ml

···

       1
       1
       -
       (** Cache module for managing local file storage

     

       2
       2
       -
           Uses the simplified cacheio library for storage *)

     

       1
       1
       +
       (** Cache management for Toru using cacheio directly

     

       3
       2
        
       

     

       4
       4
       -
       open Eio

     

       5
       5
       -
       

     

       6
       6
       -
       (** File info: size in bytes and modification time *)

     

       7
       7
       -
       type file_info = {

     

       8
       8
       -
         size: int64;

     

       9
       9
       -
         mtime: float;

     

       10
       10
       -
       }

     

       11
       11
       -
       

     

       12
       12
       -
       (** Cache usage statistics *)

     

       13
       13
       -
       type usage_stats = {

     

       14
       14
       -
         total_size: int64;

     

       15
       15
       -
         file_count: int;

     

       16
       16
       -
         oldest: float;

     

       17
       17
       -
         newest: float;

     

       18
       18
       -
       }

     

       3
       3
       +
           This module provides a thin layer over cacheio for Toru-specific

     

       4
       4
       +
           cache operations, maintaining compatibility with Pooch's cache structure.

     

       5
       5
       +
       *)

     

       19
       6
        
       

     

       20
       7
        
       type t = {

     

       21
       8
        
         cacheio : Cacheio.t;

     

       22
       22
       -
         base_path : Fs.dir_ty Path.t;

     

       9
       9
       +
         base_path : Eio.Fs.dir_ty Eio.Path.t;

     

       23
       10
        
         version : string option;

     

       24
       11
        
       }

     

       12
       12
       +
       

     

       13
       13
       +
       (** {1 Creation} *)

     

       25
       14
        
       

     

       26
       15
        
       let create ~sw:_ ~fs ?version path_str =

     

       27
       27
       -
         let base_path = Path.(fs / path_str) in

     

       16
       16
       +
         let base_path = Eio.Path.(fs / path_str) in

     

       28
       17
        
       

     

       29
       18
        
         (* Ensure base directory exists *)

     

       30
       30
       -
         (try Path.mkdir ~perm:0o755 base_path with _ -> ());

     

       19
       19
       +
         Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 base_path;

     

       31
       20
        
       

     

       32
       32
       -
         (* If version is specified, create versioned subdirectory *)

     

       21
       21
       +
         (* Create versioned subdirectory if specified *)

     

       33
       22
        
         let cache_dir = match version with

     

       34
       23
        
           | Some v ->

     

       35
       35
       -
               let versioned = Path.(base_path / v) in

     

       36
       36
       -
               (try Path.mkdir ~perm:0o755 versioned with _ -> ());

     

       24
       24
       +
               let versioned = Eio.Path.(base_path / v) in

     

       25
       25
       +
               Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 versioned;

     

       37
       26
        
               versioned

     

       38
       27
        
           | None -> base_path

     

       39
       28
        
         in

     
···

       41
       30
        
         let cacheio = Cacheio.create ~base_dir:cache_dir in

     

       42
       31
        
         { cacheio; base_path; version }

     

       43
       32
        
       

     

       44
       44
       -
       let default_cache_path ?app_name () =

     

       45
       45
       -
         let app_name = Option.value app_name ~default:"toru" in

     

       46
       46
       -
         (* Use XDG_CACHE_HOME if set, otherwise ~/.cache *)

     

       47
       47
       -
         let cache_home =

     

       48
       48
       -
           match Sys.getenv_opt "XDG_CACHE_HOME" with

     

       49
       49
       -
           | Some dir -> dir

     

       50
       50
       -
           | None ->

     

       51
       51
       -
               match Sys.getenv_opt "HOME" with

     

       52
       52
       -
               | Some home -> Filename.concat home ".cache"

     

       53
       53
       -
               | None -> "/tmp/.cache"

     

       54
       54
       -
         in

     

       55
       55
       -
         Filename.concat cache_home app_name

     

       33
       33
       +
       let default_cache_path ?(app_name="toru") () =

     

       34
       34
       +
         (* Use XDG_CACHE_HOME if set, otherwise platform-specific defaults *)

     

       35
       35
       +
         match Sys.getenv_opt "XDG_CACHE_HOME" with

     

       36
       36
       +
         | Some dir -> Filename.concat dir app_name

     

       37
       37
       +
         | None ->

     

       38
       38
       +
             match Sys.getenv_opt "HOME" with

     

       39
       39
       +
             | Some home ->

     

       40
       40
       +
                 if Sys.os_type = "Win32" || Sys.os_type = "Cygwin" then

     

       41
       41
       +
                   Filename.concat home (Filename.concat "AppData" (Filename.concat "Local" app_name))

     

       42
       42
       +
                 else

     

       43
       43
       +
                   Filename.concat home (Filename.concat ".cache" app_name)

     

       44
       44
       +
             | None -> Filename.concat "/tmp" app_name

     

       45
       45
       +
       

     

       46
       46
       +
       (** {1 Accessors} *)

     

       56
       47
        
       

     

       57
       48
        
       let base_path t = t.base_path

     

       58
       49
        
       let version t = t.version

     

       50
       50
       +
       let cacheio t = t.cacheio

     

       59
       51
        
       

     

       60
       60
       -
       let cache_path t =

     

       61
       61
       -
         match t.version with

     

       62
       62
       -
         | Some v -> Path.(t.base_path / v)

     

       63
       63
       -
         | None -> t.base_path

     

       52
       52
       +
       (** {1 File Operations} *)

     

       64
       53
        
       

     

       65
       54
        
       let file_path t filename =

     

       66
       66
       -
         (* For compatibility, we need to handle files that might be stored with their path structure *)

     

       67
       67
       -
         (* The new cacheio will hash the entire filename including any path separators *)

     

       68
       68
       -
         Path.(cache_path t / filename)

     

       55
       55
       +
         (* For Pooch compatibility, preserve the filename structure *)

     

       56
       56
       +
         Eio.Path.(

     

       57
       57
       +
           match t.version with

     

       58
       58
       +
           | Some v -> t.base_path / v / filename

     

       59
       59
       +
           | None -> t.base_path / filename

     

       60
       60
       +
         )

     

       69
       61
        
       

     

       70
       62
        
       let exists t filename =

     

       71
       63
        
         Cacheio.exists t.cacheio ~key:filename

     

       72
       64
        
       

     

       73
       73
       -
       let ensure_dir _ =

     

       74
       74
       -
         (* Base directory is already ensured during creation *)

     

       75
       75
       -
         ()

     

       65
       65
       +
       let get_stream t ~sw filename =

     

       66
       66
       +
         Cacheio.get t.cacheio ~key:filename ~sw

     

       67
       67
       +
       

     

       68
       68
       +
       let put_stream t ~source filename =

     

       69
       69
       +
         (* Store with no TTL for permanent caching *)

     

       70
       70
       +
         Cacheio.put t.cacheio ~key:filename ~source ()

     

       71
       71
       +
       

     

       72
       72
       +
       let delete t filename =

     

       73
       73
       +
         Cacheio.delete t.cacheio ~key:filename

     

       74
       74
       +
       

     

       75
       75
       +
       let size t filename =

     

       76
       76
       +
         Cacheio.size t.cacheio ~key:filename

     

       77
       77
       +
       

     

       78
       78
       +
       (** {1 Cache Management} *)

     

       79
       79
       +
       

     

       80
       80
       +
       let ensure_dir t =

     

       81
       81
       +
         (* Directory is ensured during creation *)

     

       82
       82
       +
         ignore t

     

       76
       83
        
       

     

       77
       84
        
       let clear t =

     

       78
       78
       -
         (* Clear all non-pinned entries *)

     

       79
       85
        
         Cacheio.clear t.cacheio

     

       80
       86
        
       

     

       87
       87
       +
       let stats t =

     

       88
       88
       +
         Cacheio.stats t.cacheio

     

       89
       89
       +
       

     

       90
       90
       +
       let expire t =

     

       91
       91
       +
         Cacheio.expire t.cacheio

     

       92
       92
       +
       

     

       93
       93
       +
       (** {1 Chunked Download Support} *)

     

       94
       94
       +
       

     

       95
       95
       +
       let put_chunk t ~filename ~range ~source =

     

       96
       96
       +
         Cacheio.put_chunk t.cacheio ~key:filename ~range ~source ()

     

       97
       97
       +
       

     

       98
       98
       +
       let get_chunks t ~filename =

     

       99
       99
       +
         Cacheio.list_chunks t.cacheio ~key:filename

     

       100
       100
       +
       

     

       101
       101
       +
       let has_complete_chunks t ~filename ~total_size =

     

       102
       102
       +
         Cacheio.has_complete_chunks t.cacheio ~key:filename ~total_size

     

       103
       103
       +
       

     

       104
       104
       +
       let missing_ranges t ~filename ~total_size =

     

       105
       105
       +
         let ranges = Cacheio.missing_ranges t.cacheio ~key:filename ~total_size in

     

       106
       106
       +
         List.map (fun range ->

     

       107
       107
       +
           (Cacheio.Range.start range, Cacheio.Range.end_ range)

     

       108
       108
       +
         ) ranges

     

       109
       109
       +
       

     

       110
       110
       +
       let coalesce_chunks t ~filename =

     

       111
       111
       +
         (* Mark as pinned when coalescing *)

     

       112
       112
       +
         let flags = Cacheio.Flags.add `Pinned Cacheio.Flags.empty in

     

       113
       113
       +
         Cacheio.coalesce_chunks t.cacheio ~key:filename ~flags ()

     

       114
       114
       +
       

     

       115
       115
       +
       (** {1 Utilities} *)

     

       116
       116
       +
       

     

       81
       117
        
       let size_bytes t =

     

       82
       118
        
         let stats = Cacheio.stats t.cacheio in

     

       83
       119
        
         Cacheio.Stats.total_size stats

     

       84
       120
        
       

     

       85
       121
        
       let list_files t =

     

       86
       122
        
         let entries = Cacheio.scan t.cacheio in

     

       87
       87
       -
         (* Return the original keys (which are hashes, but we'll treat them as filenames) *)

     

       88
       88
       -
         List.map Cacheio.Entry.key entries

     

       123
       123
       +
         List.map (fun entry -> Cacheio.Entry.key entry) entries

     

       89
       124
        
       

     

       90
       90
       -
       (** Get file information *)

     

       91
       91
       -
       let file_info t filename =

     

       92
       92
       -
         (* Use scan to find the entry and get its info *)

     

       93
       93
       -
         let entries = Cacheio.scan t.cacheio in

     

       94
       94
       -
         match List.find_opt (fun e -> Cacheio.Entry.key e = filename) entries with

     

       95
       95
       -
         | None -> None

     

       96
       96
       -
         | Some entry -> Some { size = Cacheio.Entry.size entry; mtime = Cacheio.Entry.mtime entry }

     

       125
       125
       +
       (** {1 Pretty Printing} *)

     

       97
       126
        
       

     

       98
       98
       -
       (** Get usage statistics *)

     

       99
       99
       -
       let usage_stats t =

     

       100
       100
       -
         let entries = Cacheio.scan t.cacheio in

     

       101
       101
       -
         let total_size = List.fold_left (fun acc e -> Int64.add acc (Cacheio.Entry.size e)) 0L entries in

     

       102
       102
       -
         let file_count = List.length entries in

     

       103
       103
       -
         let times = List.map Cacheio.Entry.mtime entries in

     

       104
       104
       -
         let oldest = List.fold_left min Float.max_float times in

     

       105
       105
       -
         let newest = List.fold_left max Float.min_float times in

     

       106
       106
       -
         { total_size; file_count; oldest; newest }

     

       107
       107
       -
       

     

       108
       108
       -
       (* Internal helper functions removed - functionality merged into public API functions *)

     

       109
       109
       -
       

     

       110
       110
       -
       (** Remove oldest files to fit within size limit *)

     

       111
       111
       -
       let trim_to_size t max_bytes =

     

       112
       112
       -
         let rec trim_until current_size =

     

       113
       113
       -
           if current_size <= max_bytes then ()

     

       114
       114
       -
           else

     

       115
       115
       -
             let entries = Cacheio.scan t.cacheio in

     

       116
       116
       -
             (* Sort by mtime, oldest first, excluding pinned *)

     

       117
       117
       -
             let unpinned = List.filter (fun e ->

     

       118
       118
       -
               not (Cacheio.Entry.is_pinned e)

     

       119
       119
       -
             ) entries in

     

       120
       120
       -
             let sorted = List.sort Cacheio.Entry.compare_by_mtime unpinned in

     

       121
       121
       -
             match sorted with

     

       122
       122
       -
             | [] -> () (* Nothing left to remove *)

     

       123
       123
       -
             | oldest :: _ ->

     

       124
       124
       -
                 Cacheio.delete t.cacheio ~key:(Cacheio.Entry.key oldest);

     

       125
       125
       -
                 let new_size = Int64.sub current_size (Cacheio.Entry.size oldest) in

     

       126
       126
       -
                 trim_until new_size

     

       127
       127
       -
         in

     

       128
       128
       -
         let current = size_bytes t in

     

       129
       129
       -
         trim_until current

     

       130
       130
       -
       

     

       131
       131
       -
       (** Remove files older than N days *)

     

       132
       132
       -
       let trim_by_age t days =

     

       133
       133
       -
         let cutoff = Unix.time () -. (days *. 86400.) in

     

       134
       134
       -
         let open Cacheio in

     

       135
       135
       -
         let entries = scan t.cacheio in

     

       136
       136
       -
         List.iter (fun e ->

     

       137
       137
       -
           if Cacheio.Entry.mtime e < cutoff && not (Cacheio.Entry.is_pinned e) then

     

       138
       138
       -
             Cacheio.delete t.cacheio ~key:(Cacheio.Entry.key e)

     

       139
       139
       -
         ) entries

     

       140
       140
       -
       

     

       141
       141
       -
       (** Remove empty directories and broken links *)

     

       142
       142
       -
       let vacuum t =

     

       143
       143
       -
         (* For now, just expire old entries - cacheio handles cleanup *)

     

       144
       144
       -
         let _expired = Cacheio.expire t.cacheio in

     

       145
       145
       -
         ()

     

       146
       146
       -
       

     

       147
       147
       -
       (** Pretty printer for cache *)

     

       148
       127
        
       let pp fmt t =

     

       149
       149
       -
         let stats = usage_stats t in

     

       128
       128
       +
         let stats = Cacheio.stats t.cacheio in

     

       129
       129
       +
         let entry_count = Cacheio.Stats.entry_count stats in

     

       130
       130
       +
         let total_size = Cacheio.Stats.total_size stats in

     

       150
       131
        
         Format.fprintf fmt "Cache[path=%s, version=%s, files=%d, size=%Ld]"

     

       151
       151
       -
           (Path.native_exn t.base_path)

     

       132
       132
       +
           (Eio.Path.native_exn t.base_path)

     

       152
       133
        
           (Option.value t.version ~default:"none")

     

       153
       153
       -
           stats.file_count

     

       154
       154
       -
           stats.total_size
     

       134
       134
       +
           entry_count

     

       135
       135
       +
           total_size

+44 -34

stack/toru/lib/toru/cache.mli

···

       1
       1
       -
       (** Cache module for managing local file storage *)

     

       1
       1
       +
       (** Cache module for managing local file storage using cacheio *)

     

       2
       2
        
       

     

       3
       3
        
       (** Abstract cache type *)

     

       4
       4
        
       type t

     

       5
       5
        
       

     

       6
       6
       -
       (** {1 Construction} *)

     

       6
       6
       +
       (** {1 Creation} *)

     

       7
       7
        
       

     

       8
       8
        
       (** Create cache with explicit path *)

     

       9
       9
        
       val create : sw:Eio.Switch.t -> fs:Eio.Fs.dir_ty Eio.Path.t ->

     

       10
       10
        
                    ?version:string -> string -> t

     

       11
       11
        
       

     

       12
       12
       -
       (** {1 Field accessors} *)

     

       12
       12
       +
       (** {1 Accessors} *)

     

       13
       13
        
       

     

       14
       14
        
       (** Get base path of cache *)

     

       15
       15
        
       val base_path : t -> Eio.Fs.dir_ty Eio.Path.t

     
···

       17
       17
        
       (** Get version string (if any) *)

     

       18
       18
        
       val version : t -> string option

     

       19
       19
        
       

     

       20
       20
       -
       (** {1 Operations} *)

     

       20
       20
       +
       (** Get underlying cacheio instance *)

     

       21
       21
       +
       val cacheio : t -> Cacheio.t

     

       22
       22
       +
       

     

       23
       23
       +
       (** {1 File Operations} *)

     

       21
       24
        
       

     

       22
       25
        
       (** Get full path for a filename within cache *)

     

       23
       26
        
       val file_path : t -> string -> Eio.Fs.dir_ty Eio.Path.t

     
···

       25
       28
        
       (** Check if file exists in cache *)

     

       26
       29
        
       val exists : t -> string -> bool

     

       27
       30
        
       

     

       31
       31
       +
       (** Get stream for reading cached file *)

     

       32
       32
       +
       val get_stream : t -> sw:Eio.Switch.t -> string -> Eio.Flow.source_ty Eio.Resource.t option

     

       33
       33
       +
       

     

       34
       34
       +
       (** Put stream to cache *)

     

       35
       35
       +
       val put_stream : t -> source:Eio.Flow.source_ty Eio.Resource.t -> string -> unit

     

       36
       36
       +
       

     

       37
       37
       +
       (** Delete file from cache *)

     

       38
       38
       +
       val delete : t -> string -> unit

     

       39
       39
       +
       

     

       40
       40
       +
       (** Get size of cached file *)

     

       41
       41
       +
       val size : t -> string -> int64 option

     

       42
       42
       +
       

     

       43
       43
       +
       (** {1 Cache Management} *)

     

       44
       44
       +
       

     

       28
       45
        
       (** Ensure cache directory exists *)

     

       29
       46
        
       val ensure_dir : t -> unit

     

       30
       47
        
       

     

       31
       48
        
       (** Clear all files from cache *)

     

       32
       49
        
       val clear : t -> unit

     

       33
       50
        
       

     

       34
       34
       -
       (** Get total size of cache in bytes *)

     

       35
       35
       -
       val size_bytes : t -> int64

     

       51
       51
       +
       (** Get cache statistics *)

     

       52
       52
       +
       val stats : t -> Cacheio.Stats.t

     

       36
       53
        
       

     

       37
       37
       -
       (** List all files in cache *)

     

       38
       38
       -
       val list_files : t -> string list

     

       39
       39
       -
       

     

       40
       40
       -
       (** {1 Cache Management} *)

     

       41
       41
       -
       

     

       42
       42
       -
       (** File info: size in bytes and modification time *)

     

       43
       43
       -
       type file_info = {

     

       44
       44
       -
         size: int64;

     

       45
       45
       -
         mtime: float;

     

       46
       46
       -
       }

     

       54
       54
       +
       (** Expire old cache entries *)

     

       55
       55
       +
       val expire : t -> int

     

       47
       56
        
       

     

       48
       48
       -
       (** Cache usage statistics *)

     

       49
       49
       -
       type usage_stats = {

     

       50
       50
       -
         total_size: int64;

     

       51
       51
       -
         file_count: int;

     

       52
       52
       -
         oldest: float;

     

       53
       53
       -
         newest: float;

     

       54
       54
       -
       }

     

       57
       57
       +
       (** {1 Chunked Download Support} *)

     

       55
       58
        
       

     

       56
       56
       -
       (** Remove oldest files to fit within size limit *)

     

       57
       57
       -
       val trim_to_size : t -> int64 -> unit

     

       59
       59
       +
       (** Put a chunk to cache *)

     

       60
       60
       +
       val put_chunk : t -> filename:string -> range:Cacheio.Range.t ->

     

       61
       61
       +
                       source:Eio.Flow.source_ty Eio.Resource.t -> unit

     

       58
       62
        
       

     

       59
       59
       -
       (** Remove files older than N days *)

     

       60
       60
       -
       val trim_by_age : t -> float -> unit

     

       63
       63
       +
       (** Get list of chunks for a file *)

     

       64
       64
       +
       val get_chunks : t -> filename:string -> Cacheio.Chunk.t list

     

       61
       65
        
       

     

       62
       62
       -
       (** Get file info (size and mtime) *)

     

       63
       63
       -
       val file_info : t -> string -> file_info option

     

       66
       66
       +
       (** Check if all chunks are present *)

     

       67
       67
       +
       val has_complete_chunks : t -> filename:string -> total_size:int64 -> bool

     

       64
       68
        
       

     

       65
       65
       -
       (** Get cache usage statistics *)

     

       66
       66
       -
       val usage_stats : t -> usage_stats

     

       69
       69
       +
       (** Get missing byte ranges *)

     

       70
       70
       +
       val missing_ranges : t -> filename:string -> total_size:int64 -> (int64 * int64) list

     

       67
       71
        
       

     

       68
       68
       -
       (** Remove empty directories and broken links *)

     

       69
       69
       -
       val vacuum : t -> unit

     

       72
       72
       +
       (** Coalesce chunks into complete file *)

     

       73
       73
       +
       val coalesce_chunks : t -> filename:string -> (unit, exn) result Eio.Promise.t

     

       70
       74
        
       

     

       71
       75
        
       (** {1 Utilities} *)

     

       72
       76
        
       

     

       73
       77
        
       (** Get default cache path for application *)

     

       74
       78
        
       val default_cache_path : ?app_name:string -> unit -> string

     

       75
       79
        
       

     

       80
       80
       +
       (** Get total size of cache in bytes *)

     

       81
       81
       +
       val size_bytes : t -> int64

     

       82
       82
       +
       

     

       83
       83
       +
       (** List all files in cache *)

     

       84
       84
       +
       val list_files : t -> string list

     

       85
       85
       +
       

     

       76
       86
        
       (** {1 Pretty printing} *)

     

       77
       87
        
       

     

       78
       88
        
       (** Pretty printer for cache *)

     

       79
       79
       -
       val pp : Format.formatter -> t -> unit

     

       89
       89
       +
       val pp : Format.formatter -> t -> unit

+221 -292

stack/toru/lib/toru/downloader.ml

···

       1
       1
       -
       (** Downloader module - now using requests library for HTTP operations *)

     

       1
       1
       +
       (** HTTP downloader module using requests library

     

       2
       2
       +
       

     

       3
       3
       +
           This module provides download functionality for Toru,

     

       4
       4
       +
           leveraging the requests library for all HTTP operations.

     

       5
       5
       +
       *)

     

       6
       6
       +
       

     

       7
       7
       +
       let src = Logs.Src.create "toru.downloader" ~doc:"Toru downloader"

     

       8
       8
       +
       module Log = (val Logs.src_log src : Logs.LOG)

     

       9
       9
       +
       

     

       10
       10
       +
       (** {1 Progress Reporting} *)

     

       2
       11
        
       

     

       3
       3
       -
       module Progress_reporter = struct

     

       12
       12
       +
       module Progress = struct

     

       4
       13
        
         type t = {

     

       5
       14
        
           name : string;

     

       6
       15
        
           total_bytes : int64 option;

     

       7
       16
        
           mutable current_bytes : int64;

     

       8
       8
       -
           mutable last_percent : int;

     

       17
       17
       +
           mutable last_report_time : float;

     

       18
       18
       +
           mutable last_report_bytes : int64;

     

       9
       19
        
         }

     

       10
       20
        
       

     

       11
       21
        
         let create ?total_bytes name =

     

       12
       12
       -
           Printf.printf "Starting download: %s\n%!" name;

     

       13
       13
       -
           { name; total_bytes; current_bytes = 0L; last_percent = -1 }

     

       22
       22
       +
           Log.info (fun m -> m "Starting download: %s" name);

     

       23
       23
       +
           {

     

       24
       24
       +
             name;

     

       25
       25
       +
             total_bytes;

     

       26
       26
       +
             current_bytes = 0L;

     

       27
       27
       +
             last_report_time = Unix.gettimeofday ();

     

       28
       28
       +
             last_report_bytes = 0L;

     

       29
       29
       +
           }

     

       14
       30
        
       

     

       15
       31
        
         let update t bytes =

     

       16
       32
        
           t.current_bytes <- bytes;

     

       17
       17
       -
           match t.total_bytes with

     

       18
       18
       -
           | Some total when total > 0L ->

     

       19
       19
       -
               let percent = Int64.(to_int (div (mul bytes 100L) total)) in

     

       20
       20
       -
               if percent > t.last_percent && percent mod 10 = 0 then (

     

       21
       21
       -
                 t.last_percent <- percent;

     

       22
       22
       -
                 Printf.printf "\r%s: %d%% (%Ld/%Ld bytes)%!"

     

       23
       23
       -
                   t.name percent bytes total

     

       24
       24
       -
               )

     

       25
       25
       -
           | _ ->

     

       26
       26
       -
               (* Unknown size, just show bytes downloaded *)

     

       27
       27
       -
               if Int64.rem bytes 1048576L = 0L then (* Every MB *)

     

       28
       28
       -
                 Printf.printf "\r%s: %Ld bytes downloaded%!" t.name bytes

     

       33
       33
       +
           let now = Unix.gettimeofday () in

     

       34
       34
       +
           let time_diff = now -. t.last_report_time in

     

       29
       35
        
       

     

       30
       30
       -
         let finish t =

     

       31
       31
       -
           Printf.printf "\n%s: Complete (%Ld bytes)\n%!" t.name t.current_bytes

     

       32
       32
       -
       end

     

       36
       36
       +
           (* Report every second *)

     

       37
       37
       +
           if time_diff >= 1.0 then begin

     

       38
       38
       +
             let bytes_diff = Int64.sub bytes t.last_report_bytes in

     

       39
       39
       +
             let speed = Int64.to_float bytes_diff /. time_diff in

     

       33
       40
        
       

     

       34
       34
       -
       module Config = struct

     

       35
       35
       -
         type auth = {

     

       36
       36
       -
           username : string option;

     

       37
       37
       -
           password : string option;

     

       38
       38
       -
         }

     

       39
       39
       -
       end

     

       41
       41
       +
             match t.total_bytes with

     

       42
       42
       +
             | Some total when total > 0L ->

     

       43
       43
       +
                 let percent = Int64.(to_int (div (mul bytes 100L) total)) in

     

       44
       44
       +
                 Log.info (fun m -> m "%s: %d%% (%Ld/%Ld bytes) - %.1f KB/s"

     

       45
       45
       +
                   t.name percent bytes total (speed /. 1024.0))

     

       46
       46
       +
             | _ ->

     

       47
       47
       +
                 Log.info (fun m -> m "%s: %Ld bytes downloaded - %.1f KB/s"

     

       48
       48
       +
                   t.name bytes (speed /. 1024.0));

     

       40
       49
        
       

     

       41
       41
       -
       module type DOWNLOADER = sig

     

       42
       42
       -
         type t

     

       50
       50
       +
             t.last_report_time <- now;

     

       51
       51
       +
             t.last_report_bytes <- bytes

     

       52
       52
       +
           end

     

       43
       53
        
       

     

       44
       44
       -
         val create : sw:Eio.Switch.t -> env:Eio_unix.Stdenv.base ->

     

       45
       45
       -
                      ?auth:Config.auth -> unit -> t

     

       46
       46
       -
       

     

       47
       47
       -
         val download : t ->

     

       48
       48
       -
           url:string ->

     

       49
       49
       -
           dest:Eio.Fs.dir_ty Eio.Path.t ->

     

       50
       50
       -
           ?hash:Hash.t ->

     

       51
       51
       -
           ?progress:Progress_reporter.t ->

     

       52
       52
       -
           ?resume:bool ->

     

       53
       53
       -
           unit -> (unit, string) result

     

       54
       54
       -
       

     

       55
       55
       -
         val supports_resume : t -> bool

     

       56
       56
       -
         val name : t -> string

     

       54
       54
       +
         let finish t =

     

       55
       55
       +
           Log.info (fun m -> m "%s: Download complete (%Ld bytes)" t.name t.current_bytes)

     

       57
       56
        
       end

     

       58
       57
        
       

     

       59
       59
       -
       (** HTTP downloader using requests library *)

     

       60
       60
       -
       module Http_downloader = struct

     

       61
       61
       -
         type t = {

     

       62
       62
       -
           sw : Eio.Switch.t;

     

       63
       63
       -
           env : Eio_unix.Stdenv.base;

     

       64
       64
       -
           net : Eio_unix.Stdenv.base;

     

       65
       65
       -
           auth : Requests.Auth.t option;

     

       66
       66
       -
         }

     

       58
       58
       +
       (** {1 Downloader Types} *)

     

       67
       59
        
       

     

       68
       68
       -
         let create ~sw ~env ?auth () =

     

       69
       69
       -
           (* Convert auth config to Requests.Auth.t *)

     

       70
       70
       -
           let auth = Option.map (fun a ->

     

       71
       71
       -
             match a.Config.username, a.Config.password with

     

       72
       72
       -
             | Some u, Some p -> Requests.Auth.basic ~username:u ~password:p

     

       73
       73
       -
             | _ -> Requests.Auth.none

     

       74
       74
       -
           ) auth in

     

       60
       60
       +
       type config = {

     

       61
       61
       +
         timeout : float;

     

       62
       62
       +
         max_redirects : int;

     

       63
       63
       +
         retry : Requests.Retry.config option;

     

       64
       64
       +
         auth : Requests.Auth.t option;

     

       65
       65
       +
         headers : Requests.Headers.t;

     

       66
       66
       +
         verify_tls : bool;

     

       67
       67
       +
       }

     

       75
       68
        
       

     

       76
       76
       -
           { sw; env; net = env; auth }

     

       69
       69
       +
       type ('clock, 'net) t_impl = {

     

       70
       70
       +
         session : ('clock, 'net) Requests.t;

     

       71
       71
       +
         config : config;

     

       72
       72
       +
       }

     

       77
       73
        
       

     

       78
       78
       -
         let download t ~url ~dest ?hash ?progress ?(resume=false) () =

     

       79
       79
       -
           let _ = resume in  (* TODO: implement resume support with range requests *)

     

       80
       80
       -
           try

     

       81
       81
       -
             let uri = Uri.of_string url in

     

       82
       82
       -
             let dest_path = Eio.Path.native_exn dest in

     

       74
       74
       +
       type t = T : ('clock Eio.Time.clock, 'net Eio.Net.t) t_impl -> t

     

       83
       75
        
       

     

       84
       84
       -
             (* Create a client for this request *)

     

       85
       85
       -
             let client = Requests.create

     

       86
       86
       -
               ~tls_config:(Requests.Tls.default ())

     

       87
       87
       -
               ~clock:t.env#clock

     

       88
       88
       -
               t.env#net

     

       89
       89
       -
             in

     

       76
       76
       +
       (** {1 Configuration} *)

     

       90
       77
        
       

     

       91
       91
       -
             (* Configure request *)

     

       92
       92
       -
             let config =

     

       93
       93
       -
               let base_config = Requests.Config.create

     

       94
       94
       -
                 ~timeout:300.0

     

       95
       95
       -
                 ~follow_redirects:true

     

       96
       96
       -
                 ~max_redirects:10

     

       97
       97
       -
                 () in

     

       98
       98
       -
               match t.auth with

     

       99
       99
       -
               | Some auth -> Requests.Config.create ~auth ~timeout:300.0 ~follow_redirects:true ()

     

       100
       100
       -
               | None -> base_config

     

       101
       101
       -
             in

     

       102
       102
       -
       

     

       103
       103
       -
             (* Download the file *)

     

       104
       104
       -
             Requests.download_file ~sw:t.sw client ~config uri ~path:dest;

     

       105
       105
       -
       

     

       106
       106
       -
             (* Update progress if provided *)

     

       107
       107
       -
             Option.iter (fun p ->

     

       108
       108
       -
               (* Get file size for final progress update *)

     

       109
       109
       -
               let stat = Eio.Path.stat ~follow:false dest in

     

       110
       110
       -
               let size = Optint.Int63.to_int64 stat.size in

     

       111
       111
       -
               Progress_reporter.update p size;

     

       112
       112
       -
               Progress_reporter.finish p

     

       113
       113
       -
             ) progress;

     

       114
       114
       -
       

     

       115
       115
       -
             (* Verify hash if provided *)

     

       116
       116
       -
             match hash with

     

       117
       117
       -
             | Some h ->

     

       118
       118
       -
                 if Hash.verify dest h then

     

       119
       119
       -
                   Ok ()

     

       120
       120
       -
                 else

     

       121
       121
       -
                   Error ("Hash verification failed for " ^ dest_path)

     

       122
       122
       -
             | None -> Ok ()

     

       123
       123
       -
       

     

       124
       124
       -
           with

     

       125
       125
       -
           | Requests.Request_error err ->

     

       126
       126
       -
               Error (Format.asprintf "HTTP download failed: %a" Requests.pp_error err)

     

       127
       127
       -
           | exn ->

     

       128
       128
       -
               Error ("Download failed: " ^ Printexc.to_string exn)

     

       129
       129
       -
       

     

       130
       130
       -
         let supports_resume _ = false  (* Requests library doesn't support resume yet *)

     

       131
       131
       -
         let name _ = "http"

     

       132
       132
       -
       end

     

       78
       78
       +
       let default_config = {

     

       79
       79
       +
         timeout = 300.0;

     

       80
       80
       +
         max_redirects = 10;

     

       81
       81
       +
         retry = Some (Requests.Retry.create_config ~max_retries:3 ~backoff_factor:1.0 ());

     

       82
       82
       +
         auth = None;

     

       83
       83
       +
         headers = Requests.Headers.empty;

     

       84
       84
       +
         verify_tls = true;

     

       85
       85
       +
       }

     

       133
       86
        
       

     

       134
       134
       -
       (** Legacy wget downloader for systems that prefer external tools *)

     

       135
       135
       -
       module Wget_downloader = struct

     

       136
       136
       -
         type t = {

     

       137
       137
       -
           sw : Eio.Switch.t;

     

       138
       138
       -
           env : Eio_unix.Stdenv.base;

     

       139
       139
       -
           auth : Config.auth option;

     

       140
       140
       -
           timeout : float;

     

       87
       87
       +
       let create_config ?timeout ?max_redirects ?retry ?auth ?headers ?verify_tls () =

     

       88
       88
       +
         let base = default_config in

     

       89
       89
       +
         {

     

       90
       90
       +
           timeout = Option.value timeout ~default:base.timeout;

     

       91
       91
       +
           max_redirects = Option.value max_redirects ~default:base.max_redirects;

     

       92
       92
       +
           retry = Option.value retry ~default:base.retry;

     

       93
       93
       +
           auth = Option.value auth ~default:base.auth;

     

       94
       94
       +
           headers = Option.value headers ~default:base.headers;

     

       95
       95
       +
           verify_tls = Option.value verify_tls ~default:base.verify_tls;

     

       141
       96
        
         }

     

       142
       97
        
       

     

       143
       143
       -
         let create ~sw ~env ?auth () = { sw; env; auth; timeout = 300.0 }

     

       98
       98
       +
       (** {1 Creation} *)

     

       144
       99
        
       

     

       145
       145
       -
         let download t ~url ~dest ?hash ?progress:_ ?(resume=true) () =

     

       146
       146
       -
           let dest_path = Eio.Path.native_exn dest in

     

       100
       100
       +
       let create ~sw ~env ?config () =

     

       101
       101
       +
         let config = Option.value config ~default:default_config in

     

       147
       102
        
       

     

       148
       148
       -
           (* Build wget arguments *)

     

       149
       149
       -
           let args = [

     

       150
       150
       -
             "--quiet";

     

       151
       151
       -
             "--show-progress";

     

       152
       152
       -
             "--timeout=300";

     

       153
       153
       -
             "--tries=3";

     

       154
       154
       -
             "--output-document=" ^ dest_path;

     

       155
       155
       -
           ] in

     

       103
       103
       +
         (* Create requests session with configuration *)

     

       104
       104
       +
         let session = Requests.create

     

       105
       105
       +
           ~sw

     

       106
       106
       +
           ~default_headers:config.headers

     

       107
       107
       +
           ?auth:config.auth

     

       108
       108
       +
           ~timeout:(Requests.Timeout.create ~total:config.timeout ())

     

       109
       109
       +
           ~follow_redirects:true

     

       110
       110
       +
           ~max_redirects:config.max_redirects

     

       111
       111
       +
           ~verify_tls:config.verify_tls

     

       112
       112
       +
           ?retry:config.retry

     

       113
       113
       +
           env

     

       114
       114
       +
         in

     

       156
       115
        
       

     

       157
       157
       -
           (* Add authentication if provided *)

     

       158
       158
       -
           let args = Option.fold t.auth ~none:args ~some:(fun auth ->

     

       159
       159
       -
             let user_arg = Option.map (fun u -> "--user=" ^ u) auth.Config.username

     

       160
       160
       -
               |> Option.to_list in

     

       161
       161
       -
             let pass_arg = Option.map (fun p -> "--password=" ^ p) auth.Config.password

     

       162
       162
       -
               |> Option.to_list in

     

       163
       163
       -
             user_arg @ pass_arg @ args) in

     

       116
       116
       +
         T { session; config }

     

       164
       117
        
       

     

       165
       165
       -
           (* Add resume support if enabled *)

     

       166
       166
       -
           let args = if resume then "--continue" :: args else args in

     

       118
       118
       +
       (** {1 Download Operations} *)

     

       167
       119
        
       

     

       168
       168
       -
           (* Add URL as last argument *)

     

       169
       169
       -
           let args = args @ [url] in

     

       120
       120
       +
       let download_to_stream (T t) ~url ~sink ?progress ?range () =

     

       121
       121
       +
         try

     

       122
       122
       +
           (* Add range header if resuming *)

     

       123
       123
       +
           let headers = match range with

     

       124
       124
       +
             | Some (start, end_) ->

     

       125
       125
       +
                 let range_value = match end_ with

     

       126
       126
       +
                   | Some e -> Printf.sprintf "bytes=%Ld-%Ld" start e

     

       127
       127
       +
                   | None -> Printf.sprintf "bytes=%Ld-" start

     

       128
       128
       +
                 in

     

       129
       129
       +
                 Requests.Headers.of_list ["Range", range_value]

     

       130
       130
       +
             | None -> Requests.Headers.empty

     

       131
       131
       +
           in

     

       170
       132
        
       

     

       171
       171
       -
           (* Build command line with wget command *)

     

       172
       172
       -
           let cmd_args = "wget" :: args in

     

       133
       133
       +
           (* Make the request *)

     

       134
       134
       +
           let response = Requests.get t.session ~headers url in

     

       173
       135
        
       

     

       174
       174
       -
           try

     

       175
       175
       -
             (* Run wget using Eio process manager *)

     

       176
       176
       -
             let cmd_line = String.concat " " (List.map Filename.quote cmd_args) in

     

       177
       177
       -
             let process = Eio.Process.spawn t.env#process_mgr ~sw:t.sw

     

       178
       178
       -
               ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd_line] in

     

       179
       179
       -
             let exit_status = Eio.Process.await process in

     

       180
       180
       -
             if exit_status <> `Exited 0 then

     

       181
       181
       -
               let error_msg = match exit_status with

     

       182
       182
       -
                 | `Exited n -> Printf.sprintf "wget exited with code %d" n

     

       183
       183
       -
                 | `Signaled n -> Printf.sprintf "wget killed by signal %d" n in

     

       184
       184
       -
               Error ("Download failed: " ^ error_msg)

     

       185
       185
       -
             else (

     

       186
       186
       -
               (* Verify hash if provided *)

     

       187
       187
       -
               match hash with

     

       188
       188
       -
               | Some h ->

     

       189
       189
       -
                   if Hash.verify dest h then

     

       190
       190
       -
                     Ok ()

     

       191
       191
       -
                   else

     

       192
       192
       -
                     Error ("Hash verification failed for " ^ dest_path)

     

       193
       193
       -
               | None -> Ok ()

     

       194
       194
       -
             )

     

       195
       195
       -
           with

     

       196
       196
       -
           | exn -> Error ("wget failed: " ^ (Printexc.to_string exn))

     

       136
       136
       +
           (* Check status *)

     

       137
       137
       +
           let status = Requests.Response.status response in

     

       138
       138
       +
           if not (Requests.Status.is_success status || Requests.Status.to_int status = 206) then

     

       139
       139
       +
             Error (Printf.sprintf "HTTP %d: %s" (Requests.Status.to_int status) (Requests.Status.to_string status))

     

       140
       140
       +
           else begin

     

       141
       141
       +
             (* Progress reporting is initialized with total_bytes at creation *)

     

       142
       142
       +
             (* We could get content-length from headers but progress is already set up *)

     

       143
       143
       +
             let _content_length =

     

       144
       144
       +
               Requests.Response.headers response

     

       145
       145
       +
               |> Requests.Headers.get "content-length"

     

       146
       146
       +
               |> Option.map Int64.of_string

     

       147
       147
       +
             in

     

       197
       148
        
       

     

       198
       198
       -
         let supports_resume _ = true

     

       199
       199
       -
         let name _ = "wget"

     

       200
       200
       -
       end

     

       201
       201
       -
       

     

       202
       202
       -
       (** Legacy curl downloader for systems that prefer external tools *)

     

       203
       203
       -
       module Curl_downloader = struct

     

       204
       204
       -
         type t = {

     

       205
       205
       -
           sw : Eio.Switch.t;

     

       206
       206
       -
           env : Eio_unix.Stdenv.base;

     

       207
       207
       -
           auth : Config.auth option;

     

       208
       208
       -
           timeout : float;

     

       209
       209
       -
         }

     

       210
       210
       -
       

     

       211
       211
       -
         let create ~sw ~env ?auth () = { sw; env; auth; timeout = 300.0 }

     

       212
       212
       -
       

     

       213
       213
       -
         let download t ~url ~dest ?hash ?progress:_ ?(resume=true) () =

     

       214
       214
       -
           let dest_path = Eio.Path.native_exn dest in

     

       149
       149
       +
             (* Stream the body to sink *)

     

       150
       150
       +
             let body = Requests.Response.body response in

     

       151
       151
       +
             Eio.Flow.copy body sink;

     

       152
       152
       +
             (* Progress reporting would need chunk-by-chunk reading which isn't available *)

     

       153
       153
       +
             Option.iter Progress.finish progress;

     

       154
       154
       +
             Ok ()

     

       155
       155
       +
           end

     

       156
       156
       +
         with

     

       157
       157
       +
         | exn ->

     

       158
       158
       +
             Error (Printf.sprintf "Download failed: %s" (Printexc.to_string exn))

     

       215
       159
        
       

     

       216
       216
       -
           (* Build curl arguments *)

     

       217
       217
       -
           let args = [

     

       218
       218
       -
             "--silent";

     

       219
       219
       -
             "--show-error";

     

       220
       220
       -
             "--location";

     

       221
       221
       -
             "--max-time"; "300";

     

       222
       222
       -
             "--retry"; "3";

     

       223
       223
       -
             "--output"; dest_path;

     

       224
       224
       -
           ] in

     

       160
       160
       +
       let download (T t as downloader) ~url ~dest ?hash ?progress ?resume () =

     

       161
       161
       +
         try

     

       162
       162
       +
           (* Check if we should resume *)

     

       163
       163
       +
           let start_offset = match resume with

     

       164
       164
       +
             | Some true when Eio.Path.is_file dest ->

     

       165
       165
       +
                 let stat = Eio.Path.stat ~follow:false dest in

     

       166
       166
       +
                 Some (Optint.Int63.to_int64 stat.size)

     

       167
       167
       +
             | _ -> None

     

       168
       168
       +
           in

     

       225
       169
        
       

     

       226
       226
       -
           (* Add authentication if provided *)

     

       227
       227
       -
           let args = Option.fold t.auth ~none:args ~some:(fun auth ->

     

       228
       228
       -
             let auth_str = match auth.Config.username, auth.Config.password with

     

       229
       229
       -
               | Some user, Some pass -> Some (user ^ ":" ^ pass)

     

       230
       230
       -
               | Some user, None -> Some user

     

       231
       231
       -
               | None, _ -> None in

     

       232
       232
       -
             match auth_str with

     

       233
       233
       -
             | Some s -> "--user" :: s :: args

     

       234
       234
       -
             | None -> args) in

     

       170
       170
       +
           (* Open file for writing (append if resuming) *)

     

       171
       171
       +
           Eio.Switch.run @@ fun sw ->

     

       172
       172
       +
           let sink = match start_offset with

     

       173
       173
       +
             | Some _ ->

     

       174
       174
       +
                 (* For append, open existing file and seek to end *)

     

       175
       175
       +
                 let file = Eio.Path.open_out ~sw ~create:`Never dest in

     

       176
       176
       +
                 let (_ : Optint.Int63.t) = Eio.File.seek file (Optint.Int63.zero) `End in

     

       177
       177
       +
                 file

     

       178
       178
       +
             | None ->

     

       179
       179
       +
                 Eio.Path.open_out ~sw ~create:(`Or_truncate 0o644) dest

     

       180
       180
       +
           in

     

       235
       181
        
       

     

       236
       236
       -
           (* Add resume support if enabled *)

     

       237
       237
       -
           let args = if resume then "--continue-at" :: "-" :: args else args in

     

       182
       182
       +
           (* Set range if resuming *)

     

       183
       183
       +
           let range = Option.map (fun offset -> (offset, None)) start_offset in

     

       238
       184
        
       

     

       239
       239
       -
           (* Add URL as last argument *)

     

       240
       240
       -
           let args = args @ [url] in

     

       185
       185
       +
           (* Get content length if needed for progress *)

     

       186
       186
       +
           let progress = match progress with

     

       187
       187
       +
             | Some p when p.Progress.total_bytes = None ->

     

       188
       188
       +
                 (* Try to get content length *)

     

       189
       189
       +
                 (try

     

       190
       190
       +
                   let response = Requests.head t.session url in

     

       191
       191
       +
                   let headers = Requests.Response.headers response in

     

       192
       192
       +
                   match Requests.Headers.get "content-length" headers with

     

       193
       193
       +
                   | Some len -> Some (Progress.create ~total_bytes:(Int64.of_string len) p.Progress.name)

     

       194
       194
       +
                   | None -> Some p

     

       195
       195
       +
                 with _ -> Some p)

     

       196
       196
       +
             | other -> other

     

       197
       197
       +
           in

     

       241
       198
        
       

     

       242
       242
       -
           (* Build command line with curl command *)

     

       243
       243
       -
           let cmd_args = "curl" :: args in

     

       199
       199
       +
           (* Download to the file *)

     

       200
       200
       +
           let result = download_to_stream downloader ~url ~sink ?progress ?range () in

     

       201
       201
       +
           Eio.Flow.close sink;

     

       244
       202
        
       

     

       245
       245
       -
           try

     

       246
       246
       -
             (* Run curl using Eio process manager *)

     

       247
       247
       -
             let cmd_line = String.concat " " (List.map Filename.quote cmd_args) in

     

       248
       248
       -
             let process = Eio.Process.spawn t.env#process_mgr ~sw:t.sw

     

       249
       249
       -
               ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd_line] in

     

       250
       250
       -
             let exit_status = Eio.Process.await process in

     

       251
       251
       -
             if exit_status <> `Exited 0 then

     

       252
       252
       -
               let error_msg = match exit_status with

     

       253
       253
       -
                 | `Exited n -> Printf.sprintf "curl exited with code %d" n

     

       254
       254
       -
                 | `Signaled n -> Printf.sprintf "curl killed by signal %d" n in

     

       255
       255
       -
               Error ("Download failed: " ^ error_msg)

     

       256
       256
       -
             else (

     

       257
       257
       -
               (* Verify hash if provided *)

     

       203
       203
       +
           (* Verify hash if provided *)

     

       204
       204
       +
           match result with

     

       205
       205
       +
           | Error e -> Error e

     

       206
       206
       +
           | Ok () ->

     

       258
       207
        
               match hash with

     

       259
       208
        
               | Some h ->

     

       260
       260
       -
                   if Hash.verify dest h then

     

       261
       261
       -
                     Ok ()

     

       262
       262
       -
                   else

     

       263
       263
       -
                     Error ("Hash verification failed for " ^ dest_path)

     

       209
       209
       +
                   if Hash.verify dest h then Ok ()

     

       210
       210
       +
                   else Error "Hash verification failed"

     

       264
       211
        
               | None -> Ok ()

     

       265
       265
       -
             )

     

       266
       266
       -
           with

     

       267
       267
       -
           | exn -> Error ("curl failed: " ^ (Printexc.to_string exn))

     

       268
       268
       -
       

     

       269
       269
       -
         let supports_resume _ = true

     

       270
       270
       -
         let name _ = "curl"

     

       271
       271
       -
       end

     

       272
       272
       -
       

     

       273
       273
       -
       (** Cohttp_downloader is now an alias for Http_downloader *)

     

       274
       274
       -
       module Cohttp_downloader = Http_downloader

     

       212
       212
       +
         with

     

       213
       213
       +
         | exn ->

     

       214
       214
       +
             Error (Printf.sprintf "Download failed: %s" (Printexc.to_string exn))

     

       275
       215
        
       

     

       276
       276
       -
       module Downloaders = struct

     

       277
       277
       -
         let wget () = (module Wget_downloader : DOWNLOADER)

     

       278
       278
       -
         let curl () = (module Curl_downloader : DOWNLOADER)

     

       279
       279
       -
         let cohttp () = (module Http_downloader : DOWNLOADER)

     

       280
       280
       -
         let http () = (module Http_downloader : DOWNLOADER)

     

       216
       216
       +
       (** {1 Chunked Download Support} *)

     

       281
       217
        
       

     

       282
       282
       -
         let detect_available ~env =

     

       283
       283
       -
           let check_command cmd =

     

       284
       284
       -
             try

     

       285
       285
       -
               let sw = Eio.Switch.run @@ fun sw ->

     

       286
       286
       -
                 let process = Eio.Process.spawn env#process_mgr ~sw

     

       287
       287
       -
                   ~executable:"/bin/sh" ["/bin/sh"; "-c"; cmd ^ " --version > /dev/null 2>&1"] in

     

       288
       288
       -
                 let exit_status = Eio.Process.await process in

     

       289
       289
       -
                 exit_status = `Exited 0

     

       290
       290
       -
               in sw

     

       291
       291
       -
             with _ -> false

     

       218
       218
       +
       let download_range (T t) ~url ~start ~end_ =

     

       219
       219
       +
         try

     

       220
       220
       +
           let headers =

     

       221
       221
       +
             let range_value = match end_ with

     

       222
       222
       +
               | Some e -> Printf.sprintf "bytes=%Ld-%Ld" start e

     

       223
       223
       +
               | None -> Printf.sprintf "bytes=%Ld-" start

     

       224
       224
       +
             in

     

       225
       225
       +
             Requests.Headers.of_list ["Range", range_value]

     

       292
       226
        
           in

     

       293
       227
        
       

     

       294
       294
       -
           let available = ref [] in

     

       295
       295
       -
       

     

       296
       296
       -
           (* Always available - uses requests library *)

     

       297
       297
       -
           available := ("http", http ()) :: !available;

     

       298
       298
       -
       

     

       299
       299
       -
           (* Check for external tools *)

     

       300
       300
       -
           if check_command "wget" then

     

       301
       301
       -
             available := ("wget", wget ()) :: !available;

     

       302
       302
       -
       

     

       303
       303
       -
           if check_command "curl" then

     

       304
       304
       -
             available := ("curl", curl ()) :: !available;

     

       228
       228
       +
           let response = Requests.get t.session ~headers url in

     

       229
       229
       +
           let status = Requests.Response.status response in

     

       305
       230
        
       

     

       306
       306
       -
           List.rev !available

     

       231
       231
       +
           if Requests.Status.to_int status = 206 || Requests.Status.is_success status then

     

       232
       232
       +
             let body = Requests.Response.body response in

     

       233
       233
       +
             let buf = Buffer.create 4096 in

     

       234
       234
       +
             Eio.Flow.copy body (Eio.Flow.buffer_sink buf);

     

       235
       235
       +
             Ok (Buffer.contents buf)

     

       236
       236
       +
           else

     

       237
       237
       +
             Error (Printf.sprintf "Range request failed: HTTP %d" (Requests.Status.to_int status))

     

       238
       238
       +
         with

     

       239
       239
       +
         | exn ->

     

       240
       240
       +
             Error (Printf.sprintf "Range download failed: %s" (Printexc.to_string exn))

     

       307
       241
        
       

     

       308
       308
       -
         let create_default ~env =

     

       309
       309
       -
           (* Default to http (requests library), fallback to wget/curl if needed *)

     

       310
       310
       -
           match detect_available ~env with

     

       311
       311
       -
           | [] -> failwith "No downloaders available"

     

       312
       312
       -
           | (_, d) :: _ -> d

     

       242
       242
       +
       let get_content_length (T t) ~url =

     

       243
       243
       +
         try

     

       244
       244
       +
           (* HEAD request to get content length *)

     

       245
       245
       +
           let response = Requests.head t.session url in

     

       246
       246
       +
           let headers = Requests.Response.headers response in

     

       247
       247
       +
           match Requests.Headers.get "content-length" headers with

     

       248
       248
       +
           | Some len -> Ok (Int64.of_string len)

     

       249
       249
       +
           | None -> Error "No Content-Length header"

     

       250
       250
       +
         with

     

       251
       251
       +
         | exn ->

     

       252
       252
       +
             Error (Printf.sprintf "Failed to get content length: %s" (Printexc.to_string exn))

     

       313
       253
        
       

     

       314
       314
       -
         let of_string name =

     

       315
       315
       -
           match name with

     

       316
       316
       -
           | "wget" -> Some (wget ())

     

       317
       317
       -
           | "curl" -> Some (curl ())

     

       318
       318
       -
           | "cohttp" | "http" -> Some (http ())

     

       319
       319
       -
           | _ -> None

     

       320
       320
       -
       end

     

       254
       254
       +
       (** {1 Utility Functions} *)

     

       321
       255
        
       

     

       322
       322
       -
       module Cli = struct

     

       323
       323
       -
         type downloader_choice = [ `Wget | `Curl | `Cohttp | `Http | `Auto ]

     

       256
       256
       +
       let supports_range (T t) ~url =

     

       257
       257
       +
         try

     

       258
       258
       +
           let response = Requests.head t.session url in

     

       259
       259
       +
           let headers = Requests.Response.headers response in

     

       260
       260
       +
           match Requests.Headers.get "accept-ranges" headers with

     

       261
       261
       +
           | Some "bytes" -> true

     

       262
       262
       +
           | _ -> false

     

       263
       263
       +
         with _ -> false

     

       324
       264
        
       

     

       325
       325
       -
         open Cmdliner

     

       265
       265
       +
       let name _ = "requests"

     

       326
       266
        
       

     

       327
       327
       -
         let downloader_term =

     

       328
       328
       -
           let choices = [

     

       329
       329
       -
             ("wget", `Wget);

     

       330
       330
       -
             ("curl", `Curl);

     

       331
       331
       -
             ("cohttp", `Cohttp);

     

       332
       332
       -
             ("http", `Http);

     

       333
       333
       -
             ("auto", `Auto)

     

       334
       334
       -
           ] in

     

       335
       335
       -
           let doc = "Download tool to use. 'auto' detects available tools, 'http' uses built-in HTTP client." in

     

       336
       336
       -
           let docv = "TOOL" in

     

       337
       337
       -
           Arg.(value & opt (enum choices) `Auto & info ["downloader"; "d"] ~doc ~docv)

     

       267
       267
       +
       (** {1 Pretty Printing} *)

     

       338
       268
        
       

     

       339
       339
       -
         let downloader_info = Arg.info ["downloader"; "d"]

     

       269
       269
       +
       let pp_config fmt config =

     

       270
       270
       +
         Format.fprintf fmt "Config{timeout=%.1f, max_redirects=%d, verify_tls=%b}"

     

       271
       271
       +
           config.timeout config.max_redirects config.verify_tls

     

       340
       272
        
       

     

       341
       341
       -
         let create_downloader ~env = function

     

       342
       342
       -
           | `Wget -> Downloaders.wget ()

     

       343
       343
       -
           | `Curl -> Downloaders.curl ()

     

       344
       344
       -
           | `Cohttp | `Http -> Downloaders.http ()

     

       345
       345
       -
           | `Auto -> Downloaders.create_default ~env

     

       346
       346
       -
       end
     

       273
       273
       +
       let pp fmt (T t) =

     

       274
       274
       +
         Format.fprintf fmt "Downloader{engine=%s, %a}"

     

       275
       275
       +
           (name (T t)) pp_config t.config

+75 -64

stack/toru/lib/toru/downloader.mli

···

       1
       1
       -
       (** Downloader module for fetching files from remote sources *)

     

       1
       1
       +
       (** HTTP downloader module interface *)

     

       2
       2
        
       

     

       3
       3
       -
       (** Progress reporter for download tracking *)

     

       4
       4
       -
       module Progress_reporter : sig

     

       3
       3
       +
       (** Progress reporting *)

     

       4
       4
       +
       module Progress : sig

     

       5
       5
        
         type t

     

       6
       6
       -
         

     

       7
       6
        
         val create : ?total_bytes:int64 -> string -> t

     

       8
       7
        
         val update : t -> int64 -> unit

     

       9
       8
        
         val finish : t -> unit

     

       10
       9
        
       end

     

       11
       10
        
       

     

       12
       12
       -
       (** Configuration for authentication *)

     

       13
       13
       -
       module Config : sig

     

       14
       14
       -
         type auth = {

     

       15
       15
       -
           username : string option;

     

       16
       16
       -
           password : string option;

     

       17
       17
       -
         }

     

       18
       18
       -
       end

     

       11
       11
       +
       (** Downloader configuration *)

     

       12
       12
       +
       type config = {

     

       13
       13
       +
         timeout : float;

     

       14
       14
       +
         max_redirects : int;

     

       15
       15
       +
         retry : Requests.Retry.config option;

     

       16
       16
       +
         auth : Requests.Auth.t option;

     

       17
       17
       +
         headers : Requests.Headers.t;

     

       18
       18
       +
         verify_tls : bool;

     

       19
       19
       +
       }

     

       20
       20
       +
       

     

       21
       21
       +
       (** Downloader instance *)

     

       22
       22
       +
       type t

     

       23
       23
       +
       

     

       24
       24
       +
       (** Create default configuration *)

     

       25
       25
       +
       val default_config : config

     

       19
       26
        
       

     

       20
       20
       -
       (** Abstract downloader interface *)

     

       21
       21
       -
       module type DOWNLOADER = sig

     

       22
       22
       -
         type t

     

       23
       23
       -
         

     

       24
       24
       -
         val create : sw:Eio.Switch.t -> env:Eio_unix.Stdenv.base -> 

     

       25
       25
       -
                      ?auth:Config.auth -> unit -> t

     

       26
       26
       -
         

     

       27
       27
       -
         val download : t ->

     

       28
       28
       -
           url:string ->

     

       29
       29
       -
           dest:Eio.Fs.dir_ty Eio.Path.t ->

     

       30
       30
       -
           ?hash:Hash.t ->

     

       31
       31
       -
           ?progress:Progress_reporter.t ->

     

       32
       32
       -
           ?resume:bool ->

     

       33
       33
       -
           unit -> (unit, string) result

     

       34
       34
       -
           

     

       35
       35
       -
         val supports_resume : t -> bool

     

       36
       36
       -
         val name : t -> string

     

       37
       37
       -
       end

     

       27
       27
       +
       (** Create custom configuration *)

     

       28
       28
       +
       val create_config :

     

       29
       29
       +
         ?timeout:float ->

     

       30
       30
       +
         ?max_redirects:int ->

     

       31
       31
       +
         ?retry:Requests.Retry.config option ->

     

       32
       32
       +
         ?auth:Requests.Auth.t option ->

     

       33
       33
       +
         ?headers:Requests.Headers.t ->

     

       34
       34
       +
         ?verify_tls:bool ->

     

       35
       35
       +
         unit -> config

     

       36
       36
       +
       

     

       37
       37
       +
       (** Create a new downloader *)

     

       38
       38
       +
       val create :

     

       39
       39
       +
         sw:Eio.Switch.t ->

     

       40
       40
       +
         env:Eio_unix.Stdenv.base ->

     

       41
       41
       +
         ?config:config ->

     

       42
       42
       +
         unit -> t

     

       43
       43
       +
       

     

       44
       44
       +
       (** Download to a file *)

     

       45
       45
       +
       val download :

     

       46
       46
       +
         t ->

     

       47
       47
       +
         url:string ->

     

       48
       48
       +
         dest:Eio.Fs.dir_ty Eio.Path.t ->

     

       49
       49
       +
         ?hash:Hash.t ->

     

       50
       50
       +
         ?progress:Progress.t ->

     

       51
       51
       +
         ?resume:bool ->

     

       52
       52
       +
         unit -> (unit, string) result

     

       53
       53
       +
       

     

       54
       54
       +
       (** Download to a stream *)

     

       55
       55
       +
       val download_to_stream :

     

       56
       56
       +
         t ->

     

       57
       57
       +
         url:string ->

     

       58
       58
       +
         sink:Eio.Flow.sink_ty Eio.Resource.t ->

     

       59
       59
       +
         ?progress:Progress.t ->

     

       60
       60
       +
         ?range:(int64 * int64 option) ->

     

       61
       61
       +
         unit -> (unit, string) result

     

       38
       62
        
       

     

       39
       39
       -
       (** Concrete downloader implementations *)

     

       40
       40
       -
       module Wget_downloader : sig

     

       41
       41
       -
         include DOWNLOADER

     

       42
       42
       -
       end

     

       63
       63
       +
       (** Download a range of bytes *)

     

       64
       64
       +
       val download_range :

     

       65
       65
       +
         t ->

     

       66
       66
       +
         url:string ->

     

       67
       67
       +
         start:int64 ->

     

       68
       68
       +
         end_:int64 option ->

     

       69
       69
       +
         (string, string) result

     

       43
       70
        
       

     

       44
       44
       -
       module Curl_downloader : sig

     

       45
       45
       -
         include DOWNLOADER  

     

       46
       46
       -
       end

     

       71
       71
       +
       (** Get content length from server *)

     

       72
       72
       +
       val get_content_length :

     

       73
       73
       +
         t ->

     

       74
       74
       +
         url:string ->

     

       75
       75
       +
         (int64, string) result

     

       47
       76
        
       

     

       48
       48
       -
       module Cohttp_downloader : sig

     

       49
       49
       -
         include DOWNLOADER

     

       50
       50
       -
       end

     

       77
       77
       +
       (** Check if server supports range requests *)

     

       78
       78
       +
       val supports_range :

     

       79
       79
       +
         t ->

     

       80
       80
       +
         url:string ->

     

       81
       81
       +
         bool

     

       51
       82
        
       

     

       52
       52
       -
       (** Downloader selection utilities *)

     

       53
       53
       -
       module Downloaders : sig

     

       54
       54
       -
         val wget : unit -> (module DOWNLOADER)

     

       55
       55
       -
         val curl : unit -> (module DOWNLOADER)

     

       56
       56
       -
         val cohttp : unit -> (module DOWNLOADER)

     

       57
       57
       -
         val http : unit -> (module DOWNLOADER)

     

       58
       58
       -
         

     

       59
       59
       -
         val detect_available : env:Eio_unix.Stdenv.base ->

     

       60
       60
       -
                               (string * (module DOWNLOADER)) list

     

       61
       61
       -
         val create_default : env:Eio_unix.Stdenv.base ->

     

       62
       62
       -
                             (module DOWNLOADER)

     

       63
       63
       -
         val of_string : string -> (module DOWNLOADER) option

     

       64
       64
       -
       end

     

       83
       83
       +
       (** Get downloader name *)

     

       84
       84
       +
       val name : t -> string

     

       65
       85
        
       

     

       66
       66
       -
       (** CLI integration *)

     

       67
       67
       -
       module Cli : sig

     

       68
       68
       -
         type downloader_choice = [ `Wget | `Curl | `Cohttp | `Http | `Auto ]

     

       69
       69
       -
         

     

       70
       70
       -
         val downloader_term : downloader_choice Cmdliner.Term.t

     

       71
       71
       -
         val downloader_info : Cmdliner.Arg.info

     

       72
       72
       -
         

     

       73
       73
       -
         val create_downloader : 

     

       74
       74
       -
           env:Eio_unix.Stdenv.base ->

     

       75
       75
       -
           downloader_choice -> 

     

       76
       76
       -
           (module DOWNLOADER)

     

       77
       77
       -
       end
     

       86
       86
       +
       (** Pretty printing *)

     

       87
       87
       +
       val pp : Format.formatter -> t -> unit

     

       88
       88
       +
       val pp_config : Format.formatter -> config -> unit

+10 -1

stack/toru/lib/toru/dune

···

       4
       4
        
        (modules hash registry cache processors downloader make_registry toru logging)

     

       5
       5
        
        (libraries

     

       6
       6
        
          eio

     

       7
       7
       +
          eio_main

     

       7
       8
        
          eio.unix

     

       8
       9
        
          digestif

     

       9
       10
        
          yojson

     

       10
       10
       -
          cmdliner

     

       11
       11
        
          re

     

       12
       12
        
          ptime

     

       13
       13
        
          xdge

     
···

       17
       17
        
          logs

     

       18
       18
        
          logs.fmt

     

       19
       19
        
          fmt))

     

       20
       20
       +
       

     

       21
       21
       +
       (library

     

       22
       22
       +
        (public_name toru.cmd)

     

       23
       23
       +
        (name toru_cmd)

     

       24
       24
       +
        (modules cmd)

     

       25
       25
       +
        (libraries

     

       26
       26
       +
          toru

     

       27
       27
       +
          cmdliner

     

       28
       28
       +
          eio_main))

     

       20
       29
        
       

     

       21
       30
        
       (documentation

     

       22
       31
        
        (package toru))

+167 -19

stack/toru/lib/toru/processors.ml

···

       1
       1
       -
       type processor = Eio.Fs.dir_ty Eio.Path.t -> Eio.Fs.dir_ty Eio.Path.t

     

       1
       1
       +
       (** Processor module for post-download transformations

     

       2
       2
       +
       

     

       3
       3
       +
           This module provides processors that can transform downloaded files,

     

       4
       4
       +
           such as decompression or unpacking archives. Designed to work with

     

       5
       5
       +
           the requests library's response processing pipeline.

     

       6
       6
       +
       *)

     

       7
       7
       +
       

     

       8
       8
       +
       let src = Logs.Src.create "toru.processors" ~doc:"File processors"

     

       9
       9
       +
       module Log = (val Logs.src_log src : Logs.LOG)

     

       10
       10
       +
       

     

       11
       11
       +
       (** {1 Processor Types} *)

     

       12
       12
       +
       

     

       13
       13
       +
       type processor = Eio.Fs.dir_ty Eio.Path.t -> (Eio.Fs.dir_ty Eio.Path.t, string) result

     

       14
       14
       +
       

     

       15
       15
       +
       (** {1 Helper Functions} *)

     

       16
       16
       +
       

     

       17
       17
       +
       let run_command ~sw ~env cmd args =

     

       18
       18
       +
         try

     

       19
       19
       +
           let proc = Eio.Process.spawn ~sw env#process_mgr

     

       20
       20
       +
             ~executable:cmd (cmd :: args) in

     

       21
       21
       +
           match Eio.Process.await proc with

     

       22
       22
       +
           | `Exited 0 -> Ok ()

     

       23
       23
       +
           | `Exited n -> Error (Printf.sprintf "%s exited with code %d" cmd n)

     

       24
       24
       +
           | `Signaled n -> Error (Printf.sprintf "%s killed by signal %d" cmd n)

     

       25
       25
       +
         with

     

       26
       26
       +
         | exn -> Error (Printf.sprintf "%s failed: %s" cmd (Printexc.to_string exn))

     

       27
       27
       +
       

     

       28
       28
       +
       let ensure_output_dir ~env output_dir =

     

       29
       29
       +
         let fs = env#fs in

     

       30
       30
       +
         let output_path = Eio.Path.(fs / output_dir) in

     

       31
       31
       +
         Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 output_path

     

       32
       32
       +
       

     

       33
       33
       +
       (** {1 Archive Processors} *)

     

       34
       34
       +
       

     

       35
       35
       +
       let untar_gz output_dir : processor = fun path ->

     

       36
       36
       +
         Log.info (fun m -> m "Extracting tar.gz: %s -> %s"

     

       37
       37
       +
           (Eio.Path.native_exn path) output_dir);

     

       38
       38
       +
       

     

       39
       39
       +
         Eio_main.run @@ fun env ->

     

       40
       40
       +
         Eio.Switch.run @@ fun sw ->

     

       41
       41
       +
         let path_str = Eio.Path.native_exn path in

     

       42
       42
       +
       

     

       43
       43
       +
         (* Create output directory *)

     

       44
       44
       +
         ensure_output_dir ~env output_dir;

     

       45
       45
       +
       

     

       46
       46
       +
         (* Extract using tar *)

     

       47
       47
       +
         match run_command ~sw ~env "tar" ["-xzf"; path_str; "-C"; output_dir] with

     

       48
       48
       +
         | Ok () -> Ok Eio.Path.(env#fs / output_dir)

     

       49
       49
       +
         | Error e -> Error e

     

       2
       50
        
       

     

       3
       3
       -
       let untar_gz _target_dir =

     

       4
       4
       -
         fun _path ->

     

       5
       5
       -
           (* TODO: Implement tar.gz extraction *)

     

       6
       6
       -
           failwith "Processors.untar_gz not yet implemented"

     

       51
       51
       +
       let untar_xz output_dir : processor = fun path ->

     

       52
       52
       +
         Log.info (fun m -> m "Extracting tar.xz: %s -> %s"

     

       53
       53
       +
           (Eio.Path.native_exn path) output_dir);

     

       7
       54
        
       

     

       8
       8
       -
       let unzip _target_dir =

     

       9
       9
       -
         fun _path ->

     

       10
       10
       -
           (* TODO: Implement zip extraction *)

     

       11
       11
       -
           failwith "Processors.unzip not yet implemented"

     

       55
       55
       +
         Eio_main.run @@ fun env ->

     

       56
       56
       +
         Eio.Switch.run @@ fun sw ->

     

       57
       57
       +
         let path_str = Eio.Path.native_exn path in

     

       12
       58
        
       

     

       13
       13
       -
       let untar_xz _target_dir =

     

       14
       14
       -
         fun _path ->

     

       15
       15
       -
           (* TODO: Implement tar.xz extraction *)

     

       16
       16
       -
           failwith "Processors.untar_xz not yet implemented"

     

       59
       59
       +
         (* Create output directory *)

     

       60
       60
       +
         ensure_output_dir ~env output_dir;

     

       17
       61
        
       

     

       18
       18
       -
       let custom _command _args =

     

       19
       19
       -
         fun _path ->

     

       20
       20
       -
           (* TODO: Implement custom command execution *)

     

       21
       21
       -
           failwith "Processors.custom not yet implemented"

     

       62
       62
       +
         (* Extract using tar *)

     

       63
       63
       +
         match run_command ~sw ~env "tar" ["-xJf"; path_str; "-C"; output_dir] with

     

       64
       64
       +
         | Ok () -> Ok Eio.Path.(env#fs / output_dir)

     

       65
       65
       +
         | Error e -> Error e

     

       66
       66
       +
       

     

       67
       67
       +
       let unzip output_dir : processor = fun path ->

     

       68
       68
       +
         Log.info (fun m -> m "Extracting zip: %s -> %s"

     

       69
       69
       +
           (Eio.Path.native_exn path) output_dir);

     

       70
       70
       +
       

     

       71
       71
       +
         Eio_main.run @@ fun env ->

     

       72
       72
       +
         Eio.Switch.run @@ fun sw ->

     

       73
       73
       +
         let path_str = Eio.Path.native_exn path in

     

       22
       74
        
       

     

       23
       23
       -
       let identity = fun path -> path

     

       75
       75
       +
         (* Create output directory *)

     

       76
       76
       +
         ensure_output_dir ~env output_dir;

     

       24
       77
        
       

     

       25
       25
       -
       let compose p1 p2 = fun path -> p2 (p1 path)
     

       78
       78
       +
         (* Extract using unzip *)

     

       79
       79
       +
         match run_command ~sw ~env "unzip" ["-q"; "-o"; path_str; "-d"; output_dir] with

     

       80
       80
       +
         | Ok () -> Ok Eio.Path.(env#fs / output_dir)

     

       81
       81
       +
         | Error e -> Error e

     

       82
       82
       +
       

     

       83
       83
       +
       (** {1 Compression Processors} *)

     

       84
       84
       +
       

     

       85
       85
       +
       let gunzip : processor = fun path ->

     

       86
       86
       +
         Log.info (fun m -> m "Decompressing gzip: %s" (Eio.Path.native_exn path));

     

       87
       87
       +
       

     

       88
       88
       +
         Eio_main.run @@ fun env ->

     

       89
       89
       +
         Eio.Switch.run @@ fun sw ->

     

       90
       90
       +
         let path_str = Eio.Path.native_exn path in

     

       91
       91
       +
       

     

       92
       92
       +
         (* Output file without .gz extension *)

     

       93
       93
       +
         let output_str =

     

       94
       94
       +
           if String.ends_with ~suffix:".gz" path_str then

     

       95
       95
       +
             String.sub path_str 0 (String.length path_str - 3)

     

       96
       96
       +
           else

     

       97
       97
       +
             path_str ^ ".uncompressed"

     

       98
       98
       +
         in

     

       99
       99
       +
       

     

       100
       100
       +
         (* Decompress using gunzip *)

     

       101
       101
       +
         match run_command ~sw ~env "gunzip" ["-c"; path_str] with

     

       102
       102
       +
         | Ok () -> Ok Eio.Path.(env#fs / output_str)

     

       103
       103
       +
         | Error e -> Error e

     

       104
       104
       +
       

     

       105
       105
       +
       let bunzip2 : processor = fun path ->

     

       106
       106
       +
         Log.info (fun m -> m "Decompressing bzip2: %s" (Eio.Path.native_exn path));

     

       107
       107
       +
       

     

       108
       108
       +
         Eio_main.run @@ fun env ->

     

       109
       109
       +
         Eio.Switch.run @@ fun sw ->

     

       110
       110
       +
         let path_str = Eio.Path.native_exn path in

     

       111
       111
       +
       

     

       112
       112
       +
         (* Output file without .bz2 extension *)

     

       113
       113
       +
         let output_str =

     

       114
       114
       +
           if String.ends_with ~suffix:".bz2" path_str then

     

       115
       115
       +
             String.sub path_str 0 (String.length path_str - 4)

     

       116
       116
       +
           else

     

       117
       117
       +
             path_str ^ ".uncompressed"

     

       118
       118
       +
         in

     

       119
       119
       +
       

     

       120
       120
       +
         (* Decompress using bunzip2 *)

     

       121
       121
       +
         match run_command ~sw ~env "bunzip2" ["-c"; path_str] with

     

       122
       122
       +
         | Ok () -> Ok Eio.Path.(env#fs / output_str)

     

       123
       123
       +
         | Error e -> Error e

     

       124
       124
       +
       

     

       125
       125
       +
       (** {1 Custom Processor} *)

     

       126
       126
       +
       

     

       127
       127
       +
       let custom cmd args : processor = fun path ->

     

       128
       128
       +
         Log.info (fun m -> m "Running custom processor: %s" cmd);

     

       129
       129
       +
       

     

       130
       130
       +
         Eio_main.run @@ fun env ->

     

       131
       131
       +
         Eio.Switch.run @@ fun sw ->

     

       132
       132
       +
         let path_str = Eio.Path.native_exn path in

     

       133
       133
       +
       

     

       134
       134
       +
         (* Replace {file} placeholder in arguments *)

     

       135
       135
       +
         let args = List.map (fun arg ->

     

       136
       136
       +
           if arg = "{file}" then path_str else arg

     

       137
       137
       +
         ) args in

     

       138
       138
       +
       

     

       139
       139
       +
         match run_command ~sw ~env cmd args with

     

       140
       140
       +
         | Ok () -> Ok path

     

       141
       141
       +
         | Error e -> Error e

     

       142
       142
       +
       

     

       143
       143
       +
       (** {1 Utility Processors} *)

     

       144
       144
       +
       

     

       145
       145
       +
       let identity : processor = fun path -> Ok path

     

       146
       146
       +
       

     

       147
       147
       +
       let compose p1 p2 : processor = fun path ->

     

       148
       148
       +
         match p1 path with

     

       149
       149
       +
         | Error e -> Error e

     

       150
       150
       +
         | Ok p -> p2 p

     

       151
       151
       +
       

     

       152
       152
       +
       let chain processors : processor = fun path ->

     

       153
       153
       +
         List.fold_left (fun acc proc ->

     

       154
       154
       +
           match acc with

     

       155
       155
       +
           | Error e -> Error e

     

       156
       156
       +
           | Ok p -> proc p

     

       157
       157
       +
         ) (Ok path) processors

     

       158
       158
       +
       

     

       159
       159
       +
       let detect_processor filename =

     

       160
       160
       +
         (* Detect processor based on file extension *)

     

       161
       161
       +
         if String.ends_with ~suffix:".tar.gz" filename ||

     

       162
       162
       +
            String.ends_with ~suffix:".tgz" filename then

     

       163
       163
       +
           Some (untar_gz ".")

     

       164
       164
       +
         else if String.ends_with ~suffix:".tar.xz" filename then

     

       165
       165
       +
           Some (untar_xz ".")

     

       166
       166
       +
         else if String.ends_with ~suffix:".zip" filename then

     

       167
       167
       +
           Some (unzip ".")

     

       168
       168
       +
         else if String.ends_with ~suffix:".gz" filename then

     

       169
       169
       +
           Some gunzip

     

       170
       170
       +
         else if String.ends_with ~suffix:".bz2" filename then

     

       171
       171
       +
           Some bunzip2

     

       172
       172
       +
         else

     

       173
       173
       +
           None

+16 -2

stack/toru/lib/toru/processors.mli

···

       1
       1
        
       (** File processors for post-download transformations *)

     

       2
       2
        
       

     

       3
       3
        
       (** Processor function type *)

     

       4
       4
       -
       type processor = Eio.Fs.dir_ty Eio.Path.t -> Eio.Fs.dir_ty Eio.Path.t

     

       4
       4
       +
       type processor = Eio.Fs.dir_ty Eio.Path.t -> (Eio.Fs.dir_ty Eio.Path.t, string) result

     

       5
       5
        
       

     

       6
       6
        
       (** {1 Archive decompression processors} *)

     

       7
       7
        
       

     
···

       23
       23
        
       val identity : processor

     

       24
       24
        
       

     

       25
       25
        
       (** Compose two processors *)

     

       26
       26
       -
       val compose : processor -> processor -> processor
     

       26
       26
       +
       val compose : processor -> processor -> processor

     

       27
       27
       +
       

     

       28
       28
       +
       (** {1 Compression processors} *)

     

       29
       29
       +
       

     

       30
       30
       +
       (** Decompress gzip file *)

     

       31
       31
       +
       val gunzip : processor

     

       32
       32
       +
       

     

       33
       33
       +
       (** Decompress bzip2 file *)

     

       34
       34
       +
       val bunzip2 : processor

     

       35
       35
       +
       

     

       36
       36
       +
       (** Chain multiple processors *)

     

       37
       37
       +
       val chain : processor list -> processor

     

       38
       38
       +
       

     

       39
       39
       +
       (** Detect processor based on file extension *)

     

       40
       40
       +
       val detect_processor : string -> processor option

+287 -107

stack/toru/lib/toru/toru.ml

···

       1
       1
       +
       (** Toru - Data repository management library

     

       2
       2
       +
       

     

       3
       3
       +
           An OCaml library for managing data file downloads and caching,

     

       4
       4
       +
           compatible with Python Pooch registry files. It provides automatic

     

       5
       5
       +
           downloading, caching, and hash verification of data files from

     

       6
       6
       +
           remote repositories.

     

       7
       7
       +
       *)

     

       8
       8
       +
       

     

       9
       9
       +
       let src = Logs.Src.create "toru" ~doc:"Toru data management"

     

       10
       10
       +
       module Log = (val Logs.src_log src : Logs.LOG)

     

       11
       11
       +
       

     

       12
       12
       +
       (** {1 Types} *)

     

       13
       13
       +
       

     

       1
       14
        
       type t = {

     

       2
       15
        
         base_url : string;

     

       3
       16
        
         cache : Cache.t;

     

       4
       17
        
         registry : Registry.t;

     

       5
       5
       -
         downloader : (module Downloader.DOWNLOADER);

     

       18
       18
       +
         downloader : Downloader.t;

     

       6
       19
        
         sw : Eio.Switch.t;

     

       7
       20
        
         env : Eio_unix.Stdenv.base;

     

       8
       21
        
       }

     

       9
       22
        
       

     

       10
       10
       -
       let create ~sw ~env ~base_url ~cache_path ?version ?registry_file ?registry_url ?downloader () =

     

       23
       23
       +
       (** {1 Creation} *)

     

       24
       24
       +
       

     

       25
       25
       +
       let create ~sw ~env ~base_url ~cache_path ?version ?registry_file ?registry_url ?downloader_config () =

     

       26
       26
       +
         Log.info (fun m -> m "Creating Toru instance with base_url=%s, cache=%s"

     

       27
       27
       +
           base_url cache_path);

     

       28
       28
       +
       

     

       29
       29
       +
         (* Create cache *)

     

       11
       30
        
         let cache = Cache.create ~sw ~fs:env#fs ?version cache_path in

     

       12
       12
       -
         let registry = match registry_file with

     

       13
       13
       -
           | Some file -> Registry.load (Eio.Path.(env#fs / file))

     

       14
       14
       -
           | None -> 

     

       15
       15
       -
               (match registry_url with

     

       16
       16
       -
                | Some url -> Registry.load_from_url url

     

       17
       17
       -
                | None -> Registry.empty)

     

       31
       31
       +
       

     

       32
       32
       +
         (* Load registry *)

     

       33
       33
       +
         let registry = match registry_file, registry_url with

     

       34
       34
       +
           | Some file, _ ->

     

       35
       35
       +
               Log.info (fun m -> m "Loading registry from file: %s" file);

     

       36
       36
       +
               Registry.load Eio.Path.(env#fs / file)

     

       37
       37
       +
           | None, Some url ->

     

       38
       38
       +
               Log.info (fun m -> m "Loading registry from URL: %s" url);

     

       39
       39
       +
               Registry.load_from_url url

     

       40
       40
       +
           | None, None ->

     

       41
       41
       +
               Log.info (fun m -> m "Starting with empty registry");

     

       42
       42
       +
               Registry.empty

     

       18
       43
        
         in

     

       19
       19
       -
         let downloader = match downloader with

     

       20
       20
       -
           | Some d -> d

     

       21
       21
       -
           | None -> 

     

       22
       22
       -
               Downloader.Downloaders.create_default ~env

     

       23
       23
       -
         in

     

       44
       44
       +
       

     

       45
       45
       +
         (* Create downloader *)

     

       46
       46
       +
         let downloader = Downloader.create ~sw ~env ?config:downloader_config () in

     

       47
       47
       +
       

     

       24
       48
        
         { base_url; cache; registry; downloader; sw; env }

     

       25
       49
        
       

     

       50
       50
       +
       (** {1 Accessors} *)

     

       51
       51
       +
       

     

       26
       52
        
       let base_url t = t.base_url

     

       27
       53
        
       let cache t = t.cache

     

       28
       54
        
       let registry t = t.registry

     

       55
       55
       +
       let downloader t = t.downloader

     

       56
       56
       +
       

     

       57
       57
       +
       (** {1 Fetch Operations} *)

     

       29
       58
        
       

     

       30
       59
        
       let rec fetch t ~filename ?processor () =

     

       31
       31
       -
         (* 1. Check if file exists in registry *)

     

       60
       60
       +
         Log.info (fun m -> m "Fetching file: %s" filename);

     

       61
       61
       +
       

     

       62
       62
       +
         (* Check registry *)

     

       32
       63
        
         match Registry.find filename t.registry with

     

       33
       33
       -
         | None -> Error ("File not found in registry: " ^ filename)

     

       64
       64
       +
         | None ->

     

       65
       65
       +
             Log.warn (fun m -> m "File not found in registry: %s" filename);

     

       66
       66
       +
             Error ("File not found in registry: " ^ filename)

     

       34
       67
        
         | Some entry ->

     

       35
       35
       -
             (* 2. Check if file is already cached *)

     

       36
       36
       -
             let cache_path = Cache.file_path t.cache filename in

     

       37
       37
       -
             let cached_file_exists = Cache.exists t.cache filename in

     

       38
       38
       -
             

     

       39
       39
       -
             (* 3. If cached, verify hash *)

     

       40
       40
       -
             if cached_file_exists then (

     

       68
       68
       +
             (* Check cache *)

     

       69
       69
       +
             if Cache.exists t.cache filename then

     

       70
       70
       +
               (* Verify cached file *)

     

       71
       71
       +
               let cache_path = Cache.file_path t.cache filename in

     

       41
       72
        
               let entry_hash = Registry.hash entry in

     

       42
       42
       -
               if Hash.verify cache_path entry_hash then (

     

       43
       43
       -
                 (* File is cached and valid - apply processor if provided *)

     

       73
       73
       +
               if Hash.verify cache_path entry_hash then begin

     

       74
       74
       +
                 Log.info (fun m -> m "Using cached file: %s" filename);

     

       75
       75
       +
                 (* Apply processor if provided *)

     

       44
       76
        
                 match processor with

     

       45
       77
        
                 | None -> Ok cache_path

     

       46
       46
       -
                 | Some proc -> Ok (proc cache_path)

     

       47
       47
       -
               ) else (

     

       48
       48
       -
                 (* Cached file is corrupt - remove it and re-download *)

     

       49
       49
       -
                 (try Eio.Path.unlink cache_path with _ -> ());

     

       50
       50
       -
                 (* Fall through to download *)

     

       51
       51
       -
                 fetch_file_to_cache t entry filename processor

     

       52
       52
       -
               )

     

       53
       53
       -
             ) else (

     

       54
       54
       -
               (* File not cached - download it *)

     

       55
       55
       -
               fetch_file_to_cache t entry filename processor

     

       56
       56
       -
             )

     

       78
       78
       +
                 | Some proc ->

     

       79
       79
       +
                     match proc cache_path with

     

       80
       80
       +
                     | Ok path -> Ok path

     

       81
       81
       +
                     | Error e -> Error ("Processor failed: " ^ e)

     

       82
       82
       +
               end else begin

     

       83
       83
       +
                 (* Corrupted cache - delete and re-download *)

     

       84
       84
       +
                 Log.warn (fun m -> m "Cached file failed verification, re-downloading: %s" filename);

     

       85
       85
       +
                 Cache.delete t.cache filename;

     

       86
       86
       +
                 fetch_and_cache t entry filename processor

     

       87
       87
       +
               end

     

       88
       88
       +
             else

     

       89
       89
       +
               (* Not cached - download *)

     

       90
       90
       +
               fetch_and_cache t entry filename processor

     

       57
       91
        
       

     

       58
       58
       -
       and fetch_file_to_cache t entry filename processor =

     

       59
       59
       -
         (* Ensure cache directory exists *)

     

       60
       60
       -
         Cache.ensure_dir t.cache;

     

       61
       61
       -
         

     

       62
       62
       -
         (* Get the download URL *)

     

       63
       63
       -
         let download_url = match Registry.custom_url entry with

     

       92
       92
       +
       and fetch_and_cache t entry filename processor =

     

       93
       93
       +
         (* Get download URL *)

     

       94
       94
       +
         let url = match Registry.custom_url entry with

     

       64
       95
        
           | Some custom -> custom

     

       65
       96
        
           | None -> t.base_url ^ filename

     

       66
       97
        
         in

     

       67
       67
       -
         

     

       68
       68
       -
         (* Download the file *)

     

       69
       69
       -
         let (module D : Downloader.DOWNLOADER) = t.downloader in

     

       70
       70
       -
         let downloader_instance = D.create ~sw:t.sw ~env:t.env () in

     

       71
       71
       -
         let cache_path = Cache.file_path t.cache filename in

     

       72
       72
       -
         let entry_hash = Registry.hash entry in

     

       73
       73
       -
         

     

       74
       74
       -
         match D.download downloader_instance ~url:download_url ~dest:cache_path ~hash:entry_hash () with

     

       98
       98
       +
       

     

       99
       99
       +
         Log.info (fun m -> m "Downloading %s from %s" filename url);

     

       100
       100
       +
       

     

       101
       101
       +
         (* Create progress reporter *)

     

       102
       102
       +
         let progress = Downloader.Progress.create filename in

     

       103
       103
       +
       

     

       104
       104
       +
         (* Check if server supports range requests for chunked download *)

     

       105
       105
       +
         if Downloader.supports_range t.downloader ~url then

     

       106
       106
       +
           fetch_chunked t entry filename url processor

     

       107
       107
       +
         else

     

       108
       108
       +
           fetch_simple t entry filename url progress processor

     

       109
       109
       +
       

     

       110
       110
       +
       and fetch_simple t entry filename url progress processor =

     

       111
       111
       +
         (* Download to temporary location first *)

     

       112
       112
       +
         let temp_path = Eio.Path.(t.env#fs / Filename.temp_dir ~temp_dir:"/tmp" "toru" "") in

     

       113
       113
       +
         let temp_file = Eio.Path.(temp_path / filename) in

     

       114
       114
       +
       

     

       115
       115
       +
         match Downloader.download t.downloader ~url ~dest:temp_file

     

       116
       116
       +
                 ~hash:(Registry.hash entry) ~progress () with

     

       75
       117
        
         | Error msg -> Error ("Download failed: " ^ msg)

     

       76
       118
        
         | Ok () ->

     

       119
       119
       +
             (* Move to cache *)

     

       120
       120
       +
             Eio.Switch.run @@ fun sw ->

     

       121
       121
       +
             let source = Eio.Path.open_in ~sw temp_file in

     

       122
       122
       +
             Cache.put_stream t.cache ~source:(source :> Eio.Flow.source_ty Eio.Resource.t) filename;

     

       123
       123
       +
             Eio.Flow.close source;

     

       124
       124
       +
       

     

       125
       125
       +
             (* Clean up temp file *)

     

       126
       126
       +
             (try Eio.Path.unlink temp_file with _ -> ());

     

       127
       127
       +
       

     

       128
       128
       +
             let cache_path = Cache.file_path t.cache filename in

     

       129
       129
       +
       

     

       77
       130
        
             (* Apply processor if provided *)

     

       78
       131
        
             match processor with

     

       79
       132
        
             | None -> Ok cache_path

     

       80
       80
       -
             | Some proc -> Ok (proc cache_path)

     

       133
       133
       +
             | Some proc ->

     

       134
       134
       +
                 match proc cache_path with

     

       135
       135
       +
                 | Ok path -> Ok path

     

       136
       136
       +
                 | Error e -> Error ("Processor failed: " ^ e)

     

       137
       137
       +
       

     

       138
       138
       +
       and fetch_chunked t entry filename url processor =

     

       139
       139
       +
         Log.info (fun m -> m "Using chunked download for %s" filename);

     

       140
       140
       +
       

     

       141
       141
       +
         (* Get total size *)

     

       142
       142
       +
         match Downloader.get_content_length t.downloader ~url with

     

       143
       143
       +
         | Error e ->

     

       144
       144
       +
             Log.warn (fun m -> m "Cannot get content length, falling back to simple download: %s" e);

     

       145
       145
       +
             fetch_simple t entry filename url (Downloader.Progress.create filename) processor

     

       146
       146
       +
         | Ok total_size ->

     

       147
       147
       +
             (* Check existing chunks *)

     

       148
       148
       +
             let missing_ranges = Cache.missing_ranges t.cache ~filename ~total_size in

     

       81
       149
        
       

     

       82
       82
       -
       let fetch_all t ?concurrency:_ () =

     

       83
       83
       -
         let all_entries = Registry.entries t.registry in

     

       84
       84
       -
         let total = List.length all_entries in

     

       85
       85
       -
         

     

       86
       86
       -
         if total = 0 then

     

       150
       150
       +
             if missing_ranges = [] then begin

     

       151
       151
       +
               (* All chunks present - coalesce *)

     

       152
       152
       +
               Log.info (fun m -> m "All chunks present, coalescing");

     

       153
       153
       +
               let promise = Cache.coalesce_chunks t.cache ~filename in

     

       154
       154
       +
               match Eio.Promise.await promise with

     

       155
       155
       +
               | Ok () ->

     

       156
       156
       +
                   let cache_path = Cache.file_path t.cache filename in

     

       157
       157
       +
                   (* Verify hash *)

     

       158
       158
       +
                   let entry_hash = Registry.hash entry in

     

       159
       159
       +
                   if Hash.verify cache_path entry_hash then

     

       160
       160
       +
                     (* Apply processor if provided *)

     

       161
       161
       +
                     match processor with

     

       162
       162
       +
                     | None -> Ok cache_path

     

       163
       163
       +
                     | Some proc ->

     

       164
       164
       +
                         match proc cache_path with

     

       165
       165
       +
                         | Ok path -> Ok path

     

       166
       166
       +
                         | Error e -> Error ("Processor failed: " ^ e)

     

       167
       167
       +
                   else

     

       168
       168
       +
                     Error "Hash verification failed after coalescing chunks"

     

       169
       169
       +
               | Error e ->

     

       170
       170
       +
                   Error (Printf.sprintf "Failed to coalesce chunks: %s" (Printexc.to_string e))

     

       171
       171
       +
             end else begin

     

       172
       172
       +
               (* Download missing chunks *)

     

       173
       173
       +
               let rec download_chunks = function

     

       174
       174
       +
                 | [] -> Ok ()

     

       175
       175
       +
                 | (start, end_) :: rest ->

     

       176
       176
       +
                     Log.info (fun m -> m "Downloading chunk %Ld-%Ld" start end_);

     

       177
       177
       +
                     match Downloader.download_range t.downloader ~url ~start ~end_:(Some end_) with

     

       178
       178
       +
                     | Error e -> Error e

     

       179
       179
       +
                     | Ok data ->

     

       180
       180
       +
                         (* Save chunk *)

     

       181
       181
       +
                         Eio.Switch.run @@ fun _sw ->

     

       182
       182
       +
                         let source = Eio.Flow.string_source data in

     

       183
       183
       +
                         let range = Cacheio.Range.create ~start ~end_ in

     

       184
       184
       +
                         Cache.put_chunk t.cache ~filename ~range ~source;

     

       185
       185
       +
                         download_chunks rest

     

       186
       186
       +
               in

     

       187
       187
       +
       

     

       188
       188
       +
               (* Download all missing ranges *)

     

       189
       189
       +
               match download_chunks missing_ranges with

     

       190
       190
       +
               | Error e -> Error ("Chunk download failed: " ^ e)

     

       191
       191
       +
               | Ok () ->

     

       192
       192
       +
                   (* Coalesce chunks *)

     

       193
       193
       +
                   Log.info (fun m -> m "All chunks downloaded, coalescing");

     

       194
       194
       +
                   let promise = Cache.coalesce_chunks t.cache ~filename in

     

       195
       195
       +
                   match Eio.Promise.await promise with

     

       196
       196
       +
                   | Ok () ->

     

       197
       197
       +
                       let cache_path = Cache.file_path t.cache filename in

     

       198
       198
       +
                       (* Verify hash *)

     

       199
       199
       +
                       let entry_hash = Registry.hash entry in

     

       200
       200
       +
                       if Hash.verify cache_path entry_hash then

     

       201
       201
       +
                         (* Apply processor if provided *)

     

       202
       202
       +
                         match processor with

     

       203
       203
       +
                         | None -> Ok cache_path

     

       204
       204
       +
                         | Some proc ->

     

       205
       205
       +
                             match proc cache_path with

     

       206
       206
       +
                             | Ok path -> Ok path

     

       207
       207
       +
                             | Error e -> Error ("Processor failed: " ^ e)

     

       208
       208
       +
                       else

     

       209
       209
       +
                         Error "Hash verification failed after downloading chunks"

     

       210
       210
       +
                   | Error e ->

     

       211
       211
       +
                       Error (Printf.sprintf "Failed to coalesce chunks: %s" (Printexc.to_string e))

     

       212
       212
       +
             end

     

       213
       213
       +
       

     

       214
       214
       +
       (** {1 Batch Operations} *)

     

       215
       215
       +
       

     

       216
       216
       +
       let fetch_all t ?concurrency () =

     

       217
       217
       +
         let entries = Registry.entries t.registry in

     

       218
       218
       +
         let total = List.length entries in

     

       219
       219
       +
       

     

       220
       220
       +
         if total = 0 then begin

     

       221
       221
       +
           Log.info (fun m -> m "No files to fetch");

     

       87
       222
        
           Ok ()

     

       88
       88
       -
         else (

     

       89
       89
       -
           (* Create a semaphore to limit concurrency *)

     

       90
       90
       -
           let results = ref [] in

     

       223
       223
       +
         end else begin

     

       224
       224
       +
           Log.info (fun m -> m "Fetching %d files" total);

     

       225
       225
       +
       

     

       226
       226
       +
           (* Determine concurrency *)

     

       227
       227
       +
           let concurrency = Option.value concurrency ~default:4 in

     

       228
       228
       +
       

     

       229
       229
       +
           (* Use Eio fibers for concurrent downloads *)

     

       91
       230
        
           let errors = ref [] in

     

       92
       231
        
           let completed = ref 0 in

     

       93
       93
       -
           

     

       94
       94
       -
           (* Process entries in batches *)

     

       95
       95
       -
           let rec process_batch entries =

     

       232
       232
       +
           let mutex = Eio.Mutex.create () in

     

       233
       233
       +
       

     

       234
       234
       +
           (* Create a pool of workers *)

     

       235
       235
       +
           let rec process_entries entries =

     

       96
       236
        
             match entries with

     

       97
       97
       -
             | [] -> 

     

       98
       98
       -
                 if !completed = total then

     

       99
       99
       -
                   if List.length !errors > 0 then

     

       100
       100
       -
                     Error ("Multiple failures: " ^ String.concat "; " !errors)

     

       237
       237
       +
             | [] -> ()

     

       238
       238
       +
             | batch ->

     

       239
       239
       +
                 (* Take up to concurrency items *)

     

       240
       240
       +
                 let current_batch, rest =

     

       241
       241
       +
                   if List.length batch <= concurrency then

     

       242
       242
       +
                     batch, []

     

       101
       243
        
                   else

     

       102
       102
       -
                     Ok ()

     

       103
       103
       -
                 else

     

       104
       104
       -
                   Error "Internal error: not all files processed"

     

       105
       105
       -
             | entry :: rest ->

     

       106
       106
       -
                 let filename = Registry.filename entry in

     

       107
       107
       -
                 (match fetch t ~filename () with

     

       108
       108
       -
                  | Ok path -> 

     

       109
       109
       -
                      results := path :: !results;

     

       110
       110
       -
                      incr completed

     

       111
       111
       -
                  | Error msg ->

     

       112
       112
       -
                      errors := (filename ^ ": " ^ msg) :: !errors;

     

       113
       113
       -
                      incr completed);

     

       114
       114
       -
                 process_batch rest

     

       244
       244
       +
                     let rec take n acc = function

     

       245
       245
       +
                       | [] -> List.rev acc, []

     

       246
       246
       +
                       | _ when n = 0 -> List.rev acc, batch

     

       247
       247
       +
                       | h :: t -> take (n - 1) (h :: acc) t

     

       248
       248
       +
                     in

     

       249
       249
       +
                     take concurrency [] batch

     

       250
       250
       +
                 in

     

       251
       251
       +
       

     

       252
       252
       +
                 (* Process batch concurrently *)

     

       253
       253
       +
                 Eio.Fiber.all (List.map (fun entry ->

     

       254
       254
       +
                   fun () ->

     

       255
       255
       +
                     let filename = Registry.filename entry in

     

       256
       256
       +
                     match fetch t ~filename () with

     

       257
       257
       +
                     | Ok _ ->

     

       258
       258
       +
                         Eio.Mutex.use_rw ~protect:false mutex (fun () ->

     

       259
       259
       +
                           incr completed;

     

       260
       260
       +
                           Log.info (fun m -> m "Progress: %d/%d files" !completed total)

     

       261
       261
       +
                         )

     

       262
       262
       +
                     | Error msg ->

     

       263
       263
       +
                         Eio.Mutex.use_rw ~protect:false mutex (fun () ->

     

       264
       264
       +
                           errors := (filename ^ ": " ^ msg) :: !errors;

     

       265
       265
       +
                           incr completed

     

       266
       266
       +
                         )

     

       267
       267
       +
                 ) current_batch);

     

       268
       268
       +
       

     

       269
       269
       +
                 (* Process next batch *)

     

       270
       270
       +
                 process_entries rest

     

       115
       271
        
           in

     

       116
       116
       -
           

     

       117
       117
       -
           (* For now, implement simple sequential processing *)

     

       118
       118
       -
           (* TODO: Add actual concurrent processing with Eio fibers *)

     

       119
       119
       -
           process_batch all_entries

     

       120
       120
       -
         )

     

       272
       272
       +
       

     

       273
       273
       +
           process_entries entries;

     

       274
       274
       +
       

     

       275
       275
       +
           if !errors = [] then

     

       276
       276
       +
             Ok ()

     

       277
       277
       +
           else

     

       278
       278
       +
             Error (Printf.sprintf "%d failures: %s"

     

       279
       279
       +
               (List.length !errors)

     

       280
       280
       +
               (String.concat "; " !errors))

     

       281
       281
       +
         end

     

       282
       282
       +
       

     

       283
       283
       +
       (** {1 Registry Management} *)

     

       121
       284
        
       

     

       122
       285
        
       let load_registry t source =

     

       123
       123
       -
         let new_registry = 

     

       286
       286
       +
         Log.info (fun m -> m "Loading registry from: %s" source);

     

       287
       287
       +
         let new_registry =

     

       124
       288
        
           if String.contains source '/' || String.contains source ':' then

     

       125
       289
        
             Registry.load_from_url source

     

       126
       290
        
           else

     

       127
       127
       -
             (* TODO: Need a way to resolve relative paths *)

     

       128
       128
       -
             failwith "load_registry: relative paths not yet supported"

     

       291
       291
       +
             Registry.load Eio.Path.(t.env#fs / source)

     

       129
       292
        
         in

     

       130
       293
        
         { t with registry = new_registry }

     

       131
       294
        
       

     
···

       134
       297
        
         { t with registry = new_registry }

     

       135
       298
        
       

     

       136
       299
        
       let update_base_url t new_url =

     

       300
       300
       +
         Log.info (fun m -> m "Updating base URL to: %s" new_url);

     

       137
       301
        
         { t with base_url = new_url }

     

       138
       302
        
       

     

       139
       139
       -
       let retrieve ~sw ~env ~url ?hash ?cache_path ?downloader () =

     

       140
       140
       -
         (* Get cache path *)

     

       303
       303
       +
       (** {1 One-off Downloads} *)

     

       304
       304
       +
       

     

       305
       305
       +
       let retrieve ~sw ~env ~url ?hash ?cache_path () =

     

       306
       306
       +
         Log.info (fun m -> m "Retrieving file from: %s" url);

     

       307
       307
       +
       

     

       308
       308
       +
         (* Create temporary cache if needed *)

     

       141
       309
        
         let cache_dir = match cache_path with

     

       142
       310
        
           | Some path -> path

     

       143
       311
        
           | None -> Cache.default_cache_path ~app_name:"toru-temp" ()

     

       144
       312
        
         in

     

       145
       145
       -
         

     

       146
       146
       -
         (* Create a temporary cache *)

     

       313
       313
       +
       

     

       147
       314
        
         let cache = Cache.create ~sw ~fs:env#fs cache_dir in

     

       148
       315
        
         Cache.ensure_dir cache;

     

       149
       149
       -
         

     

       316
       316
       +
       

     

       150
       317
        
         (* Extract filename from URL *)

     

       151
       151
       -
         let filename = 

     

       318
       318
       +
         let filename =

     

       152
       319
        
           match String.rindex_opt url '/' with

     

       153
       320
        
           | Some idx -> String.sub url (idx + 1) (String.length url - idx - 1)

     

       154
       154
       -
           | None -> "downloaded_file"

     

       321
       321
       +
           | None -> "download"

     

       155
       322
        
         in

     

       156
       156
       -
         

     

       157
       157
       -
         (* Get downloader *)

     

       158
       158
       -
         let downloader_module = match downloader with

     

       159
       159
       -
           | Some d -> d

     

       160
       160
       -
           | None -> Downloader.Downloaders.create_default ~env

     

       161
       161
       -
         in

     

       162
       162
       -
         

     

       323
       323
       +
       

     

       163
       324
        
         (* Download the file *)

     

       164
       164
       -
         let (module D : Downloader.DOWNLOADER) = downloader_module in

     

       165
       165
       -
         let downloader_instance = D.create ~sw ~env () in

     

       166
       325
        
         let dest_path = Cache.file_path cache filename in

     

       167
       167
       -
         

     

       168
       168
       -
         match D.download downloader_instance ~url ~dest:dest_path ?hash () with

     

       326
       326
       +
         let downloader = Downloader.create ~sw ~env () in

     

       327
       327
       +
         let progress = Downloader.Progress.create filename in

     

       328
       328
       +
       

     

       329
       329
       +
         match Downloader.download downloader ~url ~dest:dest_path ?hash ~progress () with

     

       169
       330
        
         | Error msg -> Error ("Download failed: " ^ msg)

     

       170
       331
        
         | Ok () -> Ok dest_path

     

       171
       332
        
       

     

       333
       333
       +
       (** {1 Cache Management} *)

     

       334
       334
       +
       

     

       335
       335
       +
       let clear_cache t =

     

       336
       336
       +
         Log.info (fun m -> m "Clearing cache");

     

       337
       337
       +
         Cache.clear t.cache

     

       338
       338
       +
       

     

       339
       339
       +
       let cache_stats t =

     

       340
       340
       +
         Cache.stats t.cache

     

       341
       341
       +
       

     

       342
       342
       +
       let expire_cache t =

     

       343
       343
       +
         Log.info (fun m -> m "Expiring old cache entries");

     

       344
       344
       +
         Cache.expire t.cache

     

       345
       345
       +
       

     

       346
       346
       +
       (** {1 Utility Functions} *)

     

       347
       347
       +
       

     

       172
       348
        
       let default_cache_path = Cache.default_cache_path

     

       173
       349
        
       

     

       350
       350
       +
       (** {1 Pretty Printing} *)

     

       351
       351
       +
       

     

       174
       352
        
       let pp fmt t =

     

       175
       175
       -
         Format.fprintf fmt "Toru instance:\n";

     

       176
       176
       -
         Format.fprintf fmt "  Base URL: %s\n" t.base_url;

     

       177
       177
       -
         Format.fprintf fmt "  Registry: %d entries\n" (Registry.size t.registry);

     

       178
       178
       -
         Format.fprintf fmt "  Cache: %a\n" Cache.pp t.cache

     

       353
       353
       +
         Format.fprintf fmt "Toru{base_url=%s, registry=%d entries, %a}"

     

       354
       354
       +
           t.base_url

     

       355
       355
       +
           (Registry.size t.registry)

     

       356
       356
       +
           Cache.pp t.cache

     

       357
       357
       +
       

     

       358
       358
       +
       (** {1 Module Exports} *)

     

       179
       359
        
       

     

       180
       360
        
       module Hash = Hash

     

       181
       361
        
       module Registry = Registry

+17 -4

stack/toru/lib/toru/toru.mli

···

       6
       6
        
       (** {1 Construction} *)

     

       7
       7
        
       

     

       8
       8
        
       (** Create a Toru instance *)

     

       9
       9
       -
       val create : 

     

       9
       9
       +
       val create :

     

       10
       10
        
         sw:Eio.Switch.t ->

     

       11
       11
        
         env:Eio_unix.Stdenv.base ->

     

       12
       12
        
         base_url:string ->

     
···

       14
       14
        
         ?version:string ->

     

       15
       15
        
         ?registry_file:string ->

     

       16
       16
        
         ?registry_url:string ->

     

       17
       17
       -
         ?downloader:(module Downloader.DOWNLOADER) ->

     

       17
       17
       +
         ?downloader_config:Downloader.config ->

     

       18
       18
        
         unit -> t

     

       19
       19
        
       

     

       20
       20
        
       (** {1 Field accessors} *)

     
···

       28
       28
        
       (** Get registry instance *)

     

       29
       29
        
       val registry : t -> Registry.t

     

       30
       30
        
       

     

       31
       31
       +
       (** Get downloader instance *)

     

       32
       32
       +
       val downloader : t -> Downloader.t

     

       33
       33
       +
       

     

       31
       34
        
       (** {1 File operations} *)

     

       32
       35
        
       

     

       33
       36
        
       (** Fetch a single file from registry *)

     
···

       59
       62
        
       (** {1 Static functions} *)

     

       60
       63
        
       

     

       61
       64
        
       (** Retrieve a single file without registry *)

     

       62
       62
       -
       val retrieve : 

     

       65
       65
       +
       val retrieve :

     

       63
       66
        
         sw:Eio.Switch.t ->

     

       64
       67
        
         env:Eio_unix.Stdenv.base ->

     

       65
       68
        
         url:string ->

     

       66
       69
        
         ?hash:Hash.t ->

     

       67
       70
        
         ?cache_path:string ->

     

       68
       68
       -
         ?downloader:(module Downloader.DOWNLOADER) ->

     

       69
       71
        
         unit -> (Eio.Fs.dir_ty Eio.Path.t, string) result

     

       70
       72
        
       

     

       71
       73
        
       (** Get default cache path for application *)

     

       72
       74
        
       val default_cache_path : ?app_name:string -> unit -> string

     

       75
       75
       +
       

     

       76
       76
       +
       (** {1 Cache Management} *)

     

       77
       77
       +
       

     

       78
       78
       +
       (** Clear all cached files *)

     

       79
       79
       +
       val clear_cache : t -> unit

     

       80
       80
       +
       

     

       81
       81
       +
       (** Get cache statistics *)

     

       82
       82
       +
       val cache_stats : t -> Cacheio.Stats.t

     

       83
       83
       +
       

     

       84
       84
       +
       (** Expire old cache entries *)

     

       85
       85
       +
       val expire_cache : t -> int

     

       73
       86
        
       

     

       74
       87
        
       (** {1 Pretty printing} *)

     

       75
       88

-193

stack/toru/test/CACHE_IMPLEMENTATION_REPORT.md

···

       1
       1
       -
       # Cache Module Implementation Report

     

       2
       2
       -
       

     

       3
       3
       -
       ## Overview

     

       4
       4
       -
       

     

       5
       5
       -
       The Cache module has been fully implemented according to the CLAUDE.md specification with extensive additional management APIs. The implementation provides robust, cross-platform cache management with XDG Base Directory compliance and comprehensive file operations.

     

       6
       6
       -
       

     

       7
       7
       -
       ## Core Implementation Features

     

       8
       8
       -
       

     

       9
       9
       -
       ### ✅ Basic Interface (From CLAUDE.md)

     

       10
       10
       -
       

     

       11
       11
       -
       **Construction Functions:**

     

       12
       12
       -
       - `create ~sw ~env ?version path` - Create cache with explicit path

     

       13
       13
       -
       - `default ~sw ~env ?app_name ()` - Create cache using OS-specific default location

     

       14
       14
       -
       

     

       15
       15
       -
       **Field Accessors:**

     

       16
       16
       -
       - `base_path t` - Get base cache directory path

     

       17
       17
       -
       - `version t` - Get version string (if any)

     

       18
       18
       -
       

     

       19
       19
       -
       **Basic Operations:**

     

       20
       20
       -
       - `file_path t filename` - Get full path for file in cache

     

       21
       21
       -
       - `exists t filename` - Check if file exists in cache

     

       22
       22
       -
       - `ensure_dir t` - Create cache directories lazily

     

       23
       23
       -
       - `clear t` - Remove all files from cache

     

       24
       24
       -
       - `size_bytes t` - Calculate total cache size

     

       25
       25
       -
       - `list_files t` - List all files in cache recursively

     

       26
       26
       -
       

     

       27
       27
       -
       ### ✅ Extended Management APIs

     

       28
       28
       -
       

     

       29
       29
       -
       **Advanced Cache Management:**

     

       30
       30
       -
       - `file_info t filename` - Get file size and modification time

     

       31
       31
       -
       - `usage_stats t` - Get comprehensive cache statistics

     

       32
       32
       -
       - `trim_to_size t max_size` - Remove oldest files to fit size limit

     

       33
       33
       -
       - `trim_by_age t max_age_days` - Remove files older than N days

     

       34
       34
       -
       - `vacuum t` - Clean up empty directories and broken symlinks

     

       35
       35
       -
       

     

       36
       36
       -
       **Types:**

     

       37
       37
       -
       ```ocaml

     

       38
       38
       -
       type file_info = { size: int64; mtime: float }

     

       39
       39
       -
       type usage_stats = { 

     

       40
       40
       -
         total_size: int64; 

     

       41
       41
       -
         file_count: int; 

     

       42
       42
       -
         oldest: float; 

     

       43
       43
       -
         newest: float 

     

       44
       44
       -
       }

     

       45
       45
       -
       ```

     

       46
       46
       -
       

     

       47
       47
       -
       ## Platform Support & XDG Compliance

     

       48
       48
       -
       

     

       49
       49
       -
       ### ✅ Cross-Platform Default Paths

     

       50
       50
       -
       

     

       51
       51
       -
       The implementation correctly detects the platform and follows OS-specific conventions:

     

       52
       52
       -
       

     

       53
       53
       -
       **macOS (Darwin):**

     

       54
       54
       -
       - Path: `~/Library/Caches/<app_name>`

     

       55
       55
       -
       - Detection: Uses `uname -s` to identify Darwin kernel

     

       56
       56
       -
       - Verified: ✓ Working on macOS

     

       57
       57
       -
       

     

       58
       58
       -
       **Linux/Unix:**

     

       59
       59
       -
       - XDG compliant: `$XDG_CACHE_HOME/<app_name>` if set

     

       60
       60
       -
       - Fallback: `~/.cache/<app_name>`

     

       61
       61
       -
       - Verified: ✓ XDG environment variable override support

     

       62
       62
       -
       

     

       63
       63
       -
       **Windows:**

     

       64
       64
       -
       - Path: `%LOCALAPPDATA%\<app_name>\Cache`

     

       65
       65
       -
       - Fallback: `%USERPROFILE%\AppData\Local\<app_name>\Cache`

     

       66
       66
       -
       

     

       67
       67
       -
       ### ✅ Versioned Cache Support

     

       68
       68
       -
       

     

       69
       69
       -
       Supports optional version subdirectories:

     

       70
       70
       -
       - Without version: `cache_dir/file.txt`

     

       71
       71
       -
       - With version: `cache_dir/v1.0/file.txt`

     

       72
       72
       -
       - Automatic directory creation for both modes

     

       73
       73
       -
       

     

       74
       74
       -
       ## Implementation Details

     

       75
       75
       -
       

     

       76
       76
       -
       ### Directory Creation Strategy

     

       77
       77
       -
       

     

       78
       78
       -
       Uses lazy directory creation with proper error handling:

     

       79
       79
       -
       - Creates parent directories recursively when needed

     

       80
       80
       -
       - Handles concurrent directory creation safely

     

       81
       81
       -
       - Uses appropriate permissions (0o755 for directories, 0o644 for files)

     

       82
       82
       -
       

     

       83
       83
       -
       ### File Management

     

       84
       84
       -
       

     

       85
       85
       -
       **Efficient Operations:**

     

       86
       86
       -
       - Recursive directory traversal for listing and size calculation

     

       87
       87
       -
       - Safe file operations with proper error handling

     

       88
       88
       -
       - Atomic operations where possible

     

       89
       89
       -
       

     

       90
       90
       -
       **Trim Operations:**

     

       91
       91
       -
       - `trim_to_size`: Removes oldest files first (LRU-like behavior)

     

       92
       92
       -
       - `trim_by_age`: Removes files older than specified days

     

       93
       93
       -
       - Both operations preserve cache structure and handle errors gracefully

     

       94
       94
       -
       

     

       95
       95
       -
       ### Memory and Performance

     

       96
       96
       -
       

     

       97
       97
       -
       **Optimizations:**

     

       98
       98
       -
       - Minimal memory allocations in recursive operations

     

       99
       99
       -
       - Efficient file statistics collection

     

       100
       100
       -
       - Lazy evaluation where appropriate

     

       101
       101
       -
       

     

       102
       102
       -
       ## Test Coverage

     

       103
       103
       -
       

     

       104
       104
       -
       ### ✅ Comprehensive Test Suite

     

       105
       105
       -
       

     

       106
       106
       -
       **test_cache.ml - Basic Functionality:**

     

       107
       107
       -
       - Cache creation and directory management

     

       108
       108
       -
       - File operations (create, check existence, list)

     

       109
       109
       -
       - Size calculation and statistics

     

       110
       110
       -
       - Clear operation

     

       111
       111
       -
       - Trim operations (size and age-based)

     

       112
       112
       -
       - Vacuum functionality

     

       113
       113
       -
       

     

       114
       114
       -
       **test_cache_xdg.ml - Platform & XDG Compliance:**

     

       115
       115
       -
       - XDG Base Directory specification compliance

     

       116
       116
       -
       - Platform-specific path detection (macOS vs Linux)

     

       117
       117
       -
       - Environment variable override support

     

       118
       118
       -
       - Versioned cache directory handling

     

       119
       119
       -
       - Advanced cache management features

     

       120
       120
       -
       

     

       121
       121
       -
       ### ✅ Test Results

     

       122
       122
       -
       

     

       123
       123
       -
       All tests pass successfully:

     

       124
       124
       -
       - ✅ XDG compliance verified on macOS

     

       125
       125
       -
       - ✅ Versioned cache directories work correctly  

     

       126
       126
       -
       - ✅ File operations robust and reliable

     

       127
       127
       -
       - ✅ Cache management functions operate correctly

     

       128
       128
       -
       - ✅ Cross-platform path detection works

     

       129
       129
       -
       

     

       130
       130
       -
       ## API Usage Examples

     

       131
       131
       -
       

     

       132
       132
       -
       ### Basic Usage

     

       133
       133
       -
       ```ocaml

     

       134
       134
       -
       let cache = Cache.create ~sw ~env ~version:"v1.0" "my-data-cache" in

     

       135
       135
       -
       Cache.ensure_dir cache;

     

       136
       136
       -
       let file_path = Cache.file_path cache "dataset.csv" in

     

       137
       137
       -
       let exists = Cache.exists cache "dataset.csv" in

     

       138
       138
       -
       ```

     

       139
       139
       -
       

     

       140
       140
       -
       ### Management Operations

     

       141
       141
       -
       ```ocaml

     

       142
       142
       -
       (* Get cache statistics *)

     

       143
       143
       -
       let stats = Cache.usage_stats cache in

     

       144
       144
       -
       printf "Total: %Ld bytes, %d files\n" stats.total_size stats.file_count;

     

       145
       145
       -
       

     

       146
       146
       -
       (* Trim cache to 100MB *)

     

       147
       147
       -
       Cache.trim_to_size cache (Int64.mul 100L 1024L 1024L);

     

       148
       148
       -
       

     

       149
       149
       -
       (* Remove files older than 30 days *)

     

       150
       150
       -
       Cache.trim_by_age cache 30.0;

     

       151
       151
       -
       

     

       152
       152
       -
       (* Clean up empty directories *)

     

       153
       153
       -
       Cache.vacuum cache;

     

       154
       154
       -
       ```

     

       155
       155
       -
       

     

       156
       156
       -
       ### Platform-Aware Defaults

     

       157
       157
       -
       ```ocaml

     

       158
       158
       -
       (* Uses OS-appropriate default location *)

     

       159
       159
       -
       let cache = Cache.default ~sw ~env ~app_name:"my-app" () in

     

       160
       160
       -
       

     

       161
       161
       -
       (* Get platform-specific default path *)

     

       162
       162
       -
       let default_path = Cache.default_cache_path ~app_name:"my-app" () in

     

       163
       163
       -
       ```

     

       164
       164
       -
       

     

       165
       165
       -
       ## Integration with Toru Ecosystem

     

       166
       166
       -
       

     

       167
       167
       -
       The Cache module integrates seamlessly with other Toru components:

     

       168
       168
       -
       - **Registry Module**: Files from registry stored in cache

     

       169
       169
       -
       - **Downloader Module**: Downloads saved to cache locations  

     

       170
       170
       -
       - **Hash Module**: Cached files verified against registry hashes

     

       171
       171
       -
       - **Main Toru API**: Cache provides storage backend

     

       172
       172
       -
       

     

       173
       173
       -
       ## Future Extensions

     

       174
       174
       -
       

     

       175
       175
       -
       The current implementation provides a solid foundation for:

     

       176
       176
       -
       - Cache quota management

     

       177
       177
       -
       - LRU/LFU eviction policies  

     

       178
       178
       -
       - Cache statistics and monitoring

     

       179
       179
       -
       - Compression and deduplication

     

       180
       180
       -
       - Network-aware cache policies

     

       181
       181
       -
       

     

       182
       182
       -
       ## Conclusion

     

       183
       183
       -
       

     

       184
       184
       -
       The Cache module implementation fully satisfies the CLAUDE.md specification and extends it with comprehensive management capabilities. It provides robust, cross-platform cache management that follows OS conventions and integrates well with the broader Toru library ecosystem.

     

       185
       185
       -
       

     

       186
       186
       -
       **Key Achievements:**

     

       187
       187
       -
       - ✅ Complete CLAUDE.md specification implementation

     

       188
       188
       -
       - ✅ Extended management APIs beyond specification  

     

       189
       189
       -
       - ✅ Full XDG Base Directory compliance

     

       190
       190
       -
       - ✅ Cross-platform support (macOS, Linux, Windows)

     

       191
       191
       -
       - ✅ Comprehensive test coverage

     

       192
       192
       -
       - ✅ Production-ready error handling and edge cases

     

       193
       193
       -
       - ✅ Efficient file operations and memory usage

-143

stack/toru/test/basic.t

···

       1
       1
       -
       Basic Toru CLI Tests

     

       2
       2
       -
       ===================

     

       3
       3
       -
       

     

       4
       4
       -
       Setup test environment and use actual test registry:

     

       5
       5
       -
         $ cd $TESTDIR/..

     

       6
       6
       -
         $ export TORU_CACHE_DIR=/tmp/toru-test-cache

     

       7
       7
       -
         $ mkdir -p /tmp/toru-test-cache

     

       8
       8
       -
       

     

       9
       9
       -
       Test 1: Basic inspect functionality

     

       10
       10
       -
       ===================================

     

       11
       11
       -
       

     

       12
       12
       -
       Test basic inspect without options:

     

       13
       13
       -
         $ dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt

     

       14
       14
       -
         Registry contains 8 entries

     

       15
       15
       -
       

     

       16
       16
       -
       Test inspect with verbose logging:

     

       17
       17
       -
         $ dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose

     

       18
       18
       -
         toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt

     

       19
       19
       -
         Registry contains 8 entries

     

       20
       20
       -
       

     

       21
       21
       -
       Test inspect with debug logging:

     

       22
       22
       -
         $ dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose --verbose 2>&1 | head -6

     

       23
       23
       -
         toru_main.exe: [DEBUG] Starting inspect function

     

       24
       24
       -
         toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt

     

       25
       25
       -
         toru_main.exe: [DEBUG] Determining if source is URL or file path

     

       26
       26
       -
         toru_main.exe: [DEBUG] Detected file path, loading directly without Eio

     

       27
       27
       -
         toru_main.exe: [DEBUG] Registry loaded successfully

     

       28
       28
       -
         Registry contains 8 entries

     

       29
       29
       -
       

     

       30
       30
       -
       Test inspect with statistics:

     

       31
       31
       -
         $ dune exec bin/toru_main.exe -- inspect --stats test/python/test_registry_sha256.txt | head -10

     

       32
       32
       -
         Registry contains 8 entries

     

       33
       33
       -
         

     

       34
       34
       -
         Registry Statistics:

     

       35
       35
       -
         ===================

     

       36
       36
       -
         Hash algorithms:

     

       37
       37
       -
           sha256: 8 entries

     

       38
       38
       -
         

     

       39
       39
       -
         File extensions:

     

       40
       40
       -
           .txt: 5 files

     

       41
       41
       -
           .json: 1 files

     

       42
       42
       -
       

     

       43
       43
       -
       Test inspect with file listing:

     

       44
       44
       -
         $ dune exec bin/toru_main.exe -- inspect --list test/python/test_registry_sha256.txt | head -10

     

       45
       45
       -
         Registry contains 8 entries

     

       46
       46
       -
         

     

       47
       47
       -
         Files in registry:

     

       48
       48
       -
         ==================

     

       49
       49
       -
         9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25  .hidden/secret.txt

     

       50
       50
       -
         dcfcfeb20e6334be05d4ed2e39da77ffb84b80bf835dc20ac9963f5e95820b94  config.json

     

       51
       51
       -
         d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef  data/numbers.csv

     

       52
       52
       -
         dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f  data/simple.txt

     

       53
       53
       -
         dcccba292708e4d9b8f1e2af482b09b01131ea6e22c3b10173e0d75c8ac0c310  data/unicode.txt

     

       54
       54
       -
       

     

       55
       55
       -
       Test inspect with search:

     

       56
       56
       -
         $ dune exec bin/toru_main.exe -- inspect --search .txt test/python/test_registry_sha256.txt | head -10

     

       57
       57
       -
         Registry contains 8 entries

     

       58
       58
       -
         

     

       59
       59
       -
         Files matching '.txt':

     

       60
       60
       -
         =======================

     

       61
       61
       -
         9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25  .hidden/secret.txt

     

       62
       62
       -
         dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f  data/simple.txt

     

       63
       63
       -
         dcccba292708e4d9b8f1e2af482b09b01131ea6e22c3b10173e0d75c8ac0c310  data/unicode.txt

     

       64
       64
       -
         23c8cf9515ed231a55e63e1d89399a1eac4a529c2b4ac5af29a61af03e3afdd4  docs/readme.md

     

       65
       65
       -
         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  empty.txt

     

       66
       66
       -
       

     

       67
       67
       -
       Test 2: Validate functionality

     

       68
       68
       -
       ==============================

     

       69
       69
       -
       

     

       70
       70
       -
       Test basic validate:

     

       71
       71
       -
         $ dune exec bin/toru_main.exe -- validate test/python/test_registry_sha256.txt

     

       72
       72
       -
         ✓ Registry loaded successfully with 8 entries

     

       73
       73
       -
       

     

       74
       74
       -
       Test validate with hash checking:

     

       75
       75
       -
         $ dune exec bin/toru_main.exe -- validate --check-hashes test/python/test_registry_sha256.txt

     

       76
       76
       -
         ✓ Registry loaded successfully with 8 entries

     

       77
       77
       -
         Validating hash formats...

     

       78
       78
       -
         ✓ All 8 hash formats are valid

     

       79
       79
       -
       

     

       80
       80
       -
       Test 3: Convert functionality

     

       81
       81
       -
       ============================

     

       82
       82
       -
       

     

       83
       83
       -
       Test convert to new file:

     

       84
       84
       -
         $ dune exec bin/toru_main.exe -- convert test/python/test_registry_sha256.txt /tmp/converted.txt

     

       85
       85
       -
         ✓ Converted 8 entries from test/python/test_registry_sha256.txt to /tmp/converted.txt

     

       86
       86
       -
       

     

       87
       87
       -
       Verify converted file has correct content:

     

       88
       88
       -
         $ head -5 /tmp/converted.txt

     

       89
       89
       -
         # Pooch registry generated on *-*-*T*:*:*.* (glob)

     

       90
       90
       -
         # Algorithm: sha256

     

       91
       91
       -
         .hidden/secret.txt 9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25

     

       92
       92
       -
         config.json dcfcfeb20e6334be05d4ed2e39da77ffb84b80bf835dc20ac9963f5e95820b94

     

       93
       93
       -
         data/numbers.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef

     

       94
       94
       -
       

     

       95
       95
       -
       Test 4: XDG cache directory handling

     

       96
       96
       -
       ===================================

     

       97
       97
       -
       

     

       98
       98
       -
       Test with XDG_CACHE_HOME:

     

       99
       99
       -
         $ XDG_CACHE_HOME=/tmp/xdg-test dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose

     

       100
       100
       -
         toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt

     

       101
       101
       -
         Registry contains 8 entries

     

       102
       102
       -
       

     

       103
       103
       -
       Test with application-specific cache dir:

     

       104
       104
       -
         $ TORU_CACHE_DIR=/tmp/app-test dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose

     

       105
       105
       -
         toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt

     

       106
       106
       -
         Registry contains 8 entries

     

       107
       107
       -
       

     

       108
       108
       -
       Test 5: Help system

     

       109
       109
       -
       ===================

     

       110
       110
       -
       

     

       111
       111
       -
       Test main help shows XDG info:

     

       112
       112
       -
         $ dune exec bin/toru_main.exe -- --help | grep -A 3 "ENVIRONMENT VARIABLES"

     

       113
       113
       -
         ENVIRONMENT VARIABLES

     

       114
       114
       -
                Toru respects the XDG Base Directory Specification:

     

       115
       115
       -
         

     

       116
       116
       -
                XDG_CACHE_HOME - Override default cache directory location

     

       117
       117
       -
       

     

       118
       118
       -
       Test 6: Error handling

     

       119
       119
       -
       ======================

     

       120
       120
       -
       

     

       121
       121
       -
       Test non-existent file:

     

       122
       122
       -
         $ dune exec bin/toru_main.exe -- inspect nonexistent.txt 2>&1

     

       123
       123
       -
         [1]

     

       124
       124
       -
       

     

       125
       125
       -
       Test malformed file doesn't crash:

     

       126
       126
       -
         $ echo "not a registry" > /tmp/bad.txt

     

       127
       127
       -
         $ dune exec bin/toru_main.exe -- inspect /tmp/bad.txt

     

       128
       128
       -
         Registry contains 0 entries

     

       129
       129
       -
       

     

       130
       130
       -
       Test 7: App name and version parameters

     

       131
       131
       -
       ======================================

     

       132
       132
       -
       

     

       133
       133
       -
       Test with custom app name:

     

       134
       134
       -
         $ dune exec bin/toru_main.exe -- --app-name testapp inspect test/python/test_registry_sha256.txt

     

       135
       135
       -
         Registry contains 8 entries

     

       136
       136
       -
       

     

       137
       137
       -
       Test with data version:

     

       138
       138
       -
         $ dune exec bin/toru_main.exe -- --data-version v1.0 inspect test/python/test_registry_sha256.txt

     

       139
       139
       -
         Registry contains 8 entries

     

       140
       140
       -
       

     

       141
       141
       -
       Cleanup:

     

       142
       142
       -
         $ rm -f /tmp/converted.txt /tmp/bad.txt

     

       143
       143
       -
         $ rm -rf /tmp/toru-test-cache /tmp/xdg-test /tmp/app-test

-354

stack/toru/test/cli.t

···

       1
       1
       -
       Toru CLI Comprehensive Test Suite

     

       2
       2
       -
       ==================================

     

       3
       3
       -
       

     

       4
       4
       -
       Setup test environment:

     

       5
       5
       -
         $ export HOME=/tmp/toru_test_home

     

       6
       6
       -
         $ mkdir -p $HOME

     

       7
       7
       -
         $ unset XDG_CACHE_HOME TORU_CACHE_DIR

     

       8
       8
       -
         $ cd /tmp/toru_test_home

     

       9
       9
       -
       

     

       10
       10
       -
       Create test registry file:

     

       11
       11
       -
         $ cat > test_registry.txt << 'EOF'

     

       12
       12
       -
         > # Test registry for toru CLI

     

       13
       13
       -
         > simple.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       14
       14
       -
         > data.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef

     

       15
       15
       -
         > binary.bin 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9

     

       16
       16
       -
         > EOF

     

       17
       17
       -
       

     

       18
       18
       -
       Test 1: Basic inspect command functionality

     

       19
       19
       -
       ===========================================

     

       20
       20
       -
       

     

       21
       21
       -
       Test basic inspect without options:

     

       22
       22
       -
         $ toru inspect test_registry.txt

     

       23
       23
       -
         Registry contains 3 entries

     

       24
       24
       -
       

     

       25
       25
       -
       Test inspect with stats flag:

     

       26
       26
       -
         $ toru inspect --stats test_registry.txt

     

       27
       27
       -
         Registry contains 3 entries

     

       28
       28
       -
         

     

       29
       29
       -
         Registry Statistics:

     

       30
       30
       -
         ===================

     

       31
       31
       -
         Hash algorithms:

     

       32
       32
       -
           sha256: 3 entries

     

       33
       33
       -
         

     

       34
       34
       -
         File extensions:

     

       35
       35
       -
           .txt: 1 files

     

       36
       36
       -
           .csv: 1 files

     

       37
       37
       -
           .bin: 1 files

     

       38
       38
       -
       

     

       39
       39
       -
       Test inspect with list flag:

     

       40
       40
       -
         $ toru inspect --list test_registry.txt

     

       41
       41
       -
         Registry contains 3 entries

     

       42
       42
       -
         

     

       43
       43
       -
         Files in registry:

     

       44
       44
       -
         ==================

     

       45
       45
       -
         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  simple.txt

     

       46
       46
       -
         d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef  data.csv

     

       47
       47
       -
         785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9  binary.bin

     

       48
       48
       -
       

     

       49
       49
       -
       Test inspect with search pattern:

     

       50
       50
       -
         $ toru inspect --search txt test_registry.txt

     

       51
       51
       -
         Registry contains 3 entries

     

       52
       52
       -
         

     

       53
       53
       -
         Files matching 'txt':

     

       54
       54
       -
         =======================

     

       55
       55
       -
         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  simple.txt

     

       56
       56
       -
       

     

       57
       57
       -
       Test inspect with all flags combined:

     

       58
       58
       -
         $ toru inspect --stats --list --search csv test_registry.txt

     

       59
       59
       -
         Registry contains 3 entries

     

       60
       60
       -
         

     

       61
       61
       -
         Registry Statistics:

     

       62
       62
       -
         ===================

     

       63
       63
       -
         Hash algorithms:

     

       64
       64
       -
           sha256: 3 entries

     

       65
       65
       -
         

     

       66
       66
       -
         File extensions:

     

       67
       67
       -
           .txt: 1 files

     

       68
       68
       -
           .csv: 1 files

     

       69
       69
       -
           .bin: 1 files

     

       70
       70
       -
         

     

       71
       71
       -
         Files in registry:

     

       72
       72
       -
         ==================

     

       73
       73
       -
         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  simple.txt

     

       74
       74
       -
         d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef  data.csv

     

       75
       75
       -
         785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9  binary.bin

     

       76
       76
       -
         

     

       77
       77
       -
         Files matching 'csv':

     

       78
       78
       -
         =======================

     

       79
       79
       -
         d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef  data.csv

     

       80
       80
       -
       

     

       81
       81
       -
       Test 2: Validate command functionality

     

       82
       82
       -
       ======================================

     

       83
       83
       -
       

     

       84
       84
       -
       Test basic validate:

     

       85
       85
       -
         $ toru validate test_registry.txt

     

       86
       86
       -
         ✓ Registry loaded successfully with 3 entries

     

       87
       87
       -
       

     

       88
       88
       -
       Test validate with hash checking:

     

       89
       89
       -
         $ toru validate --check-hashes test_registry.txt

     

       90
       90
       -
         ✓ Registry loaded successfully with 3 entries

     

       91
       91
       -
         Validating hash formats...

     

       92
       92
       -
         ✓ All 3 hash formats are valid

     

       93
       93
       -
       

     

       94
       94
       -
       Test 3: Convert command functionality

     

       95
       95
       -
       ====================================

     

       96
       96
       -
       

     

       97
       97
       -
       Test convert to same format:

     

       98
       98
       -
         $ toru convert test_registry.txt converted_registry.txt

     

       99
       99
       -
         ✓ Converted 3 entries from test_registry.txt to converted_registry.txt

     

       100
       100
       -
         

     

       101
       101
       -
         $ cat converted_registry.txt

     

       102
       102
       -
         # Pooch registry generated on *-*-*T*:*:*.* (glob)

     

       103
       103
       -
         # Algorithm: sha256

     

       104
       104
       -
         simple.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       105
       105
       -
         data.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef

     

       106
       106
       -
         binary.bin 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9

     

       107
       107
       -
       

     

       108
       108
       -
       Test 4: Verbose logging levels

     

       109
       109
       -
       ==============================

     

       110
       110
       -
       

     

       111
       111
       -
       Test with single verbose flag (info level):

     

       112
       112
       -
         $ toru inspect test_registry.txt --verbose

     

       113
       113
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       114
       114
       -
         Registry contains 3 entries

     

       115
       115
       -
       

     

       116
       116
       -
       Test with double verbose flags (debug level):

     

       117
       117
       -
         $ toru inspect test_registry.txt --verbose --verbose

     

       118
       118
       -
         toru_main.exe: [DEBUG] Starting inspect function

     

       119
       119
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       120
       120
       -
         toru_main.exe: [DEBUG] Determining if source is URL or file path

     

       121
       121
       -
         toru_main.exe: [DEBUG] Detected file path, loading directly without Eio

     

       122
       122
       -
         toru_main.exe: [DEBUG] Registry loaded successfully

     

       123
       123
       -
         Registry contains 3 entries

     

       124
       124
       -
       

     

       125
       125
       -
       Test 5: XDG Base Directory compliance

     

       126
       126
       -
       ====================================

     

       127
       127
       -
       

     

       128
       128
       -
       Test default cache directory behavior:

     

       129
       129
       -
         $ unset XDG_CACHE_HOME TORU_CACHE_DIR

     

       130
       130
       -
         $ toru inspect test_registry.txt --verbose

     

       131
       131
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       132
       132
       -
         Registry contains 3 entries

     

       133
       133
       -
       

     

       134
       134
       -
       Test XDG_CACHE_HOME environment variable:

     

       135
       135
       -
         $ XDG_CACHE_HOME=/tmp/xdg-cache toru inspect test_registry.txt --verbose

     

       136
       136
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       137
       137
       -
         Registry contains 3 entries

     

       138
       138
       -
       

     

       139
       139
       -
       Test application-specific cache directory override:

     

       140
       140
       -
         $ TORU_CACHE_DIR=/tmp/toru-cache toru inspect test_registry.txt --verbose

     

       141
       141
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       142
       142
       -
         Registry contains 3 entries

     

       143
       143
       -
       

     

       144
       144
       -
       Test command-line cache directory override:

     

       145
       145
       -
         $ toru inspect test_registry.txt --cache-dir /tmp/cli-cache --verbose

     

       146
       146
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       147
       147
       -
         Registry contains 3 entries

     

       148
       148
       -
       

     

       149
       149
       -
       Test precedence: CLI overrides environment variables:

     

       150
       150
       -
         $ XDG_CACHE_HOME=/tmp/xdg TORU_CACHE_DIR=/tmp/app toru inspect test_registry.txt --cache-dir /tmp/cli --verbose

     

       151
       151
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       152
       152
       -
         Registry contains 3 entries

     

       153
       153
       -
       

     

       154
       154
       -
       Test 6: Mixed hash algorithms support

     

       155
       155
       -
       ====================================

     

       156
       156
       -
       

     

       157
       157
       -
       Create registry with mixed hash types:

     

       158
       158
       -
         $ cat > mixed_hashes.txt << 'EOF'

     

       159
       159
       -
         > # Registry with different hash algorithms

     

       160
       160
       -
         > # SHA256 hash

     

       161
       161
       -
         > file1.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       162
       162
       -
         > # SHA1 hash  

     

       163
       163
       -
         > file2.txt sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709

     

       164
       164
       -
         > # MD5 hash

     

       165
       165
       -
         > file3.txt md5:d41d8cd98f00b204e9800998ecf8427e

     

       166
       166
       -
         > EOF

     

       167
       167
       -
       

     

       168
       168
       -
       Test mixed hash validation:

     

       169
       169
       -
         $ toru validate --check-hashes mixed_hashes.txt

     

       170
       170
       -
         ✓ Registry loaded successfully with 3 entries

     

       171
       171
       -
         Validating hash formats...

     

       172
       172
       -
         ✓ All 3 hash formats are valid

     

       173
       173
       -
       

     

       174
       174
       -
       Test 7: URL source handling

     

       175
       175
       -
       ===========================

     

       176
       176
       -
       

     

       177
       177
       -
       Test detection of HTTP URL (mock):

     

       178
       178
       -
         $ toru inspect https://example.com/registry.txt 2>&1 | head -3

     

       179
       179
       -
         toru_main.exe: [ERROR] * (glob)

     

       180
       180
       -
       

     

       181
       181
       -
       Test detection of HTTPS URL (mock):

     

       182
       182
       -
         $ toru inspect https://example.com/registry.txt 2>&1 | head -3

     

       183
       183
       -
         toru_main.exe: [ERROR] * (glob)

     

       184
       184
       -
       

     

       185
       185
       -
       Test 8: Error handling

     

       186
       186
       -
       ======================

     

       187
       187
       -
       

     

       188
       188
       -
       Test non-existent file:

     

       189
       189
       -
         $ toru inspect nonexistent.txt

     

       190
       190
       -
         [1]

     

       191
       191
       -
       

     

       192
       192
       -
       Test malformed registry:

     

       193
       193
       -
         $ echo "invalid registry content" > malformed.txt

     

       194
       194
       -
         $ toru inspect malformed.txt

     

       195
       195
       -
         Registry contains 0 entries

     

       196
       196
       -
       

     

       197
       197
       -
       Test empty registry:

     

       198
       198
       -
         $ touch empty.txt

     

       199
       199
       -
         $ toru inspect empty.txt

     

       200
       200
       -
         Registry contains 0 entries

     

       201
       201
       -
       

     

       202
       202
       -
       Test 9: App name parameter

     

       203
       203
       -
       =========================

     

       204
       204
       -
       

     

       205
       205
       -
       Test custom app name:

     

       206
       206
       -
         $ toru --app-name myapp inspect test_registry.txt --verbose

     

       207
       207
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       208
       208
       -
         Registry contains 3 entries

     

       209
       209
       -
       

     

       210
       210
       -
       Test 10: Data version parameter

     

       211
       211
       -
       ==============================

     

       212
       212
       -
       

     

       213
       213
       -
       Test with version string:

     

       214
       214
       -
         $ toru --data-version v1.0 inspect test_registry.txt --verbose

     

       215
       215
       -
         toru_main.exe: [INFO] Loading registry from: test_registry.txt

     

       216
       216
       -
         Registry contains 3 entries

     

       217
       217
       -
       

     

       218
       218
       -
       Test 11: Help system

     

       219
       219
       -
       ===================

     

       220
       220
       -
       

     

       221
       221
       -
       Test main help:

     

       222
       222
       -
         $ toru --help | head -10

     

       223
       223
       -
         TORU(1) * Toru Manual * TORU(1) (glob)

     

       224
       224
       -
         

     

       225
       225
       -
         

     

       226
       226
       -
         

     

       227
       227
       -
         NAME

     

       228
       228
       -
                toru - Toru data repository management tool

     

       229
       229
       -
         

     

       230
       230
       -
         SYNOPSIS

     

       231
       231
       -
                toru [COMMAND]…

     

       232
       232
       -
         

     

       233
       233
       -
       

     

       234
       234
       -
       Test inspect subcommand help:

     

       235
       235
       -
         $ toru inspect --help | head -10

     

       236
       236
       -
         TORU-INSPECT(1) * Toru Manual * TORU-INSPECT(1) (glob)

     

       237
       237
       -
         

     

       238
       238
       -
         

     

       239
       239
       -
         

     

       240
       240
       -
         NAME

     

       241
       241
       -
                toru-inspect - Inspect a registry file or URL

     

       242
       242
       -
         

     

       243
       243
       -
         SYNOPSIS

     

       244
       244
       -
                toru-inspect [OPTION]… REGISTRY

     

       245
       245
       -
         

     

       246
       246
       -
       

     

       247
       247
       -
       Test validate subcommand help:

     

       248
       248
       -
         $ toru validate --help | head -10

     

       249
       249
       -
         TORU-VALIDATE(1) * Toru Manual * TORU-VALIDATE(1) (glob)

     

       250
       250
       -
         

     

       251
       251
       -
         

     

       252
       252
       -
         

     

       253
       253
       -
         NAME

     

       254
       254
       -
                toru-validate - Validate a registry file format and integrity

     

       255
       255
       -
         

     

       256
       256
       -
         SYNOPSIS

     

       257
       257
       -
                toru-validate [OPTION]… REGISTRY

     

       258
       258
       -
         

     

       259
       259
       -
       

     

       260
       260
       -
       Test convert subcommand help:

     

       261
       261
       -
         $ toru convert --help | head -10

     

       262
       262
       -
         TORU-CONVERT(1) * Toru Manual * TORU-CONVERT(1) (glob)

     

       263
       263
       -
         

     

       264
       264
       -
         

     

       265
       265
       -
         

     

       266
       266
       -
         NAME

     

       267
       267
       -
                toru-convert - Convert registry between different formats or sources

     

       268
       268
       -
         

     

       269
       269
       -
         SYNOPSIS

     

       270
       270
       -
                toru-convert [OPTION]… INPUT OUTPUT

     

       271
       271
       -
         

     

       272
       272
       -
       

     

       273
       273
       -
       Test 12: Environment variable documentation

     

       274
       274
       -
       ==========================================

     

       275
       275
       -
       

     

       276
       276
       -
       Test that help includes XDG information:

     

       277
       277
       -
         $ toru --help | grep -A 5 "ENVIRONMENT VARIABLES"

     

       278
       278
       -
         ENVIRONMENT VARIABLES

     

       279
       279
       -
                Toru respects the XDG Base Directory Specification:

     

       280
       280
       -
         

     

       281
       281
       -
                XDG_CACHE_HOME - Override default cache directory location

     

       282
       282
       -
         

     

       283
       283
       -
                TORU_CACHE_DIR - Application-specific cache directory override

     

       284
       284
       -
       

     

       285
       285
       -
       Test 13: Edge cases and boundary conditions

     

       286
       286
       -
       ==========================================

     

       287
       287
       -
       

     

       288
       288
       -
       Test very long registry file:

     

       289
       289
       -
         $ python3 -c "

     

       290
       290
       -
         > for i in range(1000):

     

       291
       291
       -
         >     print(f'file{i:04d}.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')

     

       292
       292
       -
         > " > large_registry.txt

     

       293
       293
       -
         $ toru inspect large_registry.txt

     

       294
       294
       -
         Registry contains 1000 entries

     

       295
       295
       -
       

     

       296
       296
       -
       Test registry with comments and blank lines:

     

       297
       297
       -
         $ cat > commented_registry.txt << 'EOF'

     

       298
       298
       -
         > # This is a test registry

     

       299
       299
       -
         > # With comments and blank lines

     

       300
       300
       -
         > 

     

       301
       301
       -
         > simple.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       302
       302
       -
         > 

     

       303
       303
       -
         > # Another comment

     

       304
       304
       -
         > data.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef

     

       305
       305
       -
         > 

     

       306
       306
       -
         > EOF

     

       307
       307
       -
       

     

       308
       308
       -
         $ toru inspect commented_registry.txt

     

       309
       309
       -
         Registry contains 2 entries

     

       310
       310
       -
       

     

       311
       311
       -
       Test registry with unusual file names:

     

       312
       312
       -
         $ cat > unusual_names.txt << 'EOF'

     

       313
       313
       -
         > file with spaces.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       314
       314
       -
         > file-with-dashes.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       315
       315
       -
         > file_with_underscores.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       316
       316
       -
         > file.with.dots.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       317
       317
       -
         > file123numbers.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       318
       318
       -
         > EOF

     

       319
       319
       -
       

     

       320
       320
       -
         $ toru inspect unusual_names.txt

     

       321
       321
       -
         Registry contains 5 entries

     

       322
       322
       -
       

     

       323
       323
       -
       Test 14: Performance and scalability

     

       324
       324
       -
       ===================================

     

       325
       325
       -
       

     

       326
       326
       -
       Test registry with many file extensions:

     

       327
       327
       -
         $ python3 -c "

     

       328
       328
       -
         > import string

     

       329
       329
       -
         > extensions = [f'.{c}{c}{c}' for c in string.ascii_lowercase[:10]]

     

       330
       330
       -
         > for i, ext in enumerate(extensions):

     

       331
       331
       -
         >     print(f'file{i:03d}{ext} e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')

     

       332
       332
       -
         > " > many_extensions.txt

     

       333
       333
       -
         $ toru inspect --stats many_extensions.txt

     

       334
       334
       -
         Registry contains 10 entries

     

       335
       335
       -
         

     

       336
       336
       -
         Registry Statistics:

     

       337
       337
       -
         ===================

     

       338
       338
       -
         Hash algorithms:

     

       339
       339
       -
           sha256: 10 entries

     

       340
       340
       -
         

     

       341
       341
       -
         File extensions:

     

       342
       342
       -
           .aaa: 1 files

     

       343
       343
       -
           .bbb: 1 files

     

       344
       344
       -
           .ccc: 1 files

     

       345
       345
       -
           .ddd: 1 files

     

       346
       346
       -
           .eee: 1 files

     

       347
       347
       -
           .fff: 1 files

     

       348
       348
       -
           .ggg: 1 files

     

       349
       349
       -
           .hhh: 1 files

     

       350
       350
       -
           .iii: 1 files

     

       351
       351
       -
           .jjj: 1 files

     

       352
       352
       -
       

     

       353
       353
       -
       Cleanup:

     

       354
       354
       -
         $ rm -rf /tmp/toru_test_home

-93

stack/toru/test/downloader_demo.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let demo_downloader_usage () =

     

       4
       4
       -
         Printf.printf "Toru Downloader System Demo\n";

     

       5
       5
       -
         Printf.printf "===========================\n\n";

     

       6
       6
       -
         

     

       7
       7
       -
         Eio_main.run @@ fun env ->

     

       8
       8
       -
         Eio.Switch.run @@ fun sw ->

     

       9
       9
       -
           

     

       10
       10
       -
           (* 1. Show available downloaders *)

     

       11
       11
       -
           Printf.printf "1. Detecting available downloaders:\n";

     

       12
       12
       -
           let available = Downloader.Downloaders.detect_available ~env in

     

       13
       13
       -
           List.iter (fun (name, _) ->

     

       14
       14
       -
             Printf.printf "   ✓ %s is available\n" name

     

       15
       15
       -
           ) available;

     

       16
       16
       -
           Printf.printf "\n";

     

       17
       17
       -
           

     

       18
       18
       -
           (* 2. Create downloader with authentication *)

     

       19
       19
       -
           Printf.printf "2. Creating curl downloader with authentication:\n";

     

       20
       20
       -
           let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.curl () in

     

       21
       21
       -
           let auth = Downloader.Config.{ username = Some "demo_user"; password = Some "demo_pass" } in

     

       22
       22
       -
           let authenticated_downloader = D.create ~sw ~env ~auth () in

     

       23
       23
       -
           Printf.printf "   ✓ Created authenticated %s downloader\n" (D.name authenticated_downloader);

     

       24
       24
       -
           Printf.printf "   ✓ Resume support: %b\n\n" (D.supports_resume authenticated_downloader);

     

       25
       25
       -
           

     

       26
       26
       -
           (* 3. Create regular downloader *)

     

       27
       27
       -
           Printf.printf "3. Creating regular downloader:\n";

     

       28
       28
       -
           let regular_downloader = D.create ~sw ~env () in

     

       29
       29
       -
           Printf.printf "   ✓ Created regular %s downloader\n\n" (D.name regular_downloader);

     

       30
       30
       -
           

     

       31
       31
       -
           (* 4. Download a test file *)

     

       32
       32
       -
           Printf.printf "4. Downloading test file with hash verification:\n";

     

       33
       33
       -
           let test_url = "https://httpbin.org/robots.txt" in

     

       34
       34
       -
           let dest_path = Eio.Path.(env#fs / "demo_download.txt") in

     

       35
       35
       -
           

     

       36
       36
       -
           (* First download to compute hash *)

     

       37
       37
       -
           (match D.download regular_downloader ~url:test_url ~dest:dest_path () with

     

       38
       38
       -
           | Ok () ->

     

       39
       39
       -
               let computed_hash = Hash.compute Hash.SHA256 dest_path in

     

       40
       40
       -
               Printf.printf "   ✓ Downloaded file successfully\n";

     

       41
       41
       -
               Printf.printf "   ✓ Computed SHA256: %s\n" (Hash.value computed_hash);

     

       42
       42
       -
               

     

       43
       43
       -
               (* Clean up and re-download with hash verification *)

     

       44
       44
       -
               Eio.Path.unlink dest_path;

     

       45
       45
       -
               Printf.printf "   ✓ Testing hash verification...\n";

     

       46
       46
       -
               

     

       47
       47
       -
               (match D.download regular_downloader ~url:test_url ~dest:dest_path ~hash:computed_hash () with

     

       48
       48
       -
               | Ok () ->

     

       49
       49
       -
                   Printf.printf "   ✓ Hash verification successful!\n";

     

       50
       50
       -
                   Eio.Path.unlink dest_path;

     

       51
       51
       -
               | Error msg ->

     

       52
       52
       -
                   Printf.printf "   ✗ Hash verification failed: %s\n" msg);

     

       53
       53
       -
                   

     

       54
       54
       -
           | Error msg ->

     

       55
       55
       -
               Printf.printf "   ✗ Download failed: %s\n" msg);

     

       56
       56
       -
           Printf.printf "\n";

     

       57
       57
       -
           

     

       58
       58
       -
           (* 5. Test resume functionality *)

     

       59
       59
       -
           Printf.printf "5. Testing resume functionality:\n";

     

       60
       60
       -
           (match D.download regular_downloader ~url:test_url ~dest:dest_path ~resume:true () with

     

       61
       61
       -
           | Ok () ->

     

       62
       62
       -
               Printf.printf "   ✓ Resume download successful\n";

     

       63
       63
       -
               Eio.Path.unlink dest_path;

     

       64
       64
       -
           | Error msg ->

     

       65
       65
       -
               Printf.printf "   ✗ Resume download failed: %s\n" msg);

     

       66
       66
       -
           Printf.printf "\n";

     

       67
       67
       -
           

     

       68
       68
       -
           (* 6. Test CLI selection *)

     

       69
       69
       -
           Printf.printf "6. Testing CLI downloader selection:\n";

     

       70
       70
       -
           let (module CLI_D) = Downloader.Cli.create_downloader ~env `Auto in

     

       71
       71
       -
           Printf.printf "   ✓ CLI auto-selected downloader available\n";

     

       72
       72
       -
           Printf.printf "\n";

     

       73
       73
       -
           

     

       74
       74
       -
           (* 7. Test string-based selection *)

     

       75
       75
       -
           Printf.printf "7. Testing string-based downloader selection:\n";

     

       76
       76
       -
           (match Downloader.Downloaders.of_string "curl" with

     

       77
       77
       -
           | Some (module Selected) ->

     

       78
       78
       -
               Printf.printf "   ✓ Successfully selected curl via string\n"

     

       79
       79
       -
           | None ->

     

       80
       80
       -
               Printf.printf "   ✗ Failed to select curl via string\n");

     

       81
       81
       -
           Printf.printf "\n";

     

       82
       82
       -
           

     

       83
       83
       -
           Printf.printf "Demo completed successfully!\n";

     

       84
       84
       -
           Printf.printf "\nKey features demonstrated:\n";

     

       85
       85
       -
           Printf.printf "• Automatic downloader detection\n";

     

       86
       86
       -
           Printf.printf "• Authentication support (username/password)\n";

     

       87
       87
       -
           Printf.printf "• Hash verification (SHA256)\n";

     

       88
       88
       -
           Printf.printf "• Resume functionality\n";

     

       89
       89
       -
           Printf.printf "• Error handling\n";

     

       90
       90
       -
           Printf.printf "• CLI integration\n";

     

       91
       91
       -
           Printf.printf "• Multiple selection methods\n"

     

       92
       92
       -
       

     

       93
       93
       -
       let () = demo_downloader_usage ()

-84

stack/toru/test/dune

···

       1
       1
       -
       (executable

     

       2
       2
       -
        (public_name test_toru)

     

       3
       3
       -
        (name test_toru)

     

       4
       4
       -
        (libraries toru eio eio_main))

     

       5
       5
       -
       

     

       6
       6
       -
       (executable

     

       7
       7
       -
        (public_name test_registry)

     

       8
       8
       -
        (name test_registry)

     

       9
       9
       -
        (libraries toru eio eio_main))

     

       10
       10
       -
       

     

       11
       11
       -
       (executable

     

       12
       12
       -
        (public_name test_registry_real)

     

       13
       13
       -
        (name test_registry_real)

     

       14
       14
       -
        (libraries toru eio eio_main))

     

       15
       15
       -
       

     

       16
       16
       -
       (executable

     

       17
       17
       -
        (public_name test_tessera_load)

     

       18
       18
       -
        (name test_tessera_load) 

     

       19
       19
       -
        (libraries toru eio eio_main))

     

       20
       20
       -
       

     

       21
       21
       -
       (executable

     

       22
       22
       -
        (public_name test_cache)

     

       23
       23
       -
        (name test_cache) 

     

       24
       24
       -
        (libraries toru eio eio_main))

     

       25
       25
       -
       

     

       26
       26
       -
       (executable

     

       27
       27
       -
        (public_name test_downloader)

     

       28
       28
       -
        (name test_downloader) 

     

       29
       29
       -
        (libraries toru eio eio_main))

     

       30
       30
       -
       

     

       31
       31
       -
       (executable

     

       32
       32
       -
        (public_name test_curl_download)

     

       33
       33
       -
        (name test_curl_download) 

     

       34
       34
       -
        (libraries toru eio eio_main))

     

       35
       35
       -
       

     

       36
       36
       -
       (executable

     

       37
       37
       -
        (public_name test_hash)

     

       38
       38
       -
        (name test_hash) 

     

       39
       39
       -
        (libraries toru eio eio_main))

     

       40
       40
       -
       

     

       41
       41
       -
       (executable

     

       42
       42
       -
        (public_name test_python_cross_validation)

     

       43
       43
       -
        (name test_python_cross_validation) 

     

       44
       44
       -
        (libraries toru eio eio_main yojson))

     

       45
       45
       -
       

     

       46
       46
       -
       (executable

     

       47
       47
       -
        (public_name test_cache_xdg)

     

       48
       48
       -
        (name test_cache_xdg) 

     

       49
       49
       -
        (libraries toru eio eio_main unix))

     

       50
       50
       -
       

     

       51
       51
       -
       (executable

     

       52
       52
       -
        (public_name test_xdg_integration)

     

       53
       53
       -
        (name test_xdg_integration) 

     

       54
       54
       -
        (libraries toru eio eio_main xdg))

     

       55
       55
       -
       

     

       56
       56
       -
       (executable

     

       57
       57
       -
        (public_name test_downloader_comprehensive)

     

       58
       58
       -
        (name test_downloader_comprehensive) 

     

       59
       59
       -
        (libraries toru eio eio_main))

     

       60
       60
       -
       

     

       61
       61
       -
       (executable

     

       62
       62
       -
        (public_name downloader_demo)

     

       63
       63
       -
        (name downloader_demo) 

     

       64
       64
       -
        (libraries toru eio eio_main))

     

       65
       65
       -
       

     

       66
       66
       -
       (executable

     

       67
       67
       -
        (public_name test_tessera_integration)

     

       68
       68
       -
        (name test_tessera_integration)

     

       69
       69
       -
        (libraries toru eio eio_main))

     

       70
       70
       -
       

     

       71
       71
       -
       (executable

     

       72
       72
       -
        (public_name test_streaming_hash)

     

       73
       73
       -
        (name test_streaming_hash)

     

       74
       74
       -
        (libraries toru eio eio_main))

     

       75
       75
       -
       

     

       76
       76
       -
       (executable

     

       77
       77
       -
        (public_name test_simple_stream)

     

       78
       78
       -
        (name test_simple_stream)

     

       79
       79
       -
        (libraries toru eio eio_main))

     

       80
       80
       -
       

     

       81
       81
       -
       (executable

     

       82
       82
       -
        (public_name test_chunk_operations)

     

       83
       83
       -
        (name test_chunk_operations)

     

       84
       84
       -
        (libraries cacheio eio eio_main))

-207

stack/toru/test/python/generate_pooch_registry.py

···

       1
       1
       -
       #!/usr/bin/env -S uv run

     

       2
       2
       -
       # /// script

     

       3
       3
       -
       # requires-python = ">=3.8"

     

       4
       4
       -
       # dependencies = [

     

       5
       5
       -
       #   "pooch>=1.7.0",

     

       6
       6
       -
       # ]

     

       7
       7
       -
       # ///

     

       8
       8
       -
       

     

       9
       9
       -
       """

     

       10
       10
       -
       Generate a Pooch registry file for testing Toru OCaml implementation.

     

       11
       11
       -
       This creates sample data files and a registry with various hash formats.

     

       12
       12
       -
       

     

       13
       13
       -
       Usage:

     

       14
       14
       -
           uv run generate_pooch_registry.py

     

       15
       15
       -
       """

     

       16
       16
       -
       

     

       17
       17
       -
       import os

     

       18
       18
       -
       import hashlib

     

       19
       19
       -
       import tempfile

     

       20
       20
       -
       from pathlib import Path

     

       21
       21
       -
       import pooch

     

       22
       22
       -
       import json

     

       23
       23
       -
       from datetime import datetime

     

       24
       24
       -
       

     

       25
       25
       -
       def create_sample_files(base_dir):

     

       26
       26
       -
           """Create sample files with known content for testing."""

     

       27
       27
       -
           files = {

     

       28
       28
       -
               "data/simple.txt": b"Hello, World!",

     

       29
       29
       -
               "data/numbers.csv": b"1,2,3\n4,5,6\n7,8,9\n",

     

       30
       30
       -
               "data/unicode.txt": "Hello, 世界! 🌍\n".encode('utf-8'),

     

       31
       31
       -
               "models/small.bin": bytes([i % 256 for i in range(1024)]),  # 1KB binary

     

       32
       32
       -
               "docs/readme.md": b"# Sample Dataset\n\nThis is a test dataset for Toru.",

     

       33
       33
       -
               "config.json": json.dumps({"version": "1.0", "test": True}).encode('utf-8'),

     

       34
       34
       -
               ".hidden/secret.txt": b"Hidden file content",

     

       35
       35
       -
               "empty.txt": b"",  # Empty file

     

       36
       36
       -
           }

     

       37
       37
       -
           

     

       38
       38
       -
           created_files = {}

     

       39
       39
       -
           for rel_path, content in files.items():

     

       40
       40
       -
               file_path = base_dir / rel_path

     

       41
       41
       -
               file_path.parent.mkdir(parents=True, exist_ok=True)

     

       42
       42
       -
               file_path.write_bytes(content)

     

       43
       43
       -
               created_files[rel_path] = {

     

       44
       44
       -
                   'path': file_path,

     

       45
       45
       -
                   'content': content,

     

       46
       46
       -
                   'size': len(content)

     

       47
       47
       -
               }

     

       48
       48
       -
           

     

       49
       49
       -
           return created_files

     

       50
       50
       -
       

     

       51
       51
       -
       def calculate_hashes(file_info):

     

       52
       52
       -
           """Calculate SHA256, SHA1, and MD5 hashes for files."""

     

       53
       53
       -
           hashes = {}

     

       54
       54
       -
           for rel_path, info in file_info.items():

     

       55
       55
       -
               content = info['content']

     

       56
       56
       -
               hashes[rel_path] = {

     

       57
       57
       -
                   'sha256': hashlib.sha256(content).hexdigest(),

     

       58
       58
       -
                   'sha1': hashlib.sha1(content).hexdigest(),

     

       59
       59
       -
                   'md5': hashlib.md5(content).hexdigest(),

     

       60
       60
       -
                   'size': info['size']

     

       61
       61
       -
               }

     

       62
       62
       -
           return hashes

     

       63
       63
       -
       

     

       64
       64
       -
       def generate_pooch_registry(file_hashes, output_path, hash_algo='sha256'):

     

       65
       65
       -
           """Generate a Pooch registry file."""

     

       66
       66
       -
           registry = {}

     

       67
       67
       -
           

     

       68
       68
       -
           # Write registry in Pooch format

     

       69
       69
       -
           with open(output_path, 'w') as f:

     

       70
       70
       -
               f.write(f"# Pooch registry generated on {datetime.now().isoformat()}\n")

     

       71
       71
       -
               f.write(f"# Algorithm: {hash_algo}\n")

     

       72
       72
       -
               for filename, hashes in sorted(file_hashes.items()):

     

       73
       73
       -
                   file_hash = hashes[hash_algo]

     

       74
       74
       -
                   registry[filename] = file_hash

     

       75
       75
       -
                   # Pooch includes algorithm prefix for non-SHA256

     

       76
       76
       -
                   if hash_algo != 'sha256':

     

       77
       77
       -
                       f.write(f"{filename} {hash_algo}:{file_hash}\n")

     

       78
       78
       -
                   else:

     

       79
       79
       -
                       f.write(f"{filename} {file_hash}\n")

     

       80
       80
       -
           

     

       81
       81
       -
           return registry

     

       82
       82
       -
       

     

       83
       83
       -
       def generate_mixed_registry(file_hashes, output_path):

     

       84
       84
       -
           """Generate a registry with mixed hash formats for testing."""

     

       85
       85
       -
           with open(output_path, 'w') as f:

     

       86
       86
       -
               f.write("# Mixed hash format registry for testing\n")

     

       87
       87
       -
               f.write("# This tests various hash algorithm prefixes\n\n")

     

       88
       88
       -
               

     

       89
       89
       -
               # Mix different hash formats

     

       90
       90
       -
               for i, (rel_path, hashes) in enumerate(sorted(file_hashes.items())):

     

       91
       91
       -
                   if i % 3 == 0:

     

       92
       92
       -
                       # SHA256 without prefix (default)

     

       93
       93
       -
                       f.write(f"{rel_path} {hashes['sha256']}\n")

     

       94
       94
       -
                   elif i % 3 == 1:

     

       95
       95
       -
                       # SHA1 with prefix

     

       96
       96
       -
                       f.write(f"{rel_path} sha1:{hashes['sha1']}\n")

     

       97
       97
       -
                   else:

     

       98
       98
       -
                       # MD5 with prefix

     

       99
       99
       -
                       f.write(f"{rel_path} md5:{hashes['md5']}\n")

     

       100
       100
       -
                   

     

       101
       101
       -
                   # Add comments between some entries

     

       102
       102
       -
                   if i == 2:

     

       103
       103
       -
                       f.write("\n# Additional files\n")

     

       104
       104
       -
       

     

       105
       105
       -
       def generate_json_metadata(file_hashes, output_path):

     

       106
       106
       -
           """Generate JSON metadata file with all hash algorithms."""

     

       107
       107
       -
           metadata = {

     

       108
       108
       -
               "generated": datetime.now().isoformat(),

     

       109
       109
       -
               "generator": "pooch_test_generator.py",

     

       110
       110
       -
               "files": []

     

       111
       111
       -
           }

     

       112
       112
       -
           

     

       113
       113
       -
           for rel_path, hashes in sorted(file_hashes.items()):

     

       114
       114
       -
               metadata["files"].append({

     

       115
       115
       -
                   "path": rel_path,

     

       116
       116
       -
                   "size": hashes['size'],

     

       117
       117
       -
                   "sha256": hashes['sha256'],

     

       118
       118
       -
                   "sha1": hashes['sha1'],

     

       119
       119
       -
                   "md5": hashes['md5']

     

       120
       120
       -
               })

     

       121
       121
       -
           

     

       122
       122
       -
           with open(output_path, 'w') as f:

     

       123
       123
       -
               json.dump(metadata, f, indent=2)

     

       124
       124
       -
       

     

       125
       125
       -
       def test_pooch_loading(registry_path, base_url="https://example.com/data/"):

     

       126
       126
       -
           """Test that Pooch can load the generated registry."""

     

       127
       127
       -
           # Create a Pooch instance

     

       128
       128
       -
           pup = pooch.create(

     

       129
       129
       -
               path=pooch.os_cache("toru_test"),

     

       130
       130
       -
               base_url=base_url,

     

       131
       131
       -
               registry=None,

     

       132
       132
       -
               version="v1.0"

     

       133
       133
       -
           )

     

       134
       134
       -
           

     

       135
       135
       -
           # Load the registry

     

       136
       136
       -
           pup.load_registry(registry_path)

     

       137
       137
       -
           

     

       138
       138
       -
           print(f"Pooch successfully loaded registry with {len(pup.registry)} entries")

     

       139
       139
       -
           for filename in list(pup.registry.keys())[:3]:

     

       140
       140
       -
               print(f"  - {filename}: {pup.registry[filename]}")

     

       141
       141
       -
           

     

       142
       142
       -
           return pup

     

       143
       143
       -
       

     

       144
       144
       -
       def main():

     

       145
       145
       -
           """Main function to generate test registries."""

     

       146
       146
       -
           # Create test directory structure

     

       147
       147
       -
           test_dir = Path("test_data")

     

       148
       148
       -
           test_dir.mkdir(exist_ok=True)

     

       149
       149
       -
           

     

       150
       150
       -
           print("Creating sample files...")

     

       151
       151
       -
           file_info = create_sample_files(test_dir)

     

       152
       152
       -
           print(f"Created {len(file_info)} sample files")

     

       153
       153
       -
           

     

       154
       154
       -
           print("\nCalculating hashes...")

     

       155
       155
       -
           file_hashes = calculate_hashes(file_info)

     

       156
       156
       -
           

     

       157
       157
       -
           # Generate registries with different formats

     

       158
       158
       -
           print("\nGenerating registries...")

     

       159
       159
       -
           

     

       160
       160
       -
           # 1. Standard SHA256 registry (Pooch default)

     

       161
       161
       -
           registry_sha256 = generate_pooch_registry(

     

       162
       162
       -
               file_hashes, 

     

       163
       163
       -
               "test_registry_sha256.txt",

     

       164
       164
       -
               'sha256'

     

       165
       165
       -
           )

     

       166
       166
       -
           print(f"Generated SHA256 registry with {len(registry_sha256)} entries")

     

       167
       167
       -
           

     

       168
       168
       -
           # 2. SHA1 registry

     

       169
       169
       -
           registry_sha1 = generate_pooch_registry(

     

       170
       170
       -
               file_hashes,

     

       171
       171
       -
               "test_registry_sha1.txt", 

     

       172
       172
       -
               'sha1'

     

       173
       173
       -
           )

     

       174
       174
       -
           print(f"Generated SHA1 registry with {len(registry_sha1)} entries")

     

       175
       175
       -
           

     

       176
       176
       -
           # 3. MD5 registry

     

       177
       177
       -
           registry_md5 = generate_pooch_registry(

     

       178
       178
       -
               file_hashes,

     

       179
       179
       -
               "test_registry_md5.txt",

     

       180
       180
       -
               'md5'

     

       181
       181
       -
           )

     

       182
       182
       -
           print(f"Generated MD5 registry with {len(registry_md5)} entries")

     

       183
       183
       -
           

     

       184
       184
       -
           # 4. Mixed format registry

     

       185
       185
       -
           generate_mixed_registry(file_hashes, "test_registry_mixed.txt")

     

       186
       186
       -
           print("Generated mixed format registry")

     

       187
       187
       -
           

     

       188
       188
       -
           # 5. JSON metadata (for verification)

     

       189
       189
       -
           generate_json_metadata(file_hashes, "test_metadata.json")

     

       190
       190
       -
           print("Generated JSON metadata file")

     

       191
       191
       -
           

     

       192
       192
       -
           # Test loading with Pooch

     

       193
       193
       -
           print("\nTesting Pooch compatibility...")

     

       194
       194
       -
           test_pooch_loading("test_registry_sha256.txt")

     

       195
       195
       -
           test_pooch_loading("test_registry_mixed.txt")

     

       196
       196
       -
           

     

       197
       197
       -
           print("\n✅ All registries generated successfully!")

     

       198
       198
       -
           print("\nGenerated files:")

     

       199
       199
       -
           print("  - test_data/           (sample data directory)")

     

       200
       200
       -
           print("  - test_registry_sha256.txt")

     

       201
       201
       -
           print("  - test_registry_sha1.txt")

     

       202
       202
       -
           print("  - test_registry_md5.txt")

     

       203
       203
       -
           print("  - test_registry_mixed.txt")

     

       204
       204
       -
           print("  - test_metadata.json")

     

       205
       205
       -
       

     

       206
       206
       -
       if __name__ == "__main__":

     

       207
       207
       -
           main()

-11

stack/toru/test/python/pyproject.toml

···

       1
       1
       -
       [project]

     

       2
       2
       -
       name = "toru-test-generator"

     

       3
       3
       -
       version = "0.1.0"

     

       4
       4
       -
       description = "Generate Pooch test registries for Toru OCaml testing"

     

       5
       5
       -
       requires-python = ">=3.8"

     

       6
       6
       -
       dependencies = [

     

       7
       7
       -
           "pooch>=1.7.0",

     

       8
       8
       -
       ]

     

       9
       9
       -
       

     

       10
       10
       -
       [tool.uv]

     

       11
       11
       -
       dev-dependencies = []

-1

stack/toru/test/python/test_data/.hidden/secret.txt

···

       1
       1
       -
       Hidden file content

-1

stack/toru/test/python/test_data/config.json

···

       1
       1
       -
       {"version": "1.0", "test": true}

-3

stack/toru/test/python/test_data/data/numbers.csv

-1

stack/toru/test/python/test_data/data/simple.txt

···

       1
       1
       -
       Hello, World!

-1

stack/toru/test/python/test_data/data/unicode.txt

···

       1
       1
       -
       Hello, 世界! 🌍

-3

stack/toru/test/python/test_data/docs/readme.md

···

       1
       1
       -
       # Sample Dataset

     

       2
       2
       -
       

     

       3
       3
       -
       This is a test dataset for Toru.

stack/toru/test/python/test_data/empty.txt

This is a binary file and will not be displayed.

stack/toru/test/python/test_data/models/small.bin

This is a binary file and will not be displayed.

-62

stack/toru/test/python/test_metadata.json

···

       1
       1
       -
       {

     

       2
       2
       -
         "generated": "2025-08-21T08:19:43.410183",

     

       3
       3
       -
         "generator": "pooch_test_generator.py",

     

       4
       4
       -
         "files": [

     

       5
       5
       -
           {

     

       6
       6
       -
             "path": ".hidden/secret.txt",

     

       7
       7
       -
             "size": 19,

     

       8
       8
       -
             "sha256": "9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25",

     

       9
       9
       -
             "sha1": "ce481b91ded06b438fe4fb8d21d94a2ff56b6d92",

     

       10
       10
       -
             "md5": "e6f92a0add5468b0cda31909995bd207"

     

       11
       11
       -
           },

     

       12
       12
       -
           {

     

       13
       13
       -
             "path": "config.json",

     

       14
       14
       -
             "size": 32,

     

       15
       15
       -
             "sha256": "dcfcfeb20e6334be05d4ed2e39da77ffb84b80bf835dc20ac9963f5e95820b94",

     

       16
       16
       -
             "sha1": "3ddf402b6e0e5370a6ea18e67685e47bb362be50",

     

       17
       17
       -
             "md5": "823643193695572be03e4a5456b2b820"

     

       18
       18
       -
           },

     

       19
       19
       -
           {

     

       20
       20
       -
             "path": "data/numbers.csv",

     

       21
       21
       -
             "size": 18,

     

       22
       22
       -
             "sha256": "d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef",

     

       23
       23
       -
             "sha1": "6a65966e545d4fc87687db23c378f1bf4c1c4cdf",

     

       24
       24
       -
             "md5": "9b0137441c80f92c041748e1cfce0631"

     

       25
       25
       -
           },

     

       26
       26
       -
           {

     

       27
       27
       -
             "path": "data/simple.txt",

     

       28
       28
       -
             "size": 13,

     

       29
       29
       -
             "sha256": "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f",

     

       30
       30
       -
             "sha1": "0a0a9f2a6772942557ab5355d76af442f8f65e01",

     

       31
       31
       -
             "md5": "65a8e27d8879283831b664bd8b7f0ad4"

     

       32
       32
       -
           },

     

       33
       33
       -
           {

     

       34
       34
       -
             "path": "data/unicode.txt",

     

       35
       35
       -
             "size": 20,

     

       36
       36
       -
             "sha256": "dcccba292708e4d9b8f1e2af482b09b01131ea6e22c3b10173e0d75c8ac0c310",

     

       37
       37
       -
             "sha1": "43d79663ffe69a0befd5307bcea7295cbdc47e33",

     

       38
       38
       -
             "md5": "d8b80e3903afd84ed3708ef95b65c531"

     

       39
       39
       -
           },

     

       40
       40
       -
           {

     

       41
       41
       -
             "path": "docs/readme.md",

     

       42
       42
       -
             "size": 50,

     

       43
       43
       -
             "sha256": "23c8cf9515ed231a55e63e1d89399a1eac4a529c2b4ac5af29a61af03e3afdd4",

     

       44
       44
       -
             "sha1": "929efb30a9dad70c082af7f4eea3c369896fba5f",

     

       45
       45
       -
             "md5": "9f6c0d6f6f7db11a8e9afc7406e17cab"

     

       46
       46
       -
           },

     

       47
       47
       -
           {

     

       48
       48
       -
             "path": "empty.txt",

     

       49
       49
       -
             "size": 0,

     

       50
       50
       -
             "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",

     

       51
       51
       -
             "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",

     

       52
       52
       -
             "md5": "d41d8cd98f00b204e9800998ecf8427e"

     

       53
       53
       -
           },

     

       54
       54
       -
           {

     

       55
       55
       -
             "path": "models/small.bin",

     

       56
       56
       -
             "size": 1024,

     

       57
       57
       -
             "sha256": "785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9",

     

       58
       58
       -
             "sha1": "5b00669c480d5cffbdfa8bdba99561160f2d1b77",

     

       59
       59
       -
             "md5": "b2ea9f7fcea831a4a63b213f41a8855b"

     

       60
       60
       -
           }

     

       61
       61
       -
         ]

     

       62
       62
       -
       }

-10

stack/toru/test/python/test_registry_md5.txt

···

       1
       1
       -
       # Pooch registry generated on 2025-08-21T08:19:43.410092

     

       2
       2
       -
       # Algorithm: md5

     

       3
       3
       -
       .hidden/secret.txt md5:e6f92a0add5468b0cda31909995bd207

     

       4
       4
       -
       config.json md5:823643193695572be03e4a5456b2b820

     

       5
       5
       -
       data/numbers.csv md5:9b0137441c80f92c041748e1cfce0631

     

       6
       6
       -
       data/simple.txt md5:65a8e27d8879283831b664bd8b7f0ad4

     

       7
       7
       -
       data/unicode.txt md5:d8b80e3903afd84ed3708ef95b65c531

     

       8
       8
       -
       docs/readme.md md5:9f6c0d6f6f7db11a8e9afc7406e17cab

     

       9
       9
       -
       empty.txt md5:d41d8cd98f00b204e9800998ecf8427e

     

       10
       10
       -
       models/small.bin md5:b2ea9f7fcea831a4a63b213f41a8855b

-13

stack/toru/test/python/test_registry_mixed.txt

···

       1
       1
       -
       # Mixed hash format registry for testing

     

       2
       2
       -
       # This tests various hash algorithm prefixes

     

       3
       3
       -
       

     

       4
       4
       -
       .hidden/secret.txt 9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25

     

       5
       5
       -
       config.json sha1:3ddf402b6e0e5370a6ea18e67685e47bb362be50

     

       6
       6
       -
       data/numbers.csv md5:9b0137441c80f92c041748e1cfce0631

     

       7
       7
       -
       

     

       8
       8
       -
       # Additional files

     

       9
       9
       -
       data/simple.txt dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f

     

       10
       10
       -
       data/unicode.txt sha1:43d79663ffe69a0befd5307bcea7295cbdc47e33

     

       11
       11
       -
       docs/readme.md md5:9f6c0d6f6f7db11a8e9afc7406e17cab

     

       12
       12
       -
       empty.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       13
       13
       -
       models/small.bin sha1:5b00669c480d5cffbdfa8bdba99561160f2d1b77

-10

stack/toru/test/python/test_registry_sha1.txt

···

       1
       1
       -
       # Pooch registry generated on 2025-08-21T08:19:43.410026

     

       2
       2
       -
       # Algorithm: sha1

     

       3
       3
       -
       .hidden/secret.txt sha1:ce481b91ded06b438fe4fb8d21d94a2ff56b6d92

     

       4
       4
       -
       config.json sha1:3ddf402b6e0e5370a6ea18e67685e47bb362be50

     

       5
       5
       -
       data/numbers.csv sha1:6a65966e545d4fc87687db23c378f1bf4c1c4cdf

     

       6
       6
       -
       data/simple.txt sha1:0a0a9f2a6772942557ab5355d76af442f8f65e01

     

       7
       7
       -
       data/unicode.txt sha1:43d79663ffe69a0befd5307bcea7295cbdc47e33

     

       8
       8
       -
       docs/readme.md sha1:929efb30a9dad70c082af7f4eea3c369896fba5f

     

       9
       9
       -
       empty.txt sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709

     

       10
       10
       -
       models/small.bin sha1:5b00669c480d5cffbdfa8bdba99561160f2d1b77

-10

stack/toru/test/python/test_registry_sha256.txt

···

       1
       1
       -
       # Pooch registry generated on 2025-08-21T08:19:43.409944

     

       2
       2
       -
       # Algorithm: sha256

     

       3
       3
       -
       .hidden/secret.txt 9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25

     

       4
       4
       -
       config.json dcfcfeb20e6334be05d4ed2e39da77ffb84b80bf835dc20ac9963f5e95820b94

     

       5
       5
       -
       data/numbers.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef

     

       6
       6
       -
       data/simple.txt dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f

     

       7
       7
       -
       data/unicode.txt dcccba292708e4d9b8f1e2af482b09b01131ea6e22c3b10173e0d75c8ac0c310

     

       8
       8
       -
       docs/readme.md 23c8cf9515ed231a55e63e1d89399a1eac4a529c2b4ac5af29a61af03e3afdd4

     

       9
       9
       -
       empty.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       10
       10
       -
       models/small.bin 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9

-183

stack/toru/test/test_cache.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let create_test_file cache filename content =

     

       4
       4
       -
         let path = Cache.file_path cache filename in

     

       5
       5
       -
         (* Ensure cache directory exists *)

     

       6
       6
       -
         Cache.ensure_dir cache;

     

       7
       7
       -
         (* Create parent directory if filename contains path separators *)

     

       8
       8
       -
         (try

     

       9
       9
       -
            if String.contains filename '/' then (

     

       10
       10
       -
              let dir_part = String.sub filename 0 (String.rindex filename '/') in

     

       11
       11
       -
              let dir_path = Cache.file_path cache dir_part in

     

       12
       12
       -
              let mkdir_p path =

     

       13
       13
       -
                try Eio.Path.mkdir ~perm:0o755 path

     

       14
       14
       -
                with _ -> () (* Directory might already exist *)

     

       15
       15
       -
              in

     

       16
       16
       -
              mkdir_p dir_path

     

       17
       17
       -
            );

     

       18
       18
       -
            Eio.Path.with_open_out path ~create:(`If_missing 0o644) (fun flow ->

     

       19
       19
       -
              Eio.Flow.copy_string content flow

     

       20
       20
       -
            )

     

       21
       21
       -
          with

     

       22
       22
       -
          | exn -> Printf.printf "Warning: Failed to create test file %s: %s\n" filename (Printexc.to_string exn)

     

       23
       23
       -
         )

     

       24
       24
       -
       

     

       25
       25
       -
       let test_basic_functionality env sw =

     

       26
       26
       -
         Printf.printf "=== Testing basic Cache functionality ===\n";

     

       27
       27
       -
         

     

       28
       28
       -
         (* Create a test cache *)

     

       29
       29
       -
         let cache = Cache.create ~sw ~fs:env#fs "test_cache" in

     

       30
       30
       -
         Printf.printf "Created cache at: %s\n" 

     

       31
       31
       -
           (Eio.Path.native_exn (Cache.base_path cache));

     

       32
       32
       -
         

     

       33
       33
       -
         (* Test default cache path *)

     

       34
       34
       -
         let default_path = Cache.default_cache_path ~app_name:"test-toru" () in

     

       35
       35
       -
         Printf.printf "Default cache path: %s\n" default_path;

     

       36
       36
       -
         

     

       37
       37
       -
         (* Test XDG cache path detection *)

     

       38
       38
       -
         let cache_with_version = Cache.create ~sw ~fs:env#fs ~version:"v1.0" "test_cache_versioned" in

     

       39
       39
       -
         Printf.printf "Cache with version at: %s\n"

     

       40
       40
       -
           (Eio.Path.native_exn (Cache.base_path cache_with_version));

     

       41
       41
       -
         

     

       42
       42
       -
         (* Test directory creation *)

     

       43
       43
       -
         Printf.printf "Creating cache directories...\n";

     

       44
       44
       -
         Cache.ensure_dir cache;

     

       45
       45
       -
         Cache.ensure_dir cache_with_version;

     

       46
       46
       -
         

     

       47
       47
       -
         (* Test file existence check *)

     

       48
       48
       -
         let test_file_exists = Cache.exists cache "nonexistent.txt" in

     

       49
       49
       -
         Printf.printf "Non-existent file exists: %b\n" test_file_exists;

     

       50
       50
       -
         

     

       51
       51
       -
         (* Test cache listing (should be empty initially) *)

     

       52
       52
       -
         let files = Cache.list_files cache in

     

       53
       53
       -
         Printf.printf "Cache contains %d files initially\n" (List.length files);

     

       54
       54
       -
         

     

       55
       55
       -
         (* Test cache size *)

     

       56
       56
       -
         let size = Cache.size_bytes cache in

     

       57
       57
       -
         Printf.printf "Initial cache size: %Ld bytes\n" size;

     

       58
       58
       -
         

     

       59
       59
       -
         cache

     

       60
       60
       -
       

     

       61
       61
       -
       let test_file_operations _env _sw cache =

     

       62
       62
       -
         Printf.printf "\n=== Testing file operations ===\n";

     

       63
       63
       -
         

     

       64
       64
       -
         (* Create some test files *)

     

       65
       65
       -
         create_test_file cache "test1.txt" "Hello World!";

     

       66
       66
       -
         create_test_file cache "subdir/test2.txt" "This is a test file in a subdirectory.";

     

       67
       67
       -
         create_test_file cache "test3.txt" "Another test file with different content for size testing.";

     

       68
       68
       -
         

     

       69
       69
       -
         Printf.printf "Created test files\n";

     

       70
       70
       -
         

     

       71
       71
       -
         (* Test file existence *)

     

       72
       72
       -
         let exists1 = Cache.exists cache "test1.txt" in

     

       73
       73
       -
         let exists2 = Cache.exists cache "subdir/test2.txt" in

     

       74
       74
       -
         let exists3 = Cache.exists cache "nonexistent.txt" in

     

       75
       75
       -
         Printf.printf "test1.txt exists: %b\n" exists1;

     

       76
       76
       -
         Printf.printf "subdir/test2.txt exists: %b\n" exists2;

     

       77
       77
       -
         Printf.printf "nonexistent.txt exists: %b\n" exists3;

     

       78
       78
       -
         

     

       79
       79
       -
         (* Test file listing *)

     

       80
       80
       -
         let files = Cache.list_files cache in

     

       81
       81
       -
         Printf.printf "Cache now contains %d files:\n" (List.length files);

     

       82
       82
       -
         List.iter (fun f -> Printf.printf "  - %s\n" f) (List.sort String.compare files);

     

       83
       83
       -
         

     

       84
       84
       -
         (* Test cache size *)

     

       85
       85
       -
         let size = Cache.size_bytes cache in

     

       86
       86
       -
         Printf.printf "Cache size after adding files: %Ld bytes\n" size

     

       87
       87
       -
       

     

       88
       88
       -
       let test_file_info _env _sw cache =

     

       89
       89
       -
         Printf.printf "\n=== Testing file info ===\n";

     

       90
       90
       -
         

     

       91
       91
       -
         (* Test file info for existing file *)

     

       92
       92
       -
         (match Cache.file_info cache "test1.txt" with

     

       93
       93
       -
          | Some info ->

     

       94
       94
       -
              Printf.printf "test1.txt: size=%Ld bytes, mtime=%.0f\n" info.size info.mtime

     

       95
       95
       -
          | None ->

     

       96
       96
       -
              Printf.printf "Could not get info for test1.txt\n");

     

       97
       97
       -
         

     

       98
       98
       -
         (* Test file info for non-existent file *)

     

       99
       99
       -
         (match Cache.file_info cache "nonexistent.txt" with

     

       100
       100
       -
          | Some _ ->

     

       101
       101
       -
              Printf.printf "ERROR: Got info for non-existent file\n"

     

       102
       102
       -
          | None ->

     

       103
       103
       -
              Printf.printf "Correctly returned None for non-existent file\n")

     

       104
       104
       -
       

     

       105
       105
       -
       let test_usage_stats _env _sw cache =

     

       106
       106
       -
         Printf.printf "\n=== Testing usage stats ===\n";

     

       107
       107
       -
         

     

       108
       108
       -
         let stats = Cache.usage_stats cache in

     

       109
       109
       -
         Printf.printf "Cache statistics:\n";

     

       110
       110
       -
         Printf.printf "  Total size: %Ld bytes\n" stats.total_size;

     

       111
       111
       -
         Printf.printf "  File count: %d\n" stats.file_count;

     

       112
       112
       -
         Printf.printf "  Oldest file: %.0f (Unix timestamp)\n" stats.oldest;

     

       113
       113
       -
         Printf.printf "  Newest file: %.0f (Unix timestamp)\n" stats.newest

     

       114
       114
       -
       

     

       115
       115
       -
       let test_trim_operations _env _sw cache =

     

       116
       116
       -
         Printf.printf "\n=== Testing trim operations ===\n";

     

       117
       117
       -
         

     

       118
       118
       -
         (* Get initial stats *)

     

       119
       119
       -
         let initial_stats = Cache.usage_stats cache in

     

       120
       120
       -
         Printf.printf "Before trim - Files: %d, Size: %Ld bytes\n" 

     

       121
       121
       -
           initial_stats.file_count initial_stats.total_size;

     

       122
       122
       -
         

     

       123
       123
       -
         (* Test trim by size (set very small limit to force trimming) *)

     

       124
       124
       -
         Cache.trim_to_size cache 20L;  (* 20 bytes *)

     

       125
       125
       -
         let after_size_trim = Cache.usage_stats cache in

     

       126
       126
       -
         Printf.printf "After size trim (20 bytes) - Files: %d, Size: %Ld bytes\n"

     

       127
       127
       -
           after_size_trim.file_count after_size_trim.total_size;

     

       128
       128
       -
         

     

       129
       129
       -
         (* Create a new file for age testing *)

     

       130
       130
       -
         create_test_file cache "new_test.txt" "New file for age testing";

     

       131
       131
       -
         

     

       132
       132
       -
         (* Test trim by age (0 days = remove all files) *)

     

       133
       133
       -
         Cache.trim_by_age cache 0.0;

     

       134
       134
       -
         let after_age_trim = Cache.usage_stats cache in

     

       135
       135
       -
         Printf.printf "After age trim (0 days) - Files: %d, Size: %Ld bytes\n"

     

       136
       136
       -
           after_age_trim.file_count after_age_trim.total_size

     

       137
       137
       -
       

     

       138
       138
       -
       let test_vacuum _env _sw cache =

     

       139
       139
       -
         Printf.printf "\n=== Testing vacuum operations ===\n";

     

       140
       140
       -
         

     

       141
       141
       -
         (* Create some files and then remove them to create empty directories *)

     

       142
       142
       -
         create_test_file cache "temp/deep/nested/file.txt" "temporary";

     

       143
       143
       -
         let temp_path = Cache.file_path cache "temp/deep/nested/file.txt" in

     

       144
       144
       -
         (try Eio.Path.unlink temp_path with _ -> ());

     

       145
       145
       -
         

     

       146
       146
       -
         Printf.printf "Created and removed nested file to test vacuum\n";

     

       147
       147
       -
         

     

       148
       148
       -
         (* Run vacuum to clean up empty directories *)

     

       149
       149
       -
         Cache.vacuum cache;

     

       150
       150
       -
         Printf.printf "Vacuum completed\n"

     

       151
       151
       -
       

     

       152
       152
       -
       let test_clear _env _sw cache =

     

       153
       153
       -
         Printf.printf "\n=== Testing cache clear ===\n";

     

       154
       154
       -
         

     

       155
       155
       -
         (* Add some files first *)

     

       156
       156
       -
         create_test_file cache "clear_test1.txt" "test content 1";

     

       157
       157
       -
         create_test_file cache "clear_test2.txt" "test content 2";

     

       158
       158
       -
         

     

       159
       159
       -
         let before_clear = Cache.usage_stats cache in

     

       160
       160
       -
         Printf.printf "Before clear - Files: %d\n" before_clear.file_count;

     

       161
       161
       -
         

     

       162
       162
       -
         (* Clear the cache *)

     

       163
       163
       -
         Cache.clear cache;

     

       164
       164
       -
         

     

       165
       165
       -
         let after_clear = Cache.usage_stats cache in

     

       166
       166
       -
         Printf.printf "After clear - Files: %d\n" after_clear.file_count

     

       167
       167
       -
       

     

       168
       168
       -
       let test_cache_functionality () =

     

       169
       169
       -
         Printf.printf "Testing Cache functionality...\n";

     

       170
       170
       -
         

     

       171
       171
       -
         Eio_main.run @@ fun env ->

     

       172
       172
       -
         Eio.Switch.run @@ fun sw ->

     

       173
       173
       -
           let cache = test_basic_functionality env sw in

     

       174
       174
       -
           test_file_operations env sw cache;

     

       175
       175
       -
           test_file_info env sw cache;

     

       176
       176
       -
           test_usage_stats env sw cache;

     

       177
       177
       -
           test_trim_operations env sw cache;

     

       178
       178
       -
           test_vacuum env sw cache;

     

       179
       179
       -
           test_clear env sw cache;

     

       180
       180
       -
           

     

       181
       181
       -
           Printf.printf "\n=== All cache tests completed successfully ===\n"

     

       182
       182
       -
       

     

       183
       183
       -
       let () = test_cache_functionality ()

-159

stack/toru/test/test_cache_xdg.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       (* Test XDG Base Directory compliance and platform detection *)

     

       4
       4
       -
       let test_xdg_compliance () =

     

       5
       5
       -
         Printf.printf "=== Testing XDG Base Directory Compliance ===\n";

     

       6
       6
       -
         

     

       7
       7
       -
         (* Test default path without environment override *)

     

       8
       8
       -
         let default_path = Cache.default_cache_path ~app_name:"test-app" () in

     

       9
       9
       -
         Printf.printf "Default cache path: %s\n" default_path;

     

       10
       10
       -
         

     

       11
       11
       -
         (* Check if path contains expected platform-specific component *)

     

       12
       12
       -
         let is_macos = try

     

       13
       13
       -
           let ic = Unix.open_process_in "uname -s 2>/dev/null" in

     

       14
       14
       -
           let result = String.trim (input_line ic) in

     

       15
       15
       -
           let _ = Unix.close_process_in ic in

     

       16
       16
       -
           result = "Darwin"

     

       17
       17
       -
         with _ -> false in

     

       18
       18
       -
         

     

       19
       19
       -
         if is_macos then (

     

       20
       20
       -
           let expected_macos_path = Filename.concat (Filename.concat (Sys.getenv "HOME") "Library") "Caches" in

     

       21
       21
       -
           if String.length default_path >= String.length expected_macos_path &&

     

       22
       22
       -
              String.sub default_path 0 (String.length expected_macos_path) = expected_macos_path then

     

       23
       23
       -
             Printf.printf "✓ macOS: Using ~/Library/Caches correctly\n"

     

       24
       24
       -
           else

     

       25
       25
       -
             Printf.printf "✗ macOS: Expected ~/Library/Caches path\n"

     

       26
       26
       -
         ) else (

     

       27
       27
       -
           (* Simple check if path contains .cache substring *)

     

       28
       28
       -
           let rec contains_substr s substr pos =

     

       29
       29
       -
             if pos + String.length substr > String.length s then false

     

       30
       30
       -
             else if String.sub s pos (String.length substr) = substr then true

     

       31
       31
       -
             else contains_substr s substr (pos + 1)

     

       32
       32
       -
           in

     

       33
       33
       -
           if contains_substr default_path ".cache" 0 then

     

       34
       34
       -
             Printf.printf "✓ Unix/Linux: Using ~/.cache correctly\n"

     

       35
       35
       -
           else

     

       36
       36
       -
             Printf.printf "✗ Unix/Linux: Expected ~/.cache path\n"

     

       37
       37
       -
         );

     

       38
       38
       -
         

     

       39
       39
       -
         (* Test with XDG_CACHE_HOME override *)

     

       40
       40
       -
         let original_xdg = Sys.getenv_opt "XDG_CACHE_HOME" in

     

       41
       41
       -
         ignore (Sys.signal Sys.sigint (Sys.Signal_handle (fun _ -> 

     

       42
       42
       -
           (* Restore original XDG_CACHE_HOME on interrupt *)

     

       43
       43
       -
           (match original_xdg with

     

       44
       44
       -
            | Some v -> Unix.putenv "XDG_CACHE_HOME" v

     

       45
       45
       -
            | None -> (* Cannot unset env var *) ());

     

       46
       46
       -
           exit 1)));

     

       47
       47
       -
         

     

       48
       48
       -
         (* Set temporary XDG_CACHE_HOME *)

     

       49
       49
       -
         Unix.putenv "XDG_CACHE_HOME" "/tmp/test-xdg-cache";

     

       50
       50
       -
         let xdg_override_path = Cache.default_cache_path ~app_name:"test-app" () in

     

       51
       51
       -
         

     

       52
       52
       -
         (* Restore original environment *)

     

       53
       53
       -
         (match original_xdg with

     

       54
       54
       -
          | Some v -> Unix.putenv "XDG_CACHE_HOME" v

     

       55
       55
       -
          | None -> (* Cannot unset env var *) ());

     

       56
       56
       -
         

     

       57
       57
       -
         if not is_macos then (

     

       58
       58
       -
           let expected_xdg_path = "/tmp/test-xdg-cache" in

     

       59
       59
       -
           if String.length xdg_override_path >= String.length expected_xdg_path &&

     

       60
       60
       -
              String.sub xdg_override_path 0 (String.length expected_xdg_path) = expected_xdg_path then

     

       61
       61
       -
             Printf.printf "✓ XDG_CACHE_HOME override works: %s\n" xdg_override_path

     

       62
       62
       -
           else

     

       63
       63
       -
             Printf.printf "✗ XDG_CACHE_HOME override failed: %s\n" xdg_override_path

     

       64
       64
       -
         ) else (

     

       65
       65
       -
           Printf.printf "ℹ macOS: XDG override test skipped (uses Library/Caches)\n"

     

       66
       66
       -
         );

     

       67
       67
       -
         

     

       68
       68
       -
         Printf.printf "\n"

     

       69
       69
       -
       

     

       70
       70
       -
       let test_versioned_caches () =

     

       71
       71
       -
         Printf.printf "=== Testing Versioned Cache Directories ===\n";

     

       72
       72
       -
         

     

       73
       73
       -
         Eio_main.run @@ fun env ->

     

       74
       74
       -
         Eio.Switch.run @@ fun sw ->

     

       75
       75
       -
           (* Test cache without version *)

     

       76
       76
       -
           let cache_no_version = Cache.create ~sw ~fs:env#fs "test_cache_no_version" in

     

       77
       77
       -
           let base_path = Cache.base_path cache_no_version in

     

       78
       78
       -
           Printf.printf "Cache without version: %s\n" (Eio.Path.native_exn base_path);

     

       79
       79
       -
           

     

       80
       80
       -
           (* Test cache with version *)

     

       81
       81
       -
           let cache_with_version = Cache.create ~sw ~fs:env#fs ~version:"v2.1" "test_cache_with_version" in

     

       82
       82
       -
           let versioned_path = Cache.base_path cache_with_version in

     

       83
       83
       -
           Printf.printf "Cache with version: %s\n" (Eio.Path.native_exn versioned_path);

     

       84
       84
       -
           

     

       85
       85
       -
           (* Test file paths *)

     

       86
       86
       -
           let file_path_no_version = Cache.file_path cache_no_version "data.txt" in

     

       87
       87
       -
           let file_path_with_version = Cache.file_path cache_with_version "data.txt" in

     

       88
       88
       -
           

     

       89
       89
       -
           Printf.printf "File path without version: %s\n" (Eio.Path.native_exn file_path_no_version);

     

       90
       90
       -
           Printf.printf "File path with version: %s\n" (Eio.Path.native_exn file_path_with_version);

     

       91
       91
       -
           

     

       92
       92
       -
           (* Verify version is correctly stored *)

     

       93
       93
       -
           Printf.printf "Version accessor test: %s\n" 

     

       94
       94
       -
             (match Cache.version cache_with_version with

     

       95
       95
       -
              | Some v -> v

     

       96
       96
       -
              | None -> "no version");

     

       97
       97
       -
           

     

       98
       98
       -
           Printf.printf "\n"

     

       99
       99
       -
       

     

       100
       100
       -
       let test_cache_management () =

     

       101
       101
       -
         Printf.printf "=== Testing Advanced Cache Management ===\n";

     

       102
       102
       -
         

     

       103
       103
       -
         Eio_main.run @@ fun env ->

     

       104
       104
       -
         Eio.Switch.run @@ fun sw ->

     

       105
       105
       -
           let cache = Cache.create ~sw ~fs:env#fs "test_cache_mgmt" in

     

       106
       106
       -
           Cache.ensure_dir cache;

     

       107
       107
       -
           

     

       108
       108
       -
           (* Create test files with different content sizes *)

     

       109
       109
       -
           let test_files = [

     

       110
       110
       -
             ("small.txt", "small");

     

       111
       111
       -
             ("medium.txt", String.make 100 'M');

     

       112
       112
       -
             ("large.txt", String.make 1000 'L');

     

       113
       113
       -
           ] in

     

       114
       114
       -
           

     

       115
       115
       -
           List.iter (fun (filename, content) ->

     

       116
       116
       -
             let path = Cache.file_path cache filename in

     

       117
       117
       -
             Eio.Path.with_open_out path ~create:(`If_missing 0o644) (fun flow ->

     

       118
       118
       -
               Eio.Flow.copy_string content flow

     

       119
       119
       -
             )

     

       120
       120
       -
           ) test_files;

     

       121
       121
       -
           

     

       122
       122
       -
           Printf.printf "Created test files\n";

     

       123
       123
       -
           

     

       124
       124
       -
           (* Test file_info *)

     

       125
       125
       -
           Printf.printf "\nFile information:\n";

     

       126
       126
       -
           List.iter (fun (filename, expected_content) ->

     

       127
       127
       -
             match Cache.file_info cache filename with

     

       128
       128
       -
             | Some info ->

     

       129
       129
       -
                 let expected_size = String.length expected_content in

     

       130
       130
       -
                 Printf.printf "  %s: size=%Ld (expected=%d), mtime=%.0f\n" 

     

       131
       131
       -
                   filename info.size expected_size info.mtime;

     

       132
       132
       -
                 if Int64.to_int info.size = expected_size then

     

       133
       133
       -
                   Printf.printf "    ✓ Size matches\n"

     

       134
       134
       -
                 else

     

       135
       135
       -
                   Printf.printf "    ✗ Size mismatch\n"

     

       136
       136
       -
             | None ->

     

       137
       137
       -
                 Printf.printf "  %s: No info available\n" filename

     

       138
       138
       -
           ) test_files;

     

       139
       139
       -
           

     

       140
       140
       -
           (* Test usage stats *)

     

       141
       141
       -
           let stats = Cache.usage_stats cache in

     

       142
       142
       -
           Printf.printf "\nUsage statistics:\n";

     

       143
       143
       -
           Printf.printf "  Total files: %d\n" stats.file_count;

     

       144
       144
       -
           Printf.printf "  Total size: %Ld bytes\n" stats.total_size;

     

       145
       145
       -
           Printf.printf "  Age range: %.0f - %.0f\n" stats.oldest stats.newest;

     

       146
       146
       -
           

     

       147
       147
       -
           (* Test trim to size *)

     

       148
       148
       -
           Printf.printf "\nTesting trim to size (50 bytes):\n";

     

       149
       149
       -
           Cache.trim_to_size cache 50L;

     

       150
       150
       -
           let after_trim = Cache.usage_stats cache in

     

       151
       151
       -
           Printf.printf "  After trim: %d files, %Ld bytes\n" 

     

       152
       152
       -
             after_trim.file_count after_trim.total_size;

     

       153
       153
       -
           

     

       154
       154
       -
           Printf.printf "\n"

     

       155
       155
       -
       

     

       156
       156
       -
       let () =

     

       157
       157
       -
         test_xdg_compliance ();

     

       158
       158
       -
         test_versioned_caches ();

     

       159
       159
       -
         test_cache_management ()

-39

stack/toru/test/test_curl_download.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let test_curl_download () =

     

       4
       4
       -
         Printf.printf "Testing curl download...\n";

     

       5
       5
       -
         

     

       6
       6
       -
         Eio_main.run @@ fun env ->

     

       7
       7
       -
         Eio.Switch.run @@ fun sw ->

     

       8
       8
       -
           try

     

       9
       9
       -
             (* Create curl downloader *)

     

       10
       10
       -
             let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.curl () in

     

       11
       11
       -
             let downloader = D.create ~sw ~env () in

     

       12
       12
       -
             

     

       13
       13
       -
             (* Test a small file download *)

     

       14
       14
       -
             let test_url = "https://httpbin.org/robots.txt" in

     

       15
       15
       -
             let dest_path = Eio.Path.(env#fs / "test_download_curl.txt") in

     

       16
       16
       -
             

     

       17
       17
       -
             Printf.printf "Downloading %s with curl...\n" test_url;

     

       18
       18
       -
             

     

       19
       19
       -
             match D.download downloader ~url:test_url ~dest:dest_path () with

     

       20
       20
       -
             | Ok () ->

     

       21
       21
       -
                 Printf.printf "Curl download successful!\n";

     

       22
       22
       -
                 (* Check if file exists and has content *)

     

       23
       23
       -
                 let file_exists = try

     

       24
       24
       -
                   let _stat = Eio.Path.stat ~follow:false dest_path in true

     

       25
       25
       -
                 with _ -> false in

     

       26
       26
       -
                 if file_exists then (

     

       27
       27
       -
                   Printf.printf "File downloaded and exists\n";

     

       28
       28
       -
                   (* Clean up *)

     

       29
       29
       -
                   Eio.Path.unlink dest_path;

     

       30
       30
       -
                 ) else (

     

       31
       31
       -
                   Printf.printf "File doesn't exist after download\n"

     

       32
       32
       -
                 )

     

       33
       33
       -
             | Error msg ->

     

       34
       34
       -
                 Printf.printf "Curl download failed: %s\n" msg

     

       35
       35
       -
           with

     

       36
       36
       -
           | exn ->

     

       37
       37
       -
               Printf.printf "Exception during curl test: %s\n" (Printexc.to_string exn)

     

       38
       38
       -
       

     

       39
       39
       -
       let () = test_curl_download ()

-272

stack/toru/test/test_downloader.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let test_downloader_detection () =

     

       4
       4
       -
         Printf.printf "Testing downloader detection...\n";

     

       5
       5
       -
         

     

       6
       6
       -
         Eio_main.run @@ fun env ->

     

       7
       7
       -
         Eio.Switch.run @@ fun _sw ->

     

       8
       8
       -
           (* Test detecting available downloaders *)

     

       9
       9
       -
           let available = Downloader.Downloaders.detect_available ~env in

     

       10
       10
       -
           Printf.printf "Available downloaders:\n";

     

       11
       11
       -
           List.iter (fun (name, _) -> 

     

       12
       12
       -
             Printf.printf "  - %s\n" name

     

       13
       13
       -
           ) available;

     

       14
       14
       -
           

     

       15
       15
       -
           (* Test creating default downloader *)

     

       16
       16
       -
           if List.length available > 0 then (

     

       17
       17
       -
             let _default_downloader = Downloader.Downloaders.create_default ~env in

     

       18
       18
       -
             Printf.printf "Default downloader created successfully\n"

     

       19
       19
       -
           ) else (

     

       20
       20
       -
             Printf.printf "No downloaders available - skipping default test\n"

     

       21
       21
       -
           );

     

       22
       22
       -
           

     

       23
       23
       -
           Printf.printf "Downloader detection test completed\n"

     

       24
       24
       -
       

     

       25
       25
       -
       let test_wget_download () =

     

       26
       26
       -
         Printf.printf "\nTesting wget download...\n";

     

       27
       27
       -
         

     

       28
       28
       -
         Eio_main.run @@ fun env ->

     

       29
       29
       -
         Eio.Switch.run @@ fun sw ->

     

       30
       30
       -
           try

     

       31
       31
       -
             (* Create wget downloader *)

     

       32
       32
       -
             let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.wget () in

     

       33
       33
       -
             let downloader = D.create ~sw ~env () in

     

       34
       34
       -
             

     

       35
       35
       -
             (* Test a small file download *)

     

       36
       36
       -
             let test_url = "https://httpbin.org/robots.txt" in

     

       37
       37
       -
             let dest_path = Eio.Path.(env#fs / "test_download.txt") in

     

       38
       38
       -
             

     

       39
       39
       -
             Printf.printf "Downloading %s...\n" test_url;

     

       40
       40
       -
             

     

       41
       41
       -
             match D.download downloader ~url:test_url ~dest:dest_path () with

     

       42
       42
       -
             | Ok () ->

     

       43
       43
       -
                 Printf.printf "Download successful!\n";

     

       44
       44
       -
                 (* Check if file exists and has content *)

     

       45
       45
       -
                 let file_exists = try

     

       46
       46
       -
                   let _stat = Eio.Path.stat ~follow:false dest_path in true

     

       47
       47
       -
                 with _ -> false in

     

       48
       48
       -
                 if file_exists then (

     

       49
       49
       -
                   Printf.printf "File downloaded and exists\n";

     

       50
       50
       -
                   (* Clean up *)

     

       51
       51
       -
                   Eio.Path.unlink dest_path;

     

       52
       52
       -
                 ) else (

     

       53
       53
       -
                   Printf.printf "File doesn't exist after download\n"

     

       54
       54
       -
                 )

     

       55
       55
       -
             | Error msg ->

     

       56
       56
       -
                 Printf.printf "Download failed: %s\n" msg

     

       57
       57
       -
           with

     

       58
       58
       -
           | exn ->

     

       59
       59
       -
               Printf.printf "Exception during wget test: %s\n" (Printexc.to_string exn)

     

       60
       60
       -
               

     

       61
       61
       -
       let test_authentication () =

     

       62
       62
       -
         Printf.printf "\nTesting authentication support...\n";

     

       63
       63
       -
         

     

       64
       64
       -
         Eio_main.run @@ fun env ->

     

       65
       65
       -
         Eio.Switch.run @@ fun sw ->

     

       66
       66
       -
           try

     

       67
       67
       -
             (* Test wget with authentication *)

     

       68
       68
       -
             let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.wget () in

     

       69
       69
       -
             let auth = Downloader.Config.{ username = Some "testuser"; password = Some "testpass" } in

     

       70
       70
       -
             let downloader = D.create ~sw ~env ~auth () in

     

       71
       71
       -
             

     

       72
       72
       -
             Printf.printf "Created wget downloader with authentication\n";

     

       73
       73
       -
             Printf.printf "Downloader name: %s\n" (D.name downloader);

     

       74
       74
       -
             Printf.printf "Supports resume: %b\n" (D.supports_resume downloader);

     

       75
       75
       -
             

     

       76
       76
       -
             (* Test curl with authentication *)

     

       77
       77
       -
             let (module D2 : Downloader.DOWNLOADER) = Downloader.Downloaders.curl () in

     

       78
       78
       -
             let downloader2 = D2.create ~sw ~env ~auth () in

     

       79
       79
       -
             

     

       80
       80
       -
             Printf.printf "Created curl downloader with authentication\n";

     

       81
       81
       -
             Printf.printf "Downloader name: %s\n" (D2.name downloader2);

     

       82
       82
       -
             Printf.printf "Supports resume: %b\n" (D2.supports_resume downloader2);

     

       83
       83
       -
             

     

       84
       84
       -
             (* Test authentication without credentials *)

     

       85
       85
       -
             let downloader3 = D.create ~sw ~env () in

     

       86
       86
       -
             Printf.printf "Created downloader without credentials: %s\n" (D.name downloader3);

     

       87
       87
       -
             

     

       88
       88
       -
           with

     

       89
       89
       -
           | exn ->

     

       90
       90
       -
               Printf.printf "Exception during authentication test: %s\n" (Printexc.to_string exn)

     

       91
       91
       -
       

     

       92
       92
       -
       let test_resume_functionality () =

     

       93
       93
       -
         Printf.printf "\nTesting resume functionality...\n";

     

       94
       94
       -
         

     

       95
       95
       -
         Eio_main.run @@ fun env ->

     

       96
       96
       -
         Eio.Switch.run @@ fun sw ->

     

       97
       97
       -
           try

     

       98
       98
       -
             let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.wget () in

     

       99
       99
       -
             let downloader = D.create ~sw ~env () in

     

       100
       100
       -
             

     

       101
       101
       -
             let test_url = "https://httpbin.org/robots.txt" in

     

       102
       102
       -
             let dest_path = Eio.Path.(env#fs / "test_resume.txt") in

     

       103
       103
       -
             

     

       104
       104
       -
             Printf.printf "Testing download with resume enabled...\n";

     

       105
       105
       -
             

     

       106
       106
       -
             match D.download downloader ~url:test_url ~dest:dest_path ~resume:true () with

     

       107
       107
       -
             | Ok () ->

     

       108
       108
       -
                 Printf.printf "Download with resume=true successful\n";

     

       109
       109
       -
                 let file_exists = try

     

       110
       110
       -
                   let _stat = Eio.Path.stat ~follow:false dest_path in true

     

       111
       111
       -
                 with _ -> false in

     

       112
       112
       -
                 if file_exists then (

     

       113
       113
       -
                   Printf.printf "Resume test successful - cleaning up\n";

     

       114
       114
       -
                   Eio.Path.unlink dest_path;

     

       115
       115
       -
                 )

     

       116
       116
       -
             | Error msg ->

     

       117
       117
       -
                 Printf.printf "Resume download failed: %s\n" msg;

     

       118
       118
       -
                 

     

       119
       119
       -
             Printf.printf "Testing download with resume disabled...\n";

     

       120
       120
       -
             

     

       121
       121
       -
             match D.download downloader ~url:test_url ~dest:dest_path ~resume:false () with

     

       122
       122
       -
             | Ok () ->

     

       123
       123
       -
                 Printf.printf "Download with resume=false successful\n";

     

       124
       124
       -
                 let file_exists = try

     

       125
       125
       -
                   let _stat = Eio.Path.stat ~follow:false dest_path in true

     

       126
       126
       -
                 with _ -> false in

     

       127
       127
       -
                 if file_exists then (

     

       128
       128
       -
                   Printf.printf "No-resume test successful - cleaning up\n";

     

       129
       129
       -
                   Eio.Path.unlink dest_path;

     

       130
       130
       -
                 )

     

       131
       131
       -
             | Error msg ->

     

       132
       132
       -
                 Printf.printf "No-resume download failed: %s\n" msg;

     

       133
       133
       -
           with

     

       134
       134
       -
           | exn ->

     

       135
       135
       -
               Printf.printf "Exception during resume test: %s\n" (Printexc.to_string exn)

     

       136
       136
       -
       

     

       137
       137
       -
       let test_error_handling () =

     

       138
       138
       -
         Printf.printf "\nTesting error handling...\n";

     

       139
       139
       -
         

     

       140
       140
       -
         Eio_main.run @@ fun env ->

     

       141
       141
       -
         Eio.Switch.run @@ fun sw ->

     

       142
       142
       -
           try

     

       143
       143
       -
             let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.wget () in

     

       144
       144
       -
             let downloader = D.create ~sw ~env () in

     

       145
       145
       -
             

     

       146
       146
       -
             (* Test with invalid URL *)

     

       147
       147
       -
             let invalid_url = "https://this-domain-does-not-exist-12345.com/file.txt" in

     

       148
       148
       -
             let dest_path = Eio.Path.(env#fs / "test_error.txt") in

     

       149
       149
       -
             

     

       150
       150
       -
             Printf.printf "Testing error handling with invalid URL...\n";

     

       151
       151
       -
             

     

       152
       152
       -
             match D.download downloader ~url:invalid_url ~dest:dest_path () with

     

       153
       153
       -
             | Ok () ->

     

       154
       154
       -
                 Printf.printf "ERROR: Invalid URL should have failed!\n"

     

       155
       155
       -
             | Error msg ->

     

       156
       156
       -
                 Printf.printf "Error handling successful: %s\n" msg;

     

       157
       157
       -
                 

     

       158
       158
       -
             (* Test with invalid file path *)

     

       159
       159
       -
             let valid_url = "https://httpbin.org/robots.txt" in

     

       160
       160
       -
             let invalid_dest = Eio.Path.(env#fs / "/invalid/path/that/does/not/exist/file.txt") in

     

       161
       161
       -
             

     

       162
       162
       -
             Printf.printf "Testing error handling with invalid destination...\n";

     

       163
       163
       -
             

     

       164
       164
       -
             match D.download downloader ~url:valid_url ~dest:invalid_dest () with

     

       165
       165
       -
             | Ok () ->

     

       166
       166
       -
                 Printf.printf "WARNING: Invalid path might have succeeded unexpectedly\n"

     

       167
       167
       -
             | Error msg ->

     

       168
       168
       -
                 Printf.printf "Path error handling successful: %s\n" msg

     

       169
       169
       -
                 

     

       170
       170
       -
           with

     

       171
       171
       -
           | exn ->

     

       172
       172
       -
               Printf.printf "Exception during error handling test: %s\n" (Printexc.to_string exn)

     

       173
       173
       -
       

     

       174
       174
       -
       let test_hash_verification () =

     

       175
       175
       -
         Printf.printf "\nTesting hash verification...\n";

     

       176
       176
       -
         

     

       177
       177
       -
         Eio_main.run @@ fun env ->

     

       178
       178
       -
         Eio.Switch.run @@ fun sw ->

     

       179
       179
       -
           try

     

       180
       180
       -
             let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.wget () in

     

       181
       181
       -
             let downloader = D.create ~sw ~env () in

     

       182
       182
       -
             

     

       183
       183
       -
             let test_url = "https://httpbin.org/robots.txt" in

     

       184
       184
       -
             let dest_path = Eio.Path.(env#fs / "test_hash.txt") in

     

       185
       185
       -
             

     

       186
       186
       -
             (* First download without hash to see the actual content *)

     

       187
       187
       -
             Printf.printf "Downloading file to compute hash...\n";

     

       188
       188
       -
             

     

       189
       189
       -
             match D.download downloader ~url:test_url ~dest:dest_path () with

     

       190
       190
       -
             | Ok () ->

     

       191
       191
       -
                 (* Compute the actual hash *)

     

       192
       192
       -
                 let actual_hash = Hash.compute Hash.SHA256 dest_path in

     

       193
       193
       -
                 Printf.printf "Computed hash: %s\n" (Hash.to_string actual_hash);

     

       194
       194
       -
                 

     

       195
       195
       -
                 (* Clean up and test with correct hash *)

     

       196
       196
       -
                 Eio.Path.unlink dest_path;

     

       197
       197
       -
                 

     

       198
       198
       -
                 Printf.printf "Testing download with correct hash...\n";

     

       199
       199
       -
                 (match D.download downloader ~url:test_url ~dest:dest_path ~hash:actual_hash () with

     

       200
       200
       -
                 | Ok () ->

     

       201
       201
       -
                     Printf.printf "Hash verification successful\n";

     

       202
       202
       -
                     Eio.Path.unlink dest_path;

     

       203
       203
       -
                 | Error msg ->

     

       204
       204
       -
                     Printf.printf "Hash verification failed unexpectedly: %s\n" msg);

     

       205
       205
       -
                 

     

       206
       206
       -
                 (* Test with incorrect hash *)

     

       207
       207
       -
                 let wrong_hash = Hash.create Hash.SHA256 "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" in

     

       208
       208
       -
                 Printf.printf "Testing download with incorrect hash...\n";

     

       209
       209
       -
                 (match D.download downloader ~url:test_url ~dest:dest_path ~hash:wrong_hash () with

     

       210
       210
       -
                 | Ok () ->

     

       211
       211
       -
                     Printf.printf "ERROR: Wrong hash should have failed!\n";

     

       212
       212
       -
                     Eio.Path.unlink dest_path;

     

       213
       213
       -
                 | Error msg ->

     

       214
       214
       -
                     Printf.printf "Hash verification error handling successful: %s\n" msg)

     

       215
       215
       -
                     

     

       216
       216
       -
             | Error msg ->

     

       217
       217
       -
                 Printf.printf "Initial download for hash test failed: %s\n" msg

     

       218
       218
       -
           with

     

       219
       219
       -
           | exn ->

     

       220
       220
       -
               Printf.printf "Exception during hash verification test: %s\n" (Printexc.to_string exn)

     

       221
       221
       -
       

     

       222
       222
       -
       let test_downloader_selection () =

     

       223
       223
       -
         Printf.printf "\nTesting downloader selection...\n";

     

       224
       224
       -
         

     

       225
       225
       -
         (* Test of_string function *)

     

       226
       226
       -
         Printf.printf "Testing of_string selection...\n";

     

       227
       227
       -
         

     

       228
       228
       -
         (match Downloader.Downloaders.of_string "wget" with

     

       229
       229
       -
         | Some (module D) -> Printf.printf "Selected wget via of_string\n"

     

       230
       230
       -
         | None -> Printf.printf "Failed to select wget\n");

     

       231
       231
       -
         

     

       232
       232
       -
         (match Downloader.Downloaders.of_string "curl" with

     

       233
       233
       -
         | Some (module D) -> Printf.printf "Selected curl via of_string\n"

     

       234
       234
       -
         | None -> Printf.printf "Failed to select curl\n");

     

       235
       235
       -
         

     

       236
       236
       -
         (match Downloader.Downloaders.of_string "invalid" with

     

       237
       237
       -
         | Some (module D) -> Printf.printf "ERROR: Should not select invalid downloader\n"

     

       238
       238
       -
         | None -> Printf.printf "Correctly rejected invalid downloader\n");

     

       239
       239
       -
         

     

       240
       240
       -
         (* Test CLI integration *)

     

       241
       241
       -
         Printf.printf "Testing CLI downloader creation...\n";

     

       242
       242
       -
         

     

       243
       243
       -
         Eio_main.run @@ fun env ->

     

       244
       244
       -
         Eio.Switch.run @@ fun _sw ->

     

       245
       245
       -
           try

     

       246
       246
       -
             let (module D1) = Downloader.Cli.create_downloader ~env `Wget in

     

       247
       247
       -
             Printf.printf "CLI created wget downloader\n";

     

       248
       248
       -
             

     

       249
       249
       -
             let (module D2) = Downloader.Cli.create_downloader ~env `Curl in

     

       250
       250
       -
             Printf.printf "CLI created curl downloader\n";

     

       251
       251
       -
             

     

       252
       252
       -
             let (module D3) = Downloader.Cli.create_downloader ~env `Auto in

     

       253
       253
       -
             Printf.printf "CLI created auto downloader\n";

     

       254
       254
       -
           with

     

       255
       255
       -
           | exn ->

     

       256
       256
       -
               Printf.printf "Exception during CLI test: %s\n" (Printexc.to_string exn)

     

       257
       257
       -
               

     

       258
       258
       -
       let run_all_tests () =

     

       259
       259
       -
         Printf.printf "Comprehensive Downloader Tests\n";

     

       260
       260
       -
         Printf.printf "==============================\n";

     

       261
       261
       -
         

     

       262
       262
       -
         test_downloader_detection ();

     

       263
       263
       -
         test_authentication ();

     

       264
       264
       -
         test_resume_functionality ();

     

       265
       265
       -
         test_error_handling ();

     

       266
       266
       -
         test_hash_verification ();

     

       267
       267
       -
         test_downloader_selection ();

     

       268
       268
       -
         test_wget_download ();

     

       269
       269
       -
         

     

       270
       270
       -
         Printf.printf "\nAll comprehensive tests completed!\n"

     

       271
       271
       -
       

     

       272
       272
       -
       let () = run_all_tests ()

-159

stack/toru/test/test_downloader_comprehensive.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let test_curl_functionality () =

     

       4
       4
       -
         Printf.printf "Testing curl downloader functionality...\n";

     

       5
       5
       -
         

     

       6
       6
       -
         Eio_main.run @@ fun env ->

     

       7
       7
       -
         Eio.Switch.run @@ fun sw ->

     

       8
       8
       -
           try

     

       9
       9
       -
             let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.curl () in

     

       10
       10
       -
             

     

       11
       11
       -
             (* Test basic downloader creation *)

     

       12
       12
       -
             let downloader = D.create ~sw ~env () in

     

       13
       13
       -
             Printf.printf "Created curl downloader: %s\n" (D.name downloader);

     

       14
       14
       -
             Printf.printf "Supports resume: %b\n" (D.supports_resume downloader);

     

       15
       15
       -
             

     

       16
       16
       -
             (* Test with authentication *)

     

       17
       17
       -
             let auth = Downloader.Config.{ username = Some "testuser"; password = Some "testpass" } in

     

       18
       18
       -
             let _auth_downloader = D.create ~sw ~env ~auth () in

     

       19
       19
       -
             Printf.printf "Created authenticated curl downloader\n";

     

       20
       20
       -
             

     

       21
       21
       -
             (* Test successful download *)

     

       22
       22
       -
             let test_url = "https://httpbin.org/robots.txt" in

     

       23
       23
       -
             let dest_path = Eio.Path.(env#fs / "test_download.txt") in

     

       24
       24
       -
             

     

       25
       25
       -
             Printf.printf "Testing basic download...\n";

     

       26
       26
       -
             (match D.download downloader ~url:test_url ~dest:dest_path () with

     

       27
       27
       -
             | Ok () ->

     

       28
       28
       -
                 Printf.printf "Basic download successful\n";

     

       29
       29
       -
                 let file_exists = try

     

       30
       30
       -
                   let _stat = Eio.Path.stat ~follow:false dest_path in true

     

       31
       31
       -
                 with _ -> false in

     

       32
       32
       -
                 if file_exists then (

     

       33
       33
       -
                   Printf.printf "File verified and exists\n";

     

       34
       34
       -
                   Eio.Path.unlink dest_path;

     

       35
       35
       -
                 )

     

       36
       36
       -
             | Error msg ->

     

       37
       37
       -
                 Printf.printf "Basic download failed: %s\n" msg);

     

       38
       38
       -
             

     

       39
       39
       -
             (* Test resume functionality *)

     

       40
       40
       -
             Printf.printf "Testing resume functionality...\n";

     

       41
       41
       -
             (match D.download downloader ~url:test_url ~dest:dest_path ~resume:true () with

     

       42
       42
       -
             | Ok () ->

     

       43
       43
       -
                 Printf.printf "Resume download successful\n";

     

       44
       44
       -
                 let file_exists = try

     

       45
       45
       -
                   let _stat = Eio.Path.stat ~follow:false dest_path in true

     

       46
       46
       -
                 with _ -> false in

     

       47
       47
       -
                 if file_exists then (

     

       48
       48
       -
                   Printf.printf "Resume test file verified\n";

     

       49
       49
       -
                   Eio.Path.unlink dest_path;

     

       50
       50
       -
                 )

     

       51
       51
       -
             | Error msg ->

     

       52
       52
       -
                 Printf.printf "Resume download failed: %s\n" msg);

     

       53
       53
       -
             

     

       54
       54
       -
             (* Test hash verification *)

     

       55
       55
       -
             Printf.printf "Testing hash verification...\n";

     

       56
       56
       -
             (match D.download downloader ~url:test_url ~dest:dest_path () with

     

       57
       57
       -
             | Ok () ->

     

       58
       58
       -
                 let computed_hash = Hash.compute Hash.SHA256 dest_path in

     

       59
       59
       -
                 Printf.printf "Computed hash: %s\n" (Hash.to_string computed_hash);

     

       60
       60
       -
                 

     

       61
       61
       -
                 (* Clean up and test with computed hash *)

     

       62
       62
       -
                 Eio.Path.unlink dest_path;

     

       63
       63
       -
                 

     

       64
       64
       -
                 (* Download again with correct hash *)

     

       65
       65
       -
                 (match D.download downloader ~url:test_url ~dest:dest_path ~hash:computed_hash () with

     

       66
       66
       -
                 | Ok () ->

     

       67
       67
       -
                     Printf.printf "Hash verification successful\n";

     

       68
       68
       -
                     Eio.Path.unlink dest_path;

     

       69
       69
       -
                 | Error msg ->

     

       70
       70
       -
                     Printf.printf "Hash verification failed: %s\n" msg);

     

       71
       71
       -
                 

     

       72
       72
       -
                 (* Test with wrong hash *)

     

       73
       73
       -
                 let wrong_hash = Hash.create Hash.SHA256 "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" in

     

       74
       74
       -
                 (match D.download downloader ~url:test_url ~dest:dest_path ~hash:wrong_hash () with

     

       75
       75
       -
                 | Ok () ->

     

       76
       76
       -
                     Printf.printf "ERROR: Wrong hash should have failed!\n";

     

       77
       77
       -
                     Eio.Path.unlink dest_path;

     

       78
       78
       -
                 | Error msg ->

     

       79
       79
       -
                     Printf.printf "Wrong hash correctly rejected: %s\n" msg);

     

       80
       80
       -
                     

     

       81
       81
       -
             | Error msg ->

     

       82
       82
       -
                 Printf.printf "Initial hash test download failed: %s\n" msg);

     

       83
       83
       -
             

     

       84
       84
       -
             (* Test error handling *)

     

       85
       85
       -
             Printf.printf "Testing error handling...\n";

     

       86
       86
       -
             let invalid_url = "https://invalid-domain-12345.com/file.txt" in

     

       87
       87
       -
             (match D.download downloader ~url:invalid_url ~dest:dest_path () with

     

       88
       88
       -
             | Ok () ->

     

       89
       89
       -
                 Printf.printf "WARNING: Invalid URL unexpectedly succeeded\n"

     

       90
       90
       -
             | Error msg ->

     

       91
       91
       -
                 Printf.printf "Invalid URL correctly handled: %s\n" msg);

     

       92
       92
       -
                 

     

       93
       93
       -
           with

     

       94
       94
       -
           | exn ->

     

       95
       95
       -
               Printf.printf "Exception during curl test: %s\n" (Printexc.to_string exn)

     

       96
       96
       -
       

     

       97
       97
       -
       let test_downloader_selection () =

     

       98
       98
       -
         Printf.printf "\nTesting downloader selection and detection...\n";

     

       99
       99
       -
         

     

       100
       100
       -
         Eio_main.run @@ fun env ->

     

       101
       101
       -
         Eio.Switch.run @@ fun _sw ->

     

       102
       102
       -
           (* Test detection *)

     

       103
       103
       -
           let available = Downloader.Downloaders.detect_available ~env in

     

       104
       104
       -
           Printf.printf "Available downloaders:\n";

     

       105
       105
       -
           List.iter (fun (name, _) ->

     

       106
       106
       -
             Printf.printf "  - %s\n" name

     

       107
       107
       -
           ) available;

     

       108
       108
       -
           

     

       109
       109
       -
           (* Test default selection *)

     

       110
       110
       -
           if List.length available > 0 then (

     

       111
       111
       -
             let (module Default) = Downloader.Downloaders.create_default ~env in

     

       112
       112
       -
             Printf.printf "Default downloader selected\n";

     

       113
       113
       -
           );

     

       114
       114
       -
           

     

       115
       115
       -
           (* Test string-based selection *)

     

       116
       116
       -
           (match Downloader.Downloaders.of_string "curl" with

     

       117
       117
       -
           | Some (module D) -> Printf.printf "String-based curl selection successful\n"

     

       118
       118
       -
           | None -> Printf.printf "String-based curl selection failed\n");

     

       119
       119
       -
           

     

       120
       120
       -
           (* Test CLI integration *)

     

       121
       121
       -
           let (module CLI_Auto) = Downloader.Cli.create_downloader ~env `Auto in

     

       122
       122
       -
           Printf.printf "CLI auto-selection successful\n"

     

       123
       123
       -
       

     

       124
       124
       -
       let test_authentication_formats () =

     

       125
       125
       -
         Printf.printf "\nTesting authentication formats...\n";

     

       126
       126
       -
         

     

       127
       127
       -
         Eio_main.run @@ fun env ->

     

       128
       128
       -
         Eio.Switch.run @@ fun sw ->

     

       129
       129
       -
           let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.curl () in

     

       130
       130
       -
           

     

       131
       131
       -
           (* Test with username and password *)

     

       132
       132
       -
           let auth1 = Downloader.Config.{ username = Some "user"; password = Some "pass" } in

     

       133
       133
       -
           let downloader1 = D.create ~sw ~env ~auth:auth1 () in

     

       134
       134
       -
           Printf.printf "Created downloader with username and password\n";

     

       135
       135
       -
           

     

       136
       136
       -
           (* Test with username only *)

     

       137
       137
       -
           let auth2 = Downloader.Config.{ username = Some "user"; password = None } in

     

       138
       138
       -
           let _downloader2 = D.create ~sw ~env ~auth:auth2 () in

     

       139
       139
       -
           Printf.printf "Created downloader with username only\n";

     

       140
       140
       -
           

     

       141
       141
       -
           (* Test with no authentication *)

     

       142
       142
       -
           let _downloader3 = D.create ~sw ~env () in

     

       143
       143
       -
           Printf.printf "Created downloader with no authentication\n";

     

       144
       144
       -
           

     

       145
       145
       -
           (* Verify all have same basic properties *)

     

       146
       146
       -
           Printf.printf "All downloaders report name: %s\n" (D.name downloader1);

     

       147
       147
       -
           Printf.printf "All support resume: %b\n" (D.supports_resume downloader1)

     

       148
       148
       -
       

     

       149
       149
       -
       let run_comprehensive_tests () =

     

       150
       150
       -
         Printf.printf "Comprehensive Curl Downloader Tests\n";

     

       151
       151
       -
         Printf.printf "===================================\n";

     

       152
       152
       -
         

     

       153
       153
       -
         test_curl_functionality ();

     

       154
       154
       -
         test_downloader_selection ();

     

       155
       155
       -
         test_authentication_formats ();

     

       156
       156
       -
         

     

       157
       157
       -
         Printf.printf "\nAll comprehensive tests completed!\n"

     

       158
       158
       -
       

     

       159
       159
       -
       let () = run_comprehensive_tests ()

-134

stack/toru/test/test_hash.ml

···

       1
       1
       -
       open Toru.Hash

     

       2
       2
       -
       

     

       3
       3
       -
       let test_hash_creation () =

     

       4
       4
       -
         let hash = create SHA256 "abc123" in

     

       5
       5
       -
         assert (algorithm hash = SHA256);

     

       6
       6
       -
         assert (value hash = "abc123");

     

       7
       7
       -
         Printf.printf "✓ Hash creation test passed\n"

     

       8
       8
       -
       

     

       9
       9
       -
       let test_algorithm_conversion () =

     

       10
       10
       -
         assert (algorithm_to_string SHA256 = "sha256");

     

       11
       11
       -
         assert (algorithm_to_string SHA1 = "sha1");

     

       12
       12
       -
         assert (algorithm_to_string MD5 = "md5");

     

       13
       13
       -
         

     

       14
       14
       -
         assert (algorithm_of_string "sha256" = Some SHA256);

     

       15
       15
       -
         assert (algorithm_of_string "sha1" = Some SHA1);

     

       16
       16
       -
         assert (algorithm_of_string "md5" = Some MD5);

     

       17
       17
       -
         assert (algorithm_of_string "invalid" = None);

     

       18
       18
       -
         Printf.printf "✓ Algorithm conversion test passed\n"

     

       19
       19
       -
       

     

       20
       20
       -
       let test_prefixed_parsing () =

     

       21
       21
       -
         let test_cases = [

     

       22
       22
       -
           ("sha256:abc123", Some (SHA256, "abc123"));

     

       23
       23
       -
           ("sha1:def456", Some (SHA1, "def456"));

     

       24
       24
       -
           ("md5:789xyz", Some (MD5, "789xyz"));

     

       25
       25
       -
           ("invalid:abc", None);

     

       26
       26
       -
           ("nocolon", None);

     

       27
       27
       -
         ] in

     

       28
       28
       -
         List.iter (fun (input, expected) ->

     

       29
       29
       -
           assert (parse_prefixed input = expected)

     

       30
       30
       -
         ) test_cases;

     

       31
       31
       -
         Printf.printf "✓ Prefixed parsing test passed\n"

     

       32
       32
       -
       

     

       33
       33
       -
       let test_hash_parsing () =

     

       34
       34
       -
         (* Test prefixed format *)

     

       35
       35
       -
         let hash1 = of_string "sha1:abc123def456789" in

     

       36
       36
       -
         assert (algorithm hash1 = SHA1);

     

       37
       37
       -
         assert (value hash1 = "abc123def456789");

     

       38
       38
       -
         

     

       39
       39
       -
         (* Test non-prefixed SHA256 (64 chars) *)

     

       40
       40
       -
         let sha256_hash = String.make 64 'a' in

     

       41
       41
       -
         let hash2 = of_string sha256_hash in

     

       42
       42
       -
         assert (algorithm hash2 = SHA256);

     

       43
       43
       -
         assert (value hash2 = sha256_hash);

     

       44
       44
       -
         

     

       45
       45
       -
         (* Test non-prefixed SHA1 (40 chars) *)

     

       46
       46
       -
         let sha1_hash = String.make 40 'b' in

     

       47
       47
       -
         let hash3 = of_string sha1_hash in

     

       48
       48
       -
         assert (algorithm hash3 = SHA1);

     

       49
       49
       -
         assert (value hash3 = sha1_hash);

     

       50
       50
       -
         

     

       51
       51
       -
         (* Test non-prefixed MD5 (32 chars) *)

     

       52
       52
       -
         let md5_hash = String.make 32 'c' in

     

       53
       53
       -
         let hash4 = of_string md5_hash in

     

       54
       54
       -
         assert (algorithm hash4 = MD5);

     

       55
       55
       -
         assert (value hash4 = md5_hash);

     

       56
       56
       -
         

     

       57
       57
       -
         Printf.printf "✓ Hash parsing test passed\n"

     

       58
       58
       -
       

     

       59
       59
       -
       let test_hash_formatting () =

     

       60
       60
       -
         let hash = create SHA1 "abc123" in

     

       61
       61
       -
         let formatted = to_string hash in

     

       62
       62
       -
         assert (formatted = "sha1:abc123");

     

       63
       63
       -
         

     

       64
       64
       -
         let prefixed = format_prefixed hash in

     

       65
       65
       -
         assert (prefixed = "sha1:abc123");

     

       66
       66
       -
         Printf.printf "✓ Hash formatting test passed\n"

     

       67
       67
       -
       

     

       68
       68
       -
       let test_hash_equality () =

     

       69
       69
       -
         let hash1 = create SHA256 "abc123" in

     

       70
       70
       -
         let hash2 = create SHA256 "abc123" in

     

       71
       71
       -
         let hash3 = create SHA1 "abc123" in

     

       72
       72
       -
         let hash4 = create SHA256 "def456" in

     

       73
       73
       -
         

     

       74
       74
       -
         assert (equal hash1 hash2);

     

       75
       75
       -
         assert (not (equal hash1 hash3)); (* Different algorithm *)

     

       76
       76
       -
         assert (not (equal hash1 hash4)); (* Different value *)

     

       77
       77
       -
         Printf.printf "✓ Hash equality test passed\n"

     

       78
       78
       -
       

     

       79
       79
       -
       (* Test with actual file - create a temporary file *)

     

       80
       80
       -
       let test_hash_computation () =

     

       81
       81
       -
         let test_content = "Hello, World!" in

     

       82
       82
       -
         let test_file = "test_hash_file.txt" in

     

       83
       83
       -
         

     

       84
       84
       -
         (* Write test content to file *)

     

       85
       85
       -
         let oc = open_out test_file in

     

       86
       86
       -
         output_string oc test_content;

     

       87
       87
       -
         close_out oc;

     

       88
       88
       -
         

     

       89
       89
       -
         try

     

       90
       90
       -
           (* Compute hashes using Eio *)

     

       91
       91
       -
           Eio_main.run @@ fun env ->

     

       92
       92
       -
           let fs = env#fs in

     

       93
       93
       -
           let file_path = Eio.Path.(fs / test_file) in

     

       94
       94
       -
           

     

       95
       95
       -
           let sha256_hash = compute SHA256 file_path in

     

       96
       96
       -
           let sha1_hash = compute SHA1 file_path in

     

       97
       97
       -
           let md5_hash = compute MD5 file_path in

     

       98
       98
       -
           

     

       99
       99
       -
           (* Verify expected hash lengths *)

     

       100
       100
       -
           assert (String.length (value sha256_hash) = 64);

     

       101
       101
       -
           assert (String.length (value sha1_hash) = 40);

     

       102
       102
       -
           assert (String.length (value md5_hash) = 32);

     

       103
       103
       -
           

     

       104
       104
       -
           (* Test verification *)

     

       105
       105
       -
           assert (verify file_path sha256_hash);

     

       106
       106
       -
           assert (verify file_path sha1_hash);

     

       107
       107
       -
           assert (verify file_path md5_hash);

     

       108
       108
       -
           

     

       109
       109
       -
           (* Test verification failure *)

     

       110
       110
       -
           let wrong_hash = create SHA256 (String.make 64 '0') in

     

       111
       111
       -
           assert (not (verify file_path wrong_hash));

     

       112
       112
       -
           

     

       113
       113
       -
           Printf.printf "✓ Hash computation and verification test passed\n";

     

       114
       114
       -
           

     

       115
       115
       -
           (* Clean up *)

     

       116
       116
       -
           Sys.remove test_file

     

       117
       117
       -
         with

     

       118
       118
       -
         | exn ->

     

       119
       119
       -
             (* Clean up on error *)

     

       120
       120
       -
             if Sys.file_exists test_file then Sys.remove test_file;

     

       121
       121
       -
             raise exn

     

       122
       122
       -
       

     

       123
       123
       -
       let run_tests () =

     

       124
       124
       -
         Printf.printf "Running Hash module tests...\n\n";

     

       125
       125
       -
         test_hash_creation ();

     

       126
       126
       -
         test_algorithm_conversion ();

     

       127
       127
       -
         test_prefixed_parsing ();

     

       128
       128
       -
         test_hash_parsing ();

     

       129
       129
       -
         test_hash_formatting ();

     

       130
       130
       -
         test_hash_equality ();

     

       131
       131
       -
         test_hash_computation ();

     

       132
       132
       -
         Printf.printf "\n✅ All Hash module tests passed!\n"

     

       133
       133
       -
       

     

       134
       134
       -
       let () = run_tests ()

-28

stack/toru/test/test_hash_manual.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let test_hash_computation () =

     

       4
       4
       -
         Printf.printf "Testing Hash computation...\n";

     

       5
       5
       -
         

     

       6
       6
       -
         (* Create a test file *)

     

       7
       7
       -
         let test_content = "Hello, World!" in

     

       8
       8
       -
         let test_file = "test_hash_file.txt" in

     

       9
       9
       -
         

     

       10
       10
       -
         (* Write test file *)

     

       11
       11
       -
         let oc = open_out test_file in

     

       12
       12
       -
         output_string oc test_content;

     

       13
       13
       -
         close_out oc;

     

       14
       14
       -
         

     

       15
       15
       -
         (* Compute hash via external command for verification *)

     

       16
       16
       -
         let cmd = Printf.sprintf "echo -n '%s' | sha256sum | cut -d' ' -f1" test_content in

     

       17
       17
       -
         let expected_output = Unix.open_process_in cmd in

     

       18
       18
       -
         let expected_hash = input_line expected_output in

     

       19
       19
       -
         let _ = Unix.close_process_in expected_output in

     

       20
       20
       -
         

     

       21
       21
       -
         Printf.printf "Expected SHA256: %s\n" expected_hash;

     

       22
       22
       -
         

     

       23
       23
       -
         (* Clean up *)

     

       24
       24
       -
         Sys.remove test_file;

     

       25
       25
       -
         

     

       26
       26
       -
         Printf.printf "Hash computation test completed\n"

     

       27
       27
       -
       

     

       28
       28
       -
       let () = test_hash_computation ()

-367

stack/toru/test/test_make_registry.ml

···

       1
       1
       -
       (** Comprehensive test suite for Make_registry module *)

     

       2
       2
       -
       

     

       3
       3
       -
       open Toru.Make_registry

     

       4
       4
       -
       open Eio.Std

     

       5
       5
       -
       

     

       6
       6
       -
       let test_dir = "/tmp/toru_make_registry_test"

     

       7
       7
       -
       

     

       8
       8
       -
       (* Test utilities *)

     

       9
       9
       -
       let setup_test_directory () =

     

       10
       10
       -
         let cmd = Printf.sprintf "rm -rf %s && mkdir -p %s" test_dir test_dir in

     

       11
       11
       -
         let _ = Sys.command cmd in

     

       12
       12
       -
         

     

       13
       13
       -
         (* Create test files *)

     

       14
       14
       -
         let create_file path content =

     

       15
       15
       -
           let dir = Filename.dirname path in

     

       16
       16
       -
           let _ = Sys.command (Printf.sprintf "mkdir -p %s" dir) in

     

       17
       17
       -
           let oc = open_out path in

     

       18
       18
       -
           output_string oc content;

     

       19
       19
       -
           close_out oc

     

       20
       20
       -
         in

     

       21
       21
       -
         

     

       22
       22
       -
         (* Regular files *)

     

       23
       23
       -
         create_file (test_dir ^ "/file1.txt") "Hello world";

     

       24
       24
       -
         create_file (test_dir ^ "/file2.csv") "col1,col2\n1,2\n3,4";

     

       25
       25
       -
         create_file (test_dir ^ "/data.json") "{\"key\": \"value\"}";

     

       26
       26
       -
         

     

       27
       27
       -
         (* Subdirectory with files *)

     

       28
       28
       -
         create_file (test_dir ^ "/subdir/nested1.txt") "Nested content 1";

     

       29
       29
       -
         create_file (test_dir ^ "/subdir/nested2.md") "# Markdown content";

     

       30
       30
       -
         create_file (test_dir ^ "/subdir/deep/nested3.txt") "Deep nested content";

     

       31
       31
       -
         

     

       32
       32
       -
         (* Hidden files *)

     

       33
       33
       -
         create_file (test_dir ^ "/.hidden") "Hidden file content";

     

       34
       34
       -
         create_file (test_dir ^ "/subdir/.config") "Config content";

     

       35
       35
       -
         

     

       36
       36
       -
         (* Files to exclude *)

     

       37
       37
       -
         create_file (test_dir ^ "/temp.tmp") "Temporary file";

     

       38
       38
       -
         create_file (test_dir ^ "/debug.log") "Log content";

     

       39
       39
       -
         create_file (test_dir ^ "/subdir/another.tmp") "Another temp";

     

       40
       40
       -
         

     

       41
       41
       -
         Printf.printf "Test directory setup complete: %s\n" test_dir

     

       42
       42
       -
       

     

       43
       43
       -
       let cleanup_test_directory () =

     

       44
       44
       -
         let cmd = Printf.sprintf "rm -rf %s" test_dir in

     

       45
       45
       -
         let _ = Sys.command cmd in

     

       46
       46
       -
         Printf.printf "Test directory cleaned up\n"

     

       47
       47
       -
       

     

       48
       48
       -
       (* Test 1: Basic directory scanning *)

     

       49
       49
       -
       let test_basic_scan () =

     

       50
       50
       -
         Printf.printf "\n=== Test 1: Basic Directory Scanning ===\n";

     

       51
       51
       -
         

     

       52
       52
       -
         Eio_main.run @@ fun env ->

     

       53
       53
       -
         Eio.Switch.run @@ fun sw ->

     

       54
       54
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       55
       55
       -
           let options = { default_options with recursive = false } in

     

       56
       56
       -
           

     

       57
       57
       -
           let registry = scan_directory ~sw ~env ~options dir_path in

     

       58
       58
       -
           let entries = Toru.Registry.entries registry in

     

       59
       59
       -
           

     

       60
       60
       -
           Printf.printf "Found %d files in root directory\n" (List.length entries);

     

       61
       61
       -
           

     

       62
       62
       -
           (* Should find non-hidden files in root only *)

     

       63
       63
       -
           let expected_files = ["file1.txt"; "file2.csv"; "data.json"; "temp.tmp"; "debug.log"] in

     

       64
       64
       -
           List.iter (fun filename ->

     

       65
       65
       -
             match Toru.Registry.find filename registry with

     

       66
       66
       -
             | Some entry -> 

     

       67
       67
       -
                 Printf.printf "✓ Found: %s (hash: %s)\n" filename 

     

       68
       68
       -
                   (Toru.Hash.value (Toru.Registry.hash entry))

     

       69
       69
       -
             | None -> Printf.printf "✗ Missing: %s\n" filename

     

       70
       70
       -
           ) expected_files;

     

       71
       71
       -
           

     

       72
       72
       -
           assert (List.length entries = 5)

     

       73
       73
       -
       

     

       74
       74
       -
       (* Test 2: Recursive scanning *)

     

       75
       75
       -
       let test_recursive_scan () =

     

       76
       76
       -
         Printf.printf "\n=== Test 2: Recursive Directory Scanning ===\n";

     

       77
       77
       -
         

     

       78
       78
       -
         Eio_main.run @@ fun env ->

     

       79
       79
       -
         Eio.Switch.run @@ fun sw ->

     

       80
       80
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       81
       81
       -
           let options = { default_options with recursive = true } in

     

       82
       82
       -
           

     

       83
       83
       -
           let registry = scan_directory ~sw ~env ~options dir_path in

     

       84
       84
       -
           let entries = Toru.Registry.entries registry in

     

       85
       85
       -
           

     

       86
       86
       -
           Printf.printf "Found %d files recursively\n" (List.length entries);

     

       87
       87
       -
           

     

       88
       88
       -
           (* Should find files in subdirectories too *)

     

       89
       89
       -
           let expected_files = [

     

       90
       90
       -
             "file1.txt"; "file2.csv"; "data.json"; "temp.tmp"; "debug.log";

     

       91
       91
       -
             "subdir/nested1.txt"; "subdir/nested2.md"; "subdir/another.tmp";

     

       92
       92
       -
             "subdir/deep/nested3.txt"

     

       93
       93
       -
           ] in

     

       94
       94
       -
           

     

       95
       95
       -
           List.iter (fun filename ->

     

       96
       96
       -
             match Toru.Registry.find filename registry with

     

       97
       97
       -
             | Some entry -> 

     

       98
       98
       -
                 Printf.printf "✓ Found: %s\n" filename

     

       99
       99
       -
             | None -> Printf.printf "✗ Missing: %s\n" filename

     

       100
       100
       -
           ) expected_files;

     

       101
       101
       -
           

     

       102
       102
       -
           assert (List.length entries = 9)

     

       103
       103
       -
       

     

       104
       104
       -
       (* Test 3: Exclude patterns *)

     

       105
       105
       -
       let test_exclude_patterns () =

     

       106
       106
       -
         Printf.printf "\n=== Test 3: Exclude Patterns ===\n";

     

       107
       107
       -
         

     

       108
       108
       -
         Eio_main.run @@ fun env ->

     

       109
       109
       -
         Eio.Switch.run @@ fun sw ->

     

       110
       110
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       111
       111
       -
           let options = { 

     

       112
       112
       -
             default_options with 

     

       113
       113
       -
             recursive = true;

     

       114
       114
       -
             exclude_patterns = ["*.tmp"; "*.log"]

     

       115
       115
       -
           } in

     

       116
       116
       -
           

     

       117
       117
       -
           let registry = scan_directory ~sw ~env ~options dir_path in

     

       118
       118
       -
           let entries = Toru.Registry.entries registry in

     

       119
       119
       -
           

     

       120
       120
       -
           Printf.printf "Found %d files with exclusions\n" (List.length entries);

     

       121
       121
       -
           

     

       122
       122
       -
           (* Should exclude .tmp and .log files *)

     

       123
       123
       -
           let included_files = [

     

       124
       124
       -
             "file1.txt"; "file2.csv"; "data.json";

     

       125
       125
       -
             "subdir/nested1.txt"; "subdir/nested2.md"; "subdir/deep/nested3.txt"

     

       126
       126
       -
           ] in

     

       127
       127
       -
           let excluded_files = ["temp.tmp"; "debug.log"; "subdir/another.tmp"] in

     

       128
       128
       -
           

     

       129
       129
       -
           List.iter (fun filename ->

     

       130
       130
       -
             match Toru.Registry.find filename registry with

     

       131
       131
       -
             | Some _ -> Printf.printf "✓ Included: %s\n" filename

     

       132
       132
       -
             | None -> Printf.printf "✗ Should be included: %s\n" filename

     

       133
       133
       -
           ) included_files;

     

       134
       134
       -
           

     

       135
       135
       -
           List.iter (fun filename ->

     

       136
       136
       -
             match Toru.Registry.find filename registry with

     

       137
       137
       -
             | Some _ -> Printf.printf "✗ Should be excluded: %s\n" filename

     

       138
       138
       -
             | None -> Printf.printf "✓ Excluded: %s\n" filename

     

       139
       139
       -
           ) excluded_files;

     

       140
       140
       -
           

     

       141
       141
       -
           assert (List.length entries = 6)

     

       142
       142
       -
       

     

       143
       143
       -
       (* Test 4: Hidden files *)

     

       144
       144
       -
       let test_hidden_files () =

     

       145
       145
       -
         Printf.printf "\n=== Test 4: Hidden Files ===\n";

     

       146
       146
       -
         

     

       147
       147
       -
         Eio_main.run @@ fun env ->

     

       148
       148
       -
         Eio.Switch.run @@ fun sw ->

     

       149
       149
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       150
       150
       -
           

     

       151
       151
       -
           (* Test without hidden files *)

     

       152
       152
       -
           let options_no_hidden = { default_options with recursive = true; include_hidden = false } in

     

       153
       153
       -
           let registry_no_hidden = scan_directory ~sw ~env ~options:options_no_hidden dir_path in

     

       154
       154
       -
           

     

       155
       155
       -
           (* Test with hidden files *)

     

       156
       156
       -
           let options_with_hidden = { default_options with recursive = true; include_hidden = true } in

     

       157
       157
       -
           let registry_with_hidden = scan_directory ~sw ~env ~options:options_with_hidden dir_path in

     

       158
       158
       -
           

     

       159
       159
       -
           let count_no_hidden = List.length (Toru.Registry.entries registry_no_hidden) in

     

       160
       160
       -
           let count_with_hidden = List.length (Toru.Registry.entries registry_with_hidden) in

     

       161
       161
       -
           

     

       162
       162
       -
           Printf.printf "Files without hidden: %d\n" count_no_hidden;

     

       163
       163
       -
           Printf.printf "Files with hidden: %d\n" count_with_hidden;

     

       164
       164
       -
           

     

       165
       165
       -
           (* Should have more files when including hidden *)

     

       166
       166
       -
           assert (count_with_hidden > count_no_hidden);

     

       167
       167
       -
           

     

       168
       168
       -
           (* Check specific hidden files *)

     

       169
       169
       -
           let hidden_files = [".hidden"; "subdir/.config"] in

     

       170
       170
       -
           List.iter (fun filename ->

     

       171
       171
       -
             match Toru.Registry.find filename registry_with_hidden with

     

       172
       172
       -
             | Some _ -> Printf.printf "✓ Found hidden: %s\n" filename

     

       173
       173
       -
             | None -> Printf.printf "✗ Missing hidden: %s\n" filename

     

       174
       174
       -
           ) hidden_files

     

       175
       175
       -
       

     

       176
       176
       -
       (* Test 5: Different hash algorithms *)

     

       177
       177
       -
       let test_hash_algorithms () =

     

       178
       178
       -
         Printf.printf "\n=== Test 5: Hash Algorithms ===\n";

     

       179
       179
       -
         

     

       180
       180
       -
         Eio_main.run @@ fun env ->

     

       181
       181
       -
         Eio.Switch.run @@ fun sw ->

     

       182
       182
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       183
       183
       -
           let test_file = "file1.txt" in

     

       184
       184
       -
           

     

       185
       185
       -
           let algorithms = [

     

       186
       186
       -
             (Toru.Hash.SHA256, "SHA256");

     

       187
       187
       -
             (Toru.Hash.SHA1, "SHA1"); 

     

       188
       188
       -
             (Toru.Hash.MD5, "MD5")

     

       189
       189
       -
           ] in

     

       190
       190
       -
           

     

       191
       191
       -
           List.iter (fun (algorithm, name) ->

     

       192
       192
       -
             let options = { default_options with hash_algorithm = algorithm; recursive = false } in

     

       193
       193
       -
             let registry = scan_directory ~sw ~env ~options dir_path in

     

       194
       194
       -
             

     

       195
       195
       -
             match Toru.Registry.find test_file registry with

     

       196
       196
       -
             | Some entry ->

     

       197
       197
       -
                 let hash = Toru.Registry.hash entry in

     

       198
       198
       -
                 let algo = Toru.Hash.algorithm hash in

     

       199
       199
       -
                 let value = Toru.Hash.value hash in

     

       200
       200
       -
                 Printf.printf "✓ %s: %s (length: %d)\n" name value (String.length value);

     

       201
       201
       -
                 assert (algo = algorithm)

     

       202
       202
       -
             | None -> 

     

       203
       203
       -
                 Printf.printf "✗ Missing file for %s\n" name;

     

       204
       204
       -
                 assert false

     

       205
       205
       -
           ) algorithms

     

       206
       206
       -
       

     

       207
       207
       -
       (* Test 6: Progress callback *)

     

       208
       208
       -
       let test_progress_callback () =

     

       209
       209
       -
         Printf.printf "\n=== Test 6: Progress Callback ===\n";

     

       210
       210
       -
         

     

       211
       211
       -
         Eio_main.run @@ fun env ->

     

       212
       212
       -
         Eio.Switch.run @@ fun sw ->

     

       213
       213
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       214
       214
       -
           let options = { default_options with recursive = true } in

     

       215
       215
       -
           

     

       216
       216
       -
           let progress_calls = ref [] in

     

       217
       217
       -
           let progress_fn filename current total =

     

       218
       218
       -
             progress_calls := (filename, current, total) :: !progress_calls;

     

       219
       219
       -
             Printf.printf "Progress: %s (%d/%d)\n" filename current total

     

       220
       220
       -
           in

     

       221
       221
       -
           

     

       222
       222
       -
           let registry = scan_directory_with_progress ~sw ~env ~options ~progress dir_path in

     

       223
       223
       -
           let entries = Toru.Registry.entries registry in

     

       224
       224
       -
           

     

       225
       225
       -
           Printf.printf "Registry has %d entries\n" (List.length entries);

     

       226
       226
       -
           Printf.printf "Progress callback called %d times\n" (List.length !progress_calls);

     

       227
       227
       -
           

     

       228
       228
       -
           assert (List.length !progress_calls > 0)

     

       229
       229
       -
       

     

       230
       230
       -
       (* Test 7: Enhanced entries and JSON output *)

     

       231
       231
       -
       let test_enhanced_entries () =

     

       232
       232
       -
         Printf.printf "\n=== Test 7: Enhanced Entries and JSON Output ===\n";

     

       233
       233
       -
         

     

       234
       234
       -
         Eio_main.run @@ fun env ->

     

       235
       235
       -
         Eio.Switch.run @@ fun sw ->

     

       236
       236
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       237
       237
       -
           let options = { default_options with recursive = false } in

     

       238
       238
       -
           

     

       239
       239
       -
           let enhanced_entries = scan_directory_enhanced ~sw ~env ~options dir_path in

     

       240
       240
       -
           

     

       241
       241
       -
           Printf.printf "Enhanced entries: %d\n" (List.length enhanced_entries);

     

       242
       242
       -
           

     

       243
       243
       -
           List.iter (fun entry ->

     

       244
       244
       -
             let filename = Toru.Registry.filename entry.entry in

     

       245
       245
       -
             let metadata = entry.metadata in

     

       246
       246
       -
             Printf.printf "File: %s\n" filename;

     

       247
       247
       -
             Printf.printf "  Size: %Ld bytes\n" metadata.size;

     

       248
       248
       -
             Printf.printf "  MTime: %s\n" (Ptime.to_rfc3339 metadata.mtime);

     

       249
       249
       -
             Printf.printf "  Relative: %s\n" metadata.relative_path;

     

       250
       250
       -
             Printf.printf "  Absolute: %s\n" metadata.absolute_path;

     

       251
       251
       -
           ) enhanced_entries;

     

       252
       252
       -
           

     

       253
       253
       -
           (* Test JSON conversion *)

     

       254
       254
       -
           let json = enhanced_entries_to_json 

     

       255
       255
       -
             ~algorithm:Toru.Hash.SHA256 

     

       256
       256
       -
             ~generated:(Ptime_clock.now ()) 

     

       257
       257
       -
             enhanced_entries in

     

       258
       258
       -
           

     

       259
       259
       -
           let json_str = Yojson.Safe.pretty_to_string json in

     

       260
       260
       -
           Printf.printf "JSON output sample:\n%s\n" 

     

       261
       261
       -
             (if String.length json_str > 500 then 

     

       262
       262
       -
                String.sub json_str 0 500 ^ "..."

     

       263
       263
       -
              else json_str);

     

       264
       264
       -
           

     

       265
       265
       -
           assert (List.length enhanced_entries > 0)

     

       266
       266
       -
       

     

       267
       267
       -
       (* Test 8: File list processing *)

     

       268
       268
       -
       let test_file_list () =

     

       269
       269
       -
         Printf.printf "\n=== Test 8: File List Processing ===\n";

     

       270
       270
       -
         

     

       271
       271
       -
         Eio_main.run @@ fun env ->

     

       272
       272
       -
         Eio.Switch.run @@ fun sw ->

     

       273
       273
       -
           let file_paths = [

     

       274
       274
       -
             test_dir ^ "/file1.txt";

     

       275
       275
       -
             test_dir ^ "/file2.csv";

     

       276
       276
       -
             test_dir ^ "/subdir/nested1.txt"

     

       277
       277
       -
           ] in

     

       278
       278
       -
           

     

       279
       279
       -
           let registry = from_file_list ~sw ~env ~hash_algorithm:Toru.Hash.SHA256 file_paths in

     

       280
       280
       -
           let entries = Toru.Registry.entries registry in

     

       281
       281
       -
           

     

       282
       282
       -
           Printf.printf "Registry from file list: %d entries\n" (List.length entries);

     

       283
       283
       -
           

     

       284
       284
       -
           List.iter (fun entry ->

     

       285
       285
       -
             let filename = Toru.Registry.filename entry in

     

       286
       286
       -
             Printf.printf "✓ File: %s\n" filename

     

       287
       287
       -
           ) entries;

     

       288
       288
       -
           

     

       289
       289
       -
           assert (List.length entries = 3)

     

       290
       290
       -
       

     

       291
       291
       -
       (* Test 9: Registry update (simplified) *)

     

       292
       292
       -
       let test_registry_update () =

     

       293
       293
       -
         Printf.printf "\n=== Test 9: Registry Update ===\n";

     

       294
       294
       -
         

     

       295
       295
       -
         Eio_main.run @@ fun env ->

     

       296
       296
       -
         Eio.Switch.run @@ fun sw ->

     

       297
       297
       -
           let dir_path = env#fs |> Eio.Path.(fun fs -> fs / test_dir) in

     

       298
       298
       -
           let options = { default_options with recursive = false } in

     

       299
       299
       -
           

     

       300
       300
       -
           (* Create initial registry *)

     

       301
       301
       -
           let initial_registry = scan_directory ~sw ~env ~options dir_path in

     

       302
       302
       -
           let initial_count = List.length (Toru.Registry.entries initial_registry) in

     

       303
       303
       -
           Printf.printf "Initial registry: %d entries\n" initial_count;

     

       304
       304
       -
           

     

       305
       305
       -
           (* Simulate update (in real use case, files would have changed) *)

     

       306
       306
       -
           let updated_registry = update_registry ~sw ~env ~options initial_registry dir_path in

     

       307
       307
       -
           let updated_count = List.length (Toru.Registry.entries updated_registry) in

     

       308
       308
       -
           Printf.printf "Updated registry: %d entries\n" updated_count;

     

       309
       309
       -
           

     

       310
       310
       -
           (* For this test, counts should be the same since no files changed *)

     

       311
       311
       -
           assert (updated_count = initial_count)

     

       312
       312
       -
       

     

       313
       313
       -
       (* Test 10: Pattern matching *)

     

       314
       314
       -
       let test_pattern_matching () =

     

       315
       315
       -
         Printf.printf "\n=== Test 10: Pattern Matching ===\n";

     

       316
       316
       -
         

     

       317
       317
       -
         let test_cases = [

     

       318
       318
       -
           (["*.txt"], "file.txt", true);

     

       319
       319
       -
           (["*.txt"], "file.csv", false);

     

       320
       320
       -
           (["**/*.md"], "docs/readme.md", true);

     

       321
       321
       -
           (["**/*.md"], "readme.md", true);

     

       322
       322
       -
           (["temp/*"], "temp/file.txt", true);

     

       323
       323
       -
           (["temp/*"], "temp/sub/file.txt", false);

     

       324
       324
       -
           (["temp/**"], "temp/sub/file.txt", true);

     

       325
       325
       -
           (["*.tmp"; "*.log"], "debug.log", true);

     

       326
       326
       -
           (["*.tmp"; "*.log"], "data.csv", false);

     

       327
       327
       -
         ] in

     

       328
       328
       -
         

     

       329
       329
       -
         List.iter (fun (patterns, path, expected) ->

     

       330
       330
       -
           let result = matches_exclude_pattern patterns path in

     

       331
       331
       -
           let status = if result = expected then "✓" else "✗" in

     

       332
       332
       -
           Printf.printf "%s Pattern %s matches '%s': %b (expected %b)\n" 

     

       333
       333
       -
             status (String.concat "," patterns) path result expected;

     

       334
       334
       -
           assert (result = expected)

     

       335
       335
       -
         ) test_cases

     

       336
       336
       -
       

     

       337
       337
       -
       (* Run all tests *)

     

       338
       338
       -
       let run_all_tests () =

     

       339
       339
       -
         Printf.printf "Starting Make_registry test suite...\n";

     

       340
       340
       -
         

     

       341
       341
       -
         setup_test_directory ();

     

       342
       342
       -
         

     

       343
       343
       -
         try

     

       344
       344
       -
           test_basic_scan ();

     

       345
       345
       -
           test_recursive_scan ();

     

       346
       346
       -
           test_exclude_patterns ();

     

       347
       347
       -
           test_hidden_files ();

     

       348
       348
       -
           test_hash_algorithms ();

     

       349
       349
       -
           test_progress_callback ();

     

       350
       350
       -
           test_enhanced_entries ();

     

       351
       351
       -
           test_file_list ();

     

       352
       352
       -
           test_registry_update ();

     

       353
       353
       -
           test_pattern_matching ();

     

       354
       354
       -
           

     

       355
       355
       -
           Printf.printf "\n🎉 All tests passed!\n";

     

       356
       356
       -
           cleanup_test_directory ()

     

       357
       357
       -
         with 

     

       358
       358
       -
         | exn -> 

     

       359
       359
       -
             Printf.printf "\n❌ Test failed: %s\n" (Printexc.to_string exn);

     

       360
       360
       -
             cleanup_test_directory ();

     

       361
       361
       -
             exit 1

     

       362
       362
       -
       

     

       363
       363
       -
       let () = 

     

       364
       364
       -
         if Array.length Sys.argv > 1 && Sys.argv.(1) = "--run-tests" then

     

       365
       365
       -
           run_all_tests ()

     

       366
       366
       -
         else

     

       367
       367
       -
           Printf.printf "Use --run-tests to run the test suite\n"

-218

stack/toru/test/test_python_cross_validation.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       open Printf

     

       3
       3
       -
       

     

       4
       4
       -
       (** Cross-validation tests against Python Pooch-generated registries *)

     

       5
       5
       -
       

     

       6
       6
       -
       let test_sha256_registry () =

     

       7
       7
       -
         printf "Testing SHA256 registry compatibility...\n";

     

       8
       8
       -
         

     

       9
       9
       -
         (* Load the Python-generated registry *)

     

       10
       10
       -
         let registry_path = "test/python/test_registry_sha256.txt" in

     

       11
       11
       -
         let registry = 

     

       12
       12
       -
           if Sys.file_exists registry_path then

     

       13
       13
       -
             Eio_main.run @@ fun env ->

     

       14
       14
       -
             let fs = env#fs in

     

       15
       15
       -
             Registry.load Eio.Path.(fs / registry_path)

     

       16
       16
       -
           else

     

       17
       17
       -
             failwith ("Registry file not found: " ^ registry_path)

     

       18
       18
       -
         in

     

       19
       19
       -
         

     

       20
       20
       -
         printf "  - Loaded registry with %d entries\n" (Registry.size registry);

     

       21
       21
       -
         

     

       22
       22
       -
         (* Verify key entries exist *)

     

       23
       23
       -
         let expected_files = [

     

       24
       24
       -
           "data/simple.txt";

     

       25
       25
       -
           "data/numbers.csv";

     

       26
       26
       -
           "empty.txt";

     

       27
       27
       -
           "config.json";

     

       28
       28
       -
         ] in

     

       29
       29
       -
         

     

       30
       30
       -
         List.iter (fun filename ->

     

       31
       31
       -
           match Registry.find filename registry with

     

       32
       32
       -
           | Some entry ->

     

       33
       33
       -
               let hash = Registry.hash entry in

     

       34
       34
       -
               printf "  - ✓ Found %s: %s (%s)\n" 

     

       35
       35
       -
                      filename

     

       36
       36
       -
                      (Hash.value hash)

     

       37
       37
       -
                      (Hash.algorithm_to_string (Hash.algorithm hash))

     

       38
       38
       -
           | None ->

     

       39
       39
       -
               failwith ("Expected file not found in registry: " ^ filename)

     

       40
       40
       -
         ) expected_files;

     

       41
       41
       -
         

     

       42
       42
       -
         printf "✓ SHA256 registry test passed\n\n"

     

       43
       43
       -
       

     

       44
       44
       -
       let test_mixed_registry () =

     

       45
       45
       -
         printf "Testing mixed hash format registry...\n";

     

       46
       46
       -
         

     

       47
       47
       -
         let registry_path = "test/python/test_registry_mixed.txt" in

     

       48
       48
       -
         let registry = 

     

       49
       49
       -
           if Sys.file_exists registry_path then

     

       50
       50
       -
             Eio_main.run @@ fun env ->

     

       51
       51
       -
             let fs = env#fs in

     

       52
       52
       -
             Registry.load Eio.Path.(fs / registry_path)

     

       53
       53
       -
           else

     

       54
       54
       -
             failwith ("Registry file not found: " ^ registry_path)

     

       55
       55
       -
         in

     

       56
       56
       -
         

     

       57
       57
       -
         printf "  - Loaded mixed registry with %d entries\n" (Registry.size registry);

     

       58
       58
       -
         

     

       59
       59
       -
         (* Check that different hash algorithms are parsed correctly *)

     

       60
       60
       -
         let expected_algos = [

     

       61
       61
       -
           ("data/simple.txt", Hash.SHA256);  (* Should default to SHA256 *)

     

       62
       62
       -
           ("config.json", Hash.SHA1);       (* Should parse as SHA1 *)

     

       63
       63
       -
           ("data/numbers.csv", Hash.MD5);   (* Should parse as MD5 *)

     

       64
       64
       -
         ] in

     

       65
       65
       -
         

     

       66
       66
       -
         List.iter (fun (filename, expected_algo) ->

     

       67
       67
       -
           match Registry.find filename registry with

     

       68
       68
       -
           | Some entry ->

     

       69
       69
       -
               let hash = Registry.hash entry in

     

       70
       70
       -
               let actual_algo = Hash.algorithm hash in

     

       71
       71
       -
               if actual_algo = expected_algo then

     

       72
       72
       -
                 printf "  - ✓ %s: %s (%s)\n" 

     

       73
       73
       -
                        filename

     

       74
       74
       -
                        (Hash.value hash)

     

       75
       75
       -
                        (Hash.algorithm_to_string actual_algo)

     

       76
       76
       -
               else

     

       77
       77
       -
                 failwith (sprintf "Wrong algorithm for %s: expected %s, got %s"

     

       78
       78
       -
                                  filename

     

       79
       79
       -
                                  (Hash.algorithm_to_string expected_algo)

     

       80
       80
       -
                                  (Hash.algorithm_to_string actual_algo))

     

       81
       81
       -
           | None ->

     

       82
       82
       -
               failwith ("Expected file not found in registry: " ^ filename)

     

       83
       83
       -
         ) expected_algos;

     

       84
       84
       -
         

     

       85
       85
       -
         printf "✓ Mixed registry test passed\n\n"

     

       86
       86
       -
       

     

       87
       87
       -
       let test_hash_verification () =

     

       88
       88
       -
         printf "Testing hash verification against known data...\n";

     

       89
       89
       -
         

     

       90
       90
       -
         (* Load metadata to get expected hashes *)

     

       91
       91
       -
         let metadata_path = "test/python/test_metadata.json" in

     

       92
       92
       -
         if not (Sys.file_exists metadata_path) then

     

       93
       93
       -
           failwith ("Metadata file not found: " ^ metadata_path);

     

       94
       94
       -
         

     

       95
       95
       -
         let ic = open_in metadata_path in

     

       96
       96
       -
         let metadata_content = really_input_string ic (in_channel_length ic) in

     

       97
       97
       -
         close_in ic;

     

       98
       98
       -
         

     

       99
       99
       -
         let metadata = Yojson.Safe.from_string metadata_content in

     

       100
       100
       -
         let files = Yojson.Safe.Util.member "files" metadata |> Yojson.Safe.Util.to_list in

     

       101
       101
       -
         

     

       102
       102
       -
         printf "  - Loaded metadata for %d files\n" (List.length files);

     

       103
       103
       -
         

     

       104
       104
       -
         (* Verify a few key files exist and have correct content *)

     

       105
       105
       -
         let test_file_content = "test/python/test_data/data/simple.txt" in

     

       106
       106
       -
         if Sys.file_exists test_file_content then (

     

       107
       107
       -
           let expected_content = "Hello, World!" in

     

       108
       108
       -
           let actual_content = 

     

       109
       109
       -
             let ic = open_in test_file_content in

     

       110
       110
       -
             let content = really_input_string ic (in_channel_length ic) in

     

       111
       111
       -
             close_in ic;

     

       112
       112
       -
             content

     

       113
       113
       -
           in

     

       114
       114
       -
           if actual_content = expected_content then

     

       115
       115
       -
             printf "  - ✓ Test file content matches expected\n"

     

       116
       116
       -
           else

     

       117
       117
       -
             failwith (sprintf "Test file content mismatch: expected '%s', got '%s'"

     

       118
       118
       -
                               expected_content actual_content);

     

       119
       119
       -
           

     

       120
       120
       -
           (* Calculate hash using our implementation *)

     

       121
       121
       -
           Eio_main.run @@ fun env ->

     

       122
       122
       -
           let fs = env#fs in

     

       123
       123
       -
           let file_path = Eio.Path.(fs / test_file_content) in

     

       124
       124
       -
           

     

       125
       125
       -
           let computed_sha256 = Hash.compute SHA256 file_path in

     

       126
       126
       -
           let computed_sha1 = Hash.compute SHA1 file_path in

     

       127
       127
       -
           let computed_md5 = Hash.compute MD5 file_path in

     

       128
       128
       -
           

     

       129
       129
       -
           (* Find this file in metadata *)

     

       130
       130
       -
           let file_meta = List.find (fun file_obj ->

     

       131
       131
       -
             Yojson.Safe.Util.member "path" file_obj 

     

       132
       132
       -
             |> Yojson.Safe.Util.to_string = "data/simple.txt"

     

       133
       133
       -
           ) files in

     

       134
       134
       -
           

     

       135
       135
       -
           let expected_sha256 = Yojson.Safe.Util.member "sha256" file_meta |> Yojson.Safe.Util.to_string in

     

       136
       136
       -
           let expected_sha1 = Yojson.Safe.Util.member "sha1" file_meta |> Yojson.Safe.Util.to_string in

     

       137
       137
       -
           let expected_md5 = Yojson.Safe.Util.member "md5" file_meta |> Yojson.Safe.Util.to_string in

     

       138
       138
       -
           

     

       139
       139
       -
           (* Verify our computed hashes match Python's *)

     

       140
       140
       -
           let check_hash name computed expected =

     

       141
       141
       -
             if Hash.value computed = expected then

     

       142
       142
       -
               printf "  - ✓ %s hash matches: %s\n" name expected

     

       143
       143
       -
             else

     

       144
       144
       -
               failwith (sprintf "%s hash mismatch: computed %s, expected %s"

     

       145
       145
       -
                                name (Hash.value computed) expected)

     

       146
       146
       -
           in

     

       147
       147
       -
           

     

       148
       148
       -
           check_hash "SHA256" computed_sha256 expected_sha256;

     

       149
       149
       -
           check_hash "SHA1" computed_sha1 expected_sha1;

     

       150
       150
       -
           check_hash "MD5" computed_md5 expected_md5;

     

       151
       151
       -
         ) else

     

       152
       152
       -
           printf "  - ⚠ Test data files not found, skipping content verification\n";

     

       153
       153
       -
         

     

       154
       154
       -
         printf "✓ Hash verification test passed\n\n"

     

       155
       155
       -
       

     

       156
       156
       -
       let test_round_trip_compatibility () =

     

       157
       157
       -
         printf "Testing round-trip registry compatibility...\n";

     

       158
       158
       -
         

     

       159
       159
       -
         (* Load a Python registry and convert it back to string *)

     

       160
       160
       -
         let registry_path = "test/python/test_registry_sha256.txt" in

     

       161
       161
       -
         let original_registry = 

     

       162
       162
       -
           Eio_main.run @@ fun env ->

     

       163
       163
       -
           let fs = env#fs in

     

       164
       164
       -
           Registry.load Eio.Path.(fs / registry_path)

     

       165
       165
       -
         in

     

       166
       166
       -
         let registry_string = Registry.to_string original_registry in

     

       167
       167
       -
         

     

       168
       168
       -
         (* Parse it back *)

     

       169
       169
       -
         let reparsed_registry = Registry.of_string registry_string in

     

       170
       170
       -
         

     

       171
       171
       -
         (* Verify they're equivalent *)

     

       172
       172
       -
         if Registry.size original_registry = Registry.size reparsed_registry then

     

       173
       173
       -
           printf "  - ✓ Registry sizes match: %d entries\n" (Registry.size original_registry)

     

       174
       174
       -
         else

     

       175
       175
       -
           failwith (sprintf "Registry size mismatch: original %d, reparsed %d"

     

       176
       176
       -
                            (Registry.size original_registry)

     

       177
       177
       -
                            (Registry.size reparsed_registry));

     

       178
       178
       -
         

     

       179
       179
       -
         (* Check a few entries *)

     

       180
       180
       -
         let entries = Registry.entries original_registry in

     

       181
       181
       -
         List.iter (fun entry ->

     

       182
       182
       -
           let filename = Registry.filename entry in

     

       183
       183
       -
           let original_hash = Registry.hash entry in

     

       184
       184
       -
           match Registry.find filename reparsed_registry with

     

       185
       185
       -
           | Some reparsed_entry ->

     

       186
       186
       -
               let reparsed_hash = Registry.hash reparsed_entry in

     

       187
       187
       -
               if Hash.equal original_hash reparsed_hash then

     

       188
       188
       -
                 printf "  - ✓ %s: hashes match\n" filename

     

       189
       189
       -
               else

     

       190
       190
       -
                 failwith (sprintf "Hash mismatch for %s" filename)

     

       191
       191
       -
           | None ->

     

       192
       192
       -
               failwith (sprintf "File missing after round-trip: %s" filename)

     

       193
       193
       -
         ) (match entries with

     

       194
       194
       -
            | e1::e2::e3::_ -> [e1;e2;e3]  (* Just check first 3 *)

     

       195
       195
       -
            | all -> all);

     

       196
       196
       -
         

     

       197
       197
       -
         printf "✓ Round-trip compatibility test passed\n\n"

     

       198
       198
       -
       

     

       199
       199
       -
       let run_tests () =

     

       200
       200
       -
         printf "=== Python/Pooch Cross-Validation Tests ===\n\n";

     

       201
       201
       -
         

     

       202
       202
       -
         try

     

       203
       203
       -
           test_sha256_registry ();

     

       204
       204
       -
           test_mixed_registry ();

     

       205
       205
       -
           test_hash_verification ();

     

       206
       206
       -
           test_round_trip_compatibility ();

     

       207
       207
       -
           

     

       208
       208
       -
           printf "🎉 All cross-validation tests passed!\n";

     

       209
       209
       -
           printf "✅ Toru is fully compatible with Python Pooch registries\n\n";

     

       210
       210
       -
         with

     

       211
       211
       -
         | Failure msg ->

     

       212
       212
       -
             printf "❌ Test failed: %s\n" msg;

     

       213
       213
       -
             exit 1

     

       214
       214
       -
         | exn ->

     

       215
       215
       -
             printf "❌ Unexpected error: %s\n" (Printexc.to_string exn);

     

       216
       216
       -
             exit 1

     

       217
       217
       -
       

     

       218
       218
       -
       let () = run_tests ()

-270

stack/toru/test/test_registry.ml

···

       1
       1
       -
       (** Comprehensive tests for the Registry module *)

     

       2
       2
       -
       

     

       3
       3
       -
       open Toru

     

       4
       4
       -
       

     

       5
       5
       -
       (** Test registry content with various formats *)

     

       6
       6
       -
       let test_registry_content = {|# This is a comment

     

       7
       7
       -
       data/file1.csv sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       8
       8
       -
       data/file2.txt d1f947c87017eebc8b98d6c3944eaea813ddcfb6ceafa96db0bb70675abd4f28

     

       9
       9
       -
       

     

       10
       10
       -
       # Another comment with empty lines above

     

       11
       11
       -
       archive.zip md5:d41d8cd98f00b204e9800998ecf8427e

     

       12
       12
       -
       small.txt sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709

     

       13
       13
       -
       # Final comment

     

       14
       14
       -
       |}

     

       15
       15
       -
       

     

       16
       16
       -
       let test_basic_parsing () =

     

       17
       17
       -
         Printf.printf "=== Testing basic registry parsing ===\n";

     

       18
       18
       -
         

     

       19
       19
       -
         let registry = Registry.of_string test_registry_content in

     

       20
       20
       -
         let size = Registry.size registry in

     

       21
       21
       -
         

     

       22
       22
       -
         Printf.printf "Registry size: %d\n" size;

     

       23
       23
       -
         assert (size = 4);

     

       24
       24
       -
         

     

       25
       25
       -
         (* Test finding entries *)

     

       26
       26
       -
         let file1 = Registry.find "data/file1.csv" registry in

     

       27
       27
       -
         assert (Option.is_some file1);

     

       28
       28
       -
         

     

       29
       29
       -
         let file1_entry = Option.get file1 in

     

       30
       30
       -
         assert (Registry.filename file1_entry = "data/file1.csv");

     

       31
       31
       -
         assert (Hash.algorithm (Registry.hash file1_entry) = Hash.SHA256);

     

       32
       32
       -
         

     

       33
       33
       -
         let file2 = Registry.find "data/file2.txt" registry in

     

       34
       34
       -
         assert (Option.is_some file2);

     

       35
       35
       -
         let file2_entry = Option.get file2 in

     

       36
       36
       -
         assert (Hash.algorithm (Registry.hash file2_entry) = Hash.SHA256);

     

       37
       37
       -
         

     

       38
       38
       -
         let archive = Registry.find "archive.zip" registry in

     

       39
       39
       -
         assert (Option.is_some archive);

     

       40
       40
       -
         let archive_entry = Option.get archive in

     

       41
       41
       -
         assert (Hash.algorithm (Registry.hash archive_entry) = Hash.MD5);

     

       42
       42
       -
         

     

       43
       43
       -
         let small = Registry.find "small.txt" registry in

     

       44
       44
       -
         assert (Option.is_some small);

     

       45
       45
       -
         let small_entry = Option.get small in

     

       46
       46
       -
         assert (Hash.algorithm (Registry.hash small_entry) = Hash.SHA1);

     

       47
       47
       -
         

     

       48
       48
       -
         (* Test non-existent entry *)

     

       49
       49
       -
         assert (Registry.find "nonexistent.txt" registry = None);

     

       50
       50
       -
         assert (not (Registry.exists "nonexistent.txt" registry));

     

       51
       51
       -
         assert (Registry.exists "data/file1.csv" registry);

     

       52
       52
       -
         

     

       53
       53
       -
         Printf.printf "✓ Basic parsing tests passed\n"

     

       54
       54
       -
       

     

       55
       55
       -
       let test_round_trip_serialization () =

     

       56
       56
       -
         Printf.printf "=== Testing round-trip serialization ===\n";

     

       57
       57
       -
         

     

       58
       58
       -
         let original_registry = Registry.of_string test_registry_content in

     

       59
       59
       -
         let serialized = Registry.to_string original_registry in

     

       60
       60
       -
         let parsed_back = Registry.of_string serialized in

     

       61
       61
       -
         

     

       62
       62
       -
         assert (Registry.size original_registry = Registry.size parsed_back);

     

       63
       63
       -
         

     

       64
       64
       -
         (* Verify each entry exists in both registries *)

     

       65
       65
       -
         let entries = Registry.entries original_registry in

     

       66
       66
       -
         List.iter (fun entry ->

     

       67
       67
       -
           let filename = Registry.filename entry in

     

       68
       68
       -
           let found = Registry.find filename parsed_back in

     

       69
       69
       -
           assert (Option.is_some found);

     

       70
       70
       -
           let found_entry = Option.get found in

     

       71
       71
       -
           assert (Registry.filename found_entry = Registry.filename entry);

     

       72
       72
       -
           assert (Hash.equal (Registry.hash found_entry) (Registry.hash entry));

     

       73
       73
       -
         ) entries;

     

       74
       74
       -
         

     

       75
       75
       -
         Printf.printf "✓ Round-trip serialization tests passed\n"

     

       76
       76
       -
       

     

       77
       77
       -
       let test_entry_operations () =

     

       78
       78
       -
         Printf.printf "=== Testing entry operations ===\n";

     

       79
       79
       -
         

     

       80
       80
       -
         let empty_registry = Registry.empty in

     

       81
       81
       -
         assert (Registry.size empty_registry = 0);

     

       82
       82
       -
         assert (Registry.entries empty_registry = []);

     

       83
       83
       -
         

     

       84
       84
       -
         (* Create and add entries *)

     

       85
       85
       -
         let hash1 = Hash.create Hash.SHA256 "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" in

     

       86
       86
       -
         let entry1 = Registry.create_entry ~filename:"test1.txt" ~hash:hash1 () in

     

       87
       87
       -
         

     

       88
       88
       -
         let hash2 = Hash.create Hash.SHA1 "da39a3ee5e6b4b0d3255bfef95601890afd80709" in

     

       89
       89
       -
         let entry2 = Registry.create_entry ~filename:"test2.txt" ~hash:hash2 ~custom_url:"https://example.com/test2.txt" () in

     

       90
       90
       -
         

     

       91
       91
       -
         let registry = Registry.add entry1 empty_registry in

     

       92
       92
       -
         let registry = Registry.add entry2 registry in

     

       93
       93
       -
         

     

       94
       94
       -
         assert (Registry.size registry = 2);

     

       95
       95
       -
         assert (Registry.exists "test1.txt" registry);

     

       96
       96
       -
         assert (Registry.exists "test2.txt" registry);

     

       97
       97
       -
         

     

       98
       98
       -
         (* Test custom URL *)

     

       99
       99
       -
         let found_entry2 = Registry.find "test2.txt" registry |> Option.get in

     

       100
       100
       -
         assert (Registry.custom_url found_entry2 = Some "https://example.com/test2.txt");

     

       101
       101
       -
         

     

       102
       102
       -
         let found_entry1 = Registry.find "test1.txt" registry |> Option.get in

     

       103
       103
       -
         assert (Registry.custom_url found_entry1 = None);

     

       104
       104
       -
         

     

       105
       105
       -
         (* Test removal *)

     

       106
       106
       -
         let registry_removed = Registry.remove "test1.txt" registry in

     

       107
       107
       -
         assert (Registry.size registry_removed = 1);

     

       108
       108
       -
         assert (not (Registry.exists "test1.txt" registry_removed));

     

       109
       109
       -
         assert (Registry.exists "test2.txt" registry_removed);

     

       110
       110
       -
         

     

       111
       111
       -
         Printf.printf "✓ Entry operation tests passed\n"

     

       112
       112
       -
       

     

       113
       113
       -
       let test_hash_format_parsing () =

     

       114
       114
       -
         Printf.printf "=== Testing hash format parsing ===\n";

     

       115
       115
       -
         

     

       116
       116
       -
         let test_cases = [

     

       117
       117
       -
           "file1.txt sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";

     

       118
       118
       -
           "file2.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";

     

       119
       119
       -
           "file3.txt sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709";

     

       120
       120
       -
           "file4.txt da39a3ee5e6b4b0d3255bfef95601890afd80709";

     

       121
       121
       -
           "file5.txt md5:d41d8cd98f00b204e9800998ecf8427e";

     

       122
       122
       -
           "file6.txt d41d8cd98f00b204e9800998ecf8427e";

     

       123
       123
       -
         ] in

     

       124
       124
       -
         

     

       125
       125
       -
         List.iteri (fun i line ->

     

       126
       126
       -
           let registry = Registry.of_string line in

     

       127
       127
       -
           assert (Registry.size registry = 1);

     

       128
       128
       -
           let entries = Registry.entries registry in

     

       129
       129
       -
           let entry = List.hd entries in

     

       130
       130
       -
           let expected_filename = Printf.sprintf "file%d.txt" (i + 1) in

     

       131
       131
       -
           assert (Registry.filename entry = expected_filename);

     

       132
       132
       -
           

     

       133
       133
       -
           let hash = Registry.hash entry in

     

       134
       134
       -
           let expected_algorithm = match i with

     

       135
       135
       -
             | 0 | 1 -> Hash.SHA256

     

       136
       136
       -
             | 2 | 3 -> Hash.SHA1  

     

       137
       137
       -
             | 4 | 5 -> Hash.MD5

     

       138
       138
       -
             | _ -> assert false

     

       139
       139
       -
           in

     

       140
       140
       -
           assert (Hash.algorithm hash = expected_algorithm);

     

       141
       141
       -
         ) test_cases;

     

       142
       142
       -
         

     

       143
       143
       -
         Printf.printf "✓ Hash format parsing tests passed\n"

     

       144
       144
       -
       

     

       145
       145
       -
       let test_comment_and_empty_line_handling () =

     

       146
       146
       -
         Printf.printf "=== Testing comment and empty line handling ===\n";

     

       147
       147
       -
         

     

       148
       148
       -
         let complex_content = {|

     

       149
       149
       -
       # Header comment

     

       150
       150
       -
       # Another header comment

     

       151
       151
       -
       

     

       152
       152
       -
       data1.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

     

       153
       153
       -
       

     

       154
       154
       -
       # Mid comment

     

       155
       155
       -
       data2.txt sha256:b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78

     

       156
       156
       -
       

     

       157
       157
       -
       # End comment with spaces   

     

       158
       158
       -
          # Indented comment

     

       159
       159
       -
       data3.txt   sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709   

     

       160
       160
       -
       

     

       161
       161
       -
       # Final comment

     

       162
       162
       -
       |} in

     

       163
       163
       -
         

     

       164
       164
       -
         let registry = Registry.of_string complex_content in

     

       165
       165
       -
         assert (Registry.size registry = 3);

     

       166
       166
       -
         assert (Registry.exists "data1.txt" registry);

     

       167
       167
       -
         assert (Registry.exists "data2.txt" registry);

     

       168
       168
       -
         assert (Registry.exists "data3.txt" registry);

     

       169
       169
       -
         

     

       170
       170
       -
         Printf.printf "✓ Comment and empty line handling tests passed\n"

     

       171
       171
       -
       

     

       172
       172
       -
       let test_file_io_operations () =

     

       173
       173
       -
         Printf.printf "=== Testing file I/O operations ===\n";

     

       174
       174
       -
         

     

       175
       175
       -
         let test_dir = "/tmp/toru_registry_test" in

     

       176
       176
       -
         let test_file = test_dir ^ "/test_registry.txt" in

     

       177
       177
       -
         

     

       178
       178
       -
         (* Create test directory *)

     

       179
       179
       -
         (try Unix.mkdir test_dir 0o755 with Unix.Unix_error (Unix.EEXIST, _, _) -> ());

     

       180
       180
       -
         

     

       181
       181
       -
         (* Clean up function *)

     

       182
       182
       -
         let cleanup () =

     

       183
       183
       -
           (try Sys.remove test_file with Sys_error _ -> ());

     

       184
       184
       -
           (try Unix.rmdir test_dir with Unix.Unix_error _ -> ());

     

       185
       185
       -
         in

     

       186
       186
       -
         

     

       187
       187
       -
         Fun.protect ~finally:cleanup @@ fun () ->

     

       188
       188
       -
           (* Create a registry and save it *)

     

       189
       189
       -
           let hash1 = Hash.create Hash.SHA256 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" in

     

       190
       190
       -
           let entry1 = Registry.create_entry ~filename:"test_file1.txt" ~hash:hash1 () in

     

       191
       191
       -
           

     

       192
       192
       -
           let hash2 = Hash.create Hash.SHA1 "da39a3ee5e6b4b0d3255bfef95601890afd80709" in

     

       193
       193
       -
           let entry2 = Registry.create_entry ~filename:"test_file2.txt" ~hash:hash2 () in

     

       194
       194
       -
           

     

       195
       195
       -
           let registry = Registry.empty |> Registry.add entry1 |> Registry.add entry2 in

     

       196
       196
       -
           

     

       197
       197
       -
           (* Note: We can't use Eio.Path in this test context, so we'll test the save function

     

       198
       198
       -
              by verifying the serialized content matches what we expect *)

     

       199
       199
       -
           let serialized = Registry.to_string registry in

     

       200
       200
       -
           

     

       201
       201
       -
           (* Write to file manually *)

     

       202
       202
       -
           let oc = open_out test_file in

     

       203
       203
       -
           Fun.protect ~finally:(fun () -> close_out oc) @@ fun () ->

     

       204
       204
       -
             output_string oc serialized;

     

       205
       205
       -
             flush oc;

     

       206
       206
       -
           

     

       207
       207
       -
           (* Test that the file was written correctly *)

     

       208
       208
       -
           let ic = open_in test_file in

     

       209
       209
       -
           let content = Fun.protect ~finally:(fun () -> close_in ic) @@ fun () ->

     

       210
       210
       -
             really_input_string ic (in_channel_length ic)

     

       211
       211
       -
           in

     

       212
       212
       -
           

     

       213
       213
       -
           let loaded_registry = Registry.of_string content in

     

       214
       214
       -
           assert (Registry.size loaded_registry = 2);

     

       215
       215
       -
           assert (Registry.exists "test_file1.txt" loaded_registry);

     

       216
       216
       -
           assert (Registry.exists "test_file2.txt" loaded_registry);

     

       217
       217
       -
           

     

       218
       218
       -
           Printf.printf "✓ File I/O tests passed\n"

     

       219
       219
       -
       

     

       220
       220
       -
       let test_edge_cases () =

     

       221
       221
       -
         Printf.printf "=== Testing edge cases ===\n";

     

       222
       222
       -
         

     

       223
       223
       -
         (* Empty registry *)

     

       224
       224
       -
         let empty_str = "" in

     

       225
       225
       -
         let empty_registry = Registry.of_string empty_str in

     

       226
       226
       -
         assert (Registry.size empty_registry = 0);

     

       227
       227
       -
         

     

       228
       228
       -
         (* Only comments *)

     

       229
       229
       -
         let comments_only = "# Comment 1\n# Comment 2\n\n# Comment 3" in

     

       230
       230
       -
         let comments_registry = Registry.of_string comments_only in

     

       231
       231
       -
         assert (Registry.size comments_registry = 0);

     

       232
       232
       -
         

     

       233
       233
       -
         (* Malformed lines (should be ignored or handled gracefully) *)

     

       234
       234
       -
         let malformed = "valid.txt sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\njust_filename_no_hash\n" in

     

       235
       235
       -
         let malformed_registry = Registry.of_string malformed in

     

       236
       236
       -
         assert (Registry.size malformed_registry = 1);

     

       237
       237
       -
         assert (Registry.exists "valid.txt" malformed_registry);

     

       238
       238
       -
         

     

       239
       239
       -
         Printf.printf "✓ Edge case tests passed\n"

     

       240
       240
       -
       

     

       241
       241
       -
       let test_progress_callback () =

     

       242
       242
       -
         Printf.printf "=== Testing progress callback ===\n";

     

       243
       243
       -
         

     

       244
       244
       -
         let call_count = ref 0 in

     

       245
       245
       -
         let progress_callback line_num total_lines =

     

       246
       246
       -
           incr call_count;

     

       247
       247
       -
           Printf.printf "Progress: %d/%d\n" line_num total_lines

     

       248
       248
       -
         in

     

       249
       249
       -
         

     

       250
       250
       -
         let registry = Registry.of_string ~progress:progress_callback test_registry_content in

     

       251
       251
       -
         assert (Registry.size registry = 4);

     

       252
       252
       -
         assert (!call_count > 0);

     

       253
       253
       -
         

     

       254
       254
       -
         Printf.printf "✓ Progress callback tests passed\n"

     

       255
       255
       -
       

     

       256
       256
       -
       let run_all_tests () =

     

       257
       257
       -
         Printf.printf "Running Registry module tests...\n\n";

     

       258
       258
       -
         

     

       259
       259
       -
         test_basic_parsing ();

     

       260
       260
       -
         test_round_trip_serialization ();

     

       261
       261
       -
         test_entry_operations ();

     

       262
       262
       -
         test_hash_format_parsing ();

     

       263
       263
       -
         test_comment_and_empty_line_handling ();

     

       264
       264
       -
         test_file_io_operations ();

     

       265
       265
       -
         test_edge_cases ();

     

       266
       266
       -
         test_progress_callback ();

     

       267
       267
       -
         

     

       268
       268
       -
         Printf.printf "\n🎉 All Registry tests passed!\n"

     

       269
       269
       -
       

     

       270
       270
       -
       let () = run_all_tests ()

-29

stack/toru/test/test_registry_real.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let test_tessera_registry () =

     

       4
       4
       -
         Printf.printf "Testing Registry with tessera-manifests...\n";

     

       5
       5
       -
         

     

       6
       6
       -
         (* Load a small sample from tessera-manifests *)

     

       7
       7
       -
         let test_registry_content = "2024/grid_-5.05_50.05/grid_-5.05_50.05.npy e10d31df93ea3c907827aefce89950127f8de3f7a4b612b82d4445feedb7bc0b\n2024/grid_-5.05_50.05/grid_-5.05_50.05_scales.npy de016fd2674fc3e4562822c6ceb8c8ee671f1d04b842bf519cb37d89b725f48e\n# This is a comment\n\n2024/grid_-5.05_50.15/grid_-5.05_50.15.npy 2841b5c65699e4a355f9ab8ed6021b72ecdfdadf63d44d60e81380ceffd1a908" in

     

       8
       8
       -
         

     

       9
       9
       -
         (* Parse the registry *)

     

       10
       10
       -
         let registry = Registry.of_string test_registry_content in

     

       11
       11
       -
         

     

       12
       12
       -
         Printf.printf "Registry size: %d\n" (Registry.size registry);

     

       13
       13
       -
         

     

       14
       14
       -
         (* Test lookup *)

     

       15
       15
       -
         match Registry.find "2024/grid_-5.05_50.05/grid_-5.05_50.05.npy" registry with

     

       16
       16
       -
         | Some entry -> 

     

       17
       17
       -
             Printf.printf "Found entry: %s\n" (Registry.filename entry);

     

       18
       18
       -
             Printf.printf "Hash: %s\n" (Hash.to_string (Registry.hash entry))

     

       19
       19
       -
         | None -> Printf.printf "Entry not found\n";

     

       20
       20
       -
         

     

       21
       21
       -
         (* Test round-trip *)

     

       22
       22
       -
         let serialized = Registry.to_string registry in

     

       23
       23
       -
         let parsed_back = Registry.of_string serialized in

     

       24
       24
       -
         Printf.printf "Round-trip test - size matches: %b\n" 

     

       25
       25
       -
           (Registry.size registry = Registry.size parsed_back);

     

       26
       26
       -
           

     

       27
       27
       -
         Printf.printf "Registry parsing test completed\n"

     

       28
       28
       -
       

     

       29
       29
       -
       let () = test_tessera_registry ()

-275

stack/toru/test/test_tessera_integration.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       open Printf

     

       3
       3
       -
       

     

       4
       4
       -
       (* Test URLs for tessera-manifests *)

     

       5
       5
       -
       let embeddings_url = "https://raw.githubusercontent.com/ucam-eo/tessera-manifests/main/registry/embeddings/embeddings_2024_lon-10_lat50.txt"

     

       6
       6
       -
       let landmasks_url = "https://raw.githubusercontent.com/ucam-eo/tessera-manifests/main/registry/landmasks/landmasks_lon-10_lat50.txt"

     

       7
       7
       -
       

     

       8
       8
       -
       (* Helper function to create a test cache directory *)

     

       9
       9
       -
       let create_test_cache () =

     

       10
       10
       -
         let cache_dir = "/tmp/toru_tessera_test_" ^ (string_of_int (Random.int 10000)) in

     

       11
       11
       -
         Unix.mkdir cache_dir 0o755;

     

       12
       12
       -
         cache_dir

     

       13
       13
       -
       

     

       14
       14
       -
       (* Test 1: Registry loading from tessera-manifests URLs *)

     

       15
       15
       -
       let test_registry_loading () =

     

       16
       16
       -
         printf "=== Test 1: Registry loading from tessera-manifests URLs ===\n";

     

       17
       17
       -
         

     

       18
       18
       -
         printf "Loading embeddings registry from URL...\n";

     

       19
       19
       -
         let embeddings_registry = Registry.load_from_url embeddings_url in

     

       20
       20
       -
         printf "Embeddings registry size: %d entries\n" (Registry.size embeddings_registry);

     

       21
       21
       -
         

     

       22
       22
       -
         printf "Loading landmasks registry from URL...\n";  

     

       23
       23
       -
         let landmasks_registry = Registry.load_from_url landmasks_url in

     

       24
       24
       -
         printf "Landmasks registry size: %d entries\n" (Registry.size landmasks_registry);

     

       25
       25
       -
         

     

       26
       26
       -
         (* Verify both registries are non-empty *)

     

       27
       27
       -
         assert (Registry.size embeddings_registry > 0);

     

       28
       28
       -
         assert (Registry.size landmasks_registry > 0);

     

       29
       29
       -
         

     

       30
       30
       -
         printf "✓ Registry loading test passed\n\n";

     

       31
       31
       -
         (embeddings_registry, landmasks_registry)

     

       32
       32
       -
       

     

       33
       33
       -
       (* Test 2: Registry structure and content validation *)

     

       34
       34
       -
       let test_registry_structure embeddings_registry landmasks_registry =

     

       35
       35
       -
         printf "=== Test 2: Registry structure and content validation ===\n";

     

       36
       36
       -
         

     

       37
       37
       -
         (* Test embeddings registry structure *)

     

       38
       38
       -
         printf "Testing embeddings registry structure...\n";

     

       39
       39
       -
         let embeddings_entries = Registry.entries embeddings_registry in

     

       40
       40
       -
         List.iteri (fun i entry ->

     

       41
       41
       -
           if i < 3 then (  (* Only test first 3 entries for brevity *)

     

       42
       42
       -
             let filename = Registry.filename entry in

     

       43
       43
       -
             let hash = Registry.hash entry in

     

       44
       44
       -
             printf "  Entry %d: %s -> %s\n" (i+1) filename (Hash.to_string hash);

     

       45
       45
       -
             

     

       46
       46
       -
             (* Verify filename format for embeddings *)

     

       47
       47
       -
             assert (String.contains filename '/');

     

       48
       48
       -
             assert (Filename.extension filename = ".npy");

     

       49
       49
       -
             

     

       50
       50
       -
             (* Verify hash format (should be SHA-256: 64 chars) *)

     

       51
       51
       -
             assert (Hash.algorithm hash = SHA256);

     

       52
       52
       -
             assert (String.length (Hash.value hash) = 64);

     

       53
       53
       -
           )

     

       54
       54
       -
         ) embeddings_entries;

     

       55
       55
       -
         

     

       56
       56
       -
         (* Test landmasks registry structure *)

     

       57
       57
       -
         printf "Testing landmasks registry structure...\n";

     

       58
       58
       -
         let landmasks_entries = Registry.entries landmasks_registry in

     

       59
       59
       -
         List.iteri (fun i entry ->

     

       60
       60
       -
           if i < 3 then (  (* Only test first 3 entries for brevity *)

     

       61
       61
       -
             let filename = Registry.filename entry in

     

       62
       62
       -
             let hash = Registry.hash entry in

     

       63
       63
       -
             printf "  Entry %d: %s -> %s\n" (i+1) filename (Hash.to_string hash);

     

       64
       64
       -
             

     

       65
       65
       -
             (* Verify filename format for landmasks *)

     

       66
       66
       -
             assert (Filename.extension filename = ".tiff");

     

       67
       67
       -
             assert (String.contains filename '_');

     

       68
       68
       -
             

     

       69
       69
       -
             (* Verify hash format *)

     

       70
       70
       -
             assert (Hash.algorithm hash = SHA256);

     

       71
       71
       -
             assert (String.length (Hash.value hash) = 64);

     

       72
       72
       -
           )

     

       73
       73
       -
         ) landmasks_entries;

     

       74
       74
       -
         

     

       75
       75
       -
         printf "✓ Registry structure validation test passed\n\n"

     

       76
       76
       -
       

     

       77
       77
       -
       (* Test 3: Hash format validation with real data *)

     

       78
       78
       -
       let test_hash_validation embeddings_registry _landmasks_registry =

     

       79
       79
       -
         printf "=== Test 3: Hash format validation ===\n";

     

       80
       80
       -
         

     

       81
       81
       -
         (* Test hash parsing and formatting *)

     

       82
       82
       -
         let test_entry = List.hd (Registry.entries embeddings_registry) in

     

       83
       83
       -
         let hash = Registry.hash test_entry in

     

       84
       84
       -
         

     

       85
       85
       -
         printf "Testing hash operations on real data...\n";

     

       86
       86
       -
         printf "  Original hash: %s\n" (Hash.to_string hash);

     

       87
       87
       -
         printf "  Algorithm: %s\n" (Hash.algorithm_to_string (Hash.algorithm hash));

     

       88
       88
       -
         

     

       89
       89
       -
         (* Test round-trip parsing *)

     

       90
       90
       -
         let hash_string = Hash.to_string hash in

     

       91
       91
       -
         let parsed_hash = Hash.of_string hash_string in

     

       92
       92
       -
         assert (Hash.equal hash parsed_hash);

     

       93
       93
       -
         

     

       94
       94
       -
         (* Test prefixed format *)

     

       95
       95
       -
         let prefixed = Hash.format_prefixed hash in

     

       96
       96
       -
         printf "  Prefixed format: %s\n" prefixed;

     

       97
       97
       -
         

     

       98
       98
       -
         (match Hash.parse_prefixed prefixed with

     

       99
       99
       -
         | Some (alg, value) -> 

     

       100
       100
       -
             assert (alg = Hash.algorithm hash);

     

       101
       101
       -
             assert (value = Hash.value hash);

     

       102
       102
       -
             printf "  Prefixed parsing: ✓\n"

     

       103
       103
       -
         | None -> failwith "Failed to parse prefixed hash");

     

       104
       104
       -
         

     

       105
       105
       -
         printf "✓ Hash validation test passed\n\n"

     

       106
       106
       -
       

     

       107
       107
       -
       (* Test 4: Registry query operations *)

     

       108
       108
       -
       let test_registry_queries embeddings_registry _landmasks_registry =

     

       109
       109
       -
         printf "=== Test 4: Registry query operations ===\n";

     

       110
       110
       -
         

     

       111
       111
       -
         (* Test finding specific entries *)

     

       112
       112
       -
         let embeddings_entries = Registry.entries embeddings_registry in

     

       113
       113
       -
         let test_filename = Registry.filename (List.hd embeddings_entries) in

     

       114
       114
       -
         

     

       115
       115
       -
         printf "Testing query operations...\n";

     

       116
       116
       -
         printf "  Looking for: %s\n" test_filename;

     

       117
       117
       -
         

     

       118
       118
       -
         (match Registry.find test_filename embeddings_registry with

     

       119
       119
       -
         | Some found_entry ->

     

       120
       120
       -
             printf "  Found entry: %s\n" (Registry.filename found_entry);

     

       121
       121
       -
             assert (Registry.filename found_entry = test_filename);

     

       122
       122
       -
             assert (Registry.exists test_filename embeddings_registry);

     

       123
       123
       -
             printf "  Exists check: ✓\n"

     

       124
       124
       -
         | None -> failwith "Failed to find known entry");

     

       125
       125
       -
         

     

       126
       126
       -
         (* Test non-existent file *)

     

       127
       127
       -
         let fake_filename = "non/existent/file.npy" in

     

       128
       128
       -
         assert (not (Registry.exists fake_filename embeddings_registry));

     

       129
       129
       -
         assert (Registry.find fake_filename embeddings_registry = None);

     

       130
       130
       -
         printf "  Non-existent file handling: ✓\n";

     

       131
       131
       -
         

     

       132
       132
       -
         printf "✓ Registry query operations test passed\n\n"

     

       133
       133
       -
       

     

       134
       134
       -
       (* Test 5: Full Toru workflow without downloads *)

     

       135
       135
       -
       let test_toru_workflow embeddings_registry =

     

       136
       136
       -
         printf "=== Test 5: Toru workflow (without actual downloads) ===\n";

     

       137
       137
       -
         

     

       138
       138
       -
         (* Since we can't run Eio tests in this context, we'll simulate the workflow *)

     

       139
       139
       -
         let cache_path = create_test_cache () in

     

       140
       140
       -
         printf "Created test cache at: %s\n" cache_path;

     

       141
       141
       -
         

     

       142
       142
       -
         (* Test default cache path generation *)

     

       143
       143
       -
         let default_path = Toru.default_cache_path ~app_name:"tessera_test" () in

     

       144
       144
       -
         printf "Default cache path: %s\n" default_path;

     

       145
       145
       -
         assert (String.contains default_path '/');

     

       146
       146
       -
         

     

       147
       147
       -
         (* Test registry serialization/deserialization *)

     

       148
       148
       -
         printf "Testing registry serialization...\n";

     

       149
       149
       -
         let serialized = Registry.to_string embeddings_registry in

     

       150
       150
       -
         let deserialized = Registry.of_string serialized in

     

       151
       151
       -
         

     

       152
       152
       -
         assert (Registry.size embeddings_registry = Registry.size deserialized);

     

       153
       153
       -
         printf "  Serialization round-trip: ✓\n";

     

       154
       154
       -
         

     

       155
       155
       -
         (* Clean up test cache *)

     

       156
       156
       -
         (try Unix.rmdir cache_path with _ -> ());

     

       157
       157
       -
         

     

       158
       158
       -
         printf "✓ Toru workflow test passed\n\n"

     

       159
       159
       -
       

     

       160
       160
       -
       (* Test 6: Geographic coordinate parsing *)

     

       161
       161
       -
       let test_geographic_parsing landmasks_registry =

     

       162
       162
       -
         printf "=== Test 6: Geographic coordinate parsing ===\n";

     

       163
       163
       -
         

     

       164
       164
       -
         let entries = Registry.entries landmasks_registry in

     

       165
       165
       -
         let rec take n lst = 

     

       166
       166
       -
           match n, lst with 

     

       167
       167
       -
           | 0, _ -> [] 

     

       168
       168
       -
           | _, [] -> [] 

     

       169
       169
       -
           | n, x :: xs -> x :: (take (n-1) xs) 

     

       170
       170
       -
         in

     

       171
       171
       -
         let test_filenames = List.map Registry.filename entries |> take 5 in

     

       172
       172
       -
         

     

       173
       173
       -
         printf "Testing coordinate extraction from filenames...\n";

     

       174
       174
       -
         List.iter (fun filename ->

     

       175
       175
       -
           printf "  File: %s\n" filename;

     

       176
       176
       -
           

     

       177
       177
       -
           (* Extract coordinates using regex-like pattern matching *)

     

       178
       178
       -
           if String.contains filename '_' then (

     

       179
       179
       -
             let parts = String.split_on_char '_' filename in

     

       180
       180
       -
             match parts with

     

       181
       181
       -
             | "grid" :: lon_str :: lat_str :: _ ->

     

       182
       182
       -
                 (try

     

       183
       183
       -
                   let lon = Float.of_string lon_str in

     

       184
       184
       -
                   let lat = Float.of_string (String.split_on_char '.' lat_str |> List.hd) in

     

       185
       185
       -
                   printf "    Coordinates: %.2f, %.2f\n" lon lat;

     

       186
       186
       -
                   (* Basic coordinate validation *)

     

       187
       187
       -
                   assert (lon >= -180.0 && lon <= 180.0);

     

       188
       188
       -
                   assert (lat >= -90.0 && lat <= 90.0);

     

       189
       189
       -
                 with _ -> printf "    Could not parse coordinates\n")

     

       190
       190
       -
             | _ -> printf "    Unexpected filename format\n"

     

       191
       191
       -
           )

     

       192
       192
       -
         ) test_filenames;

     

       193
       193
       -
         

     

       194
       194
       -
         printf "✓ Geographic coordinate parsing test passed\n\n"

     

       195
       195
       -
       

     

       196
       196
       -
       (* Test 7: Registry comparison and merging *)

     

       197
       197
       -
       let test_registry_operations embeddings_registry _landmasks_registry =

     

       198
       198
       -
         printf "=== Test 7: Registry operations ===\n";

     

       199
       199
       -
         

     

       200
       200
       -
         (* Create a small test registry *)

     

       201
       201
       -
         let test_entry = Registry.create_entry 

     

       202
       202
       -
           ~filename:"test/file.txt" 

     

       203
       203
       -
           ~hash:(Hash.create SHA256 "abc123def456") 

     

       204
       204
       -
           () in

     

       205
       205
       -
         

     

       206
       206
       -
         let small_registry = Registry.add test_entry Registry.empty in

     

       207
       207
       -
         printf "Created test registry with %d entries\n" (Registry.size small_registry);

     

       208
       208
       -
         

     

       209
       209
       -
         (* Test adding to existing registry *)

     

       210
       210
       -
         let expanded_registry = Registry.add test_entry embeddings_registry in

     

       211
       211
       -
         assert (Registry.size expanded_registry = Registry.size embeddings_registry + 1);

     

       212
       212
       -
         printf "Successfully added entry to registry\n";

     

       213
       213
       -
         

     

       214
       214
       -
         (* Test removal *)

     

       215
       215
       -
         let reduced_registry = Registry.remove "test/file.txt" expanded_registry in

     

       216
       216
       -
         assert (Registry.size reduced_registry = Registry.size embeddings_registry);

     

       217
       217
       -
         printf "Successfully removed entry from registry\n";

     

       218
       218
       -
         

     

       219
       219
       -
         printf "✓ Registry operations test passed\n\n"

     

       220
       220
       -
       

     

       221
       221
       -
       (* Test 8: Error handling *)

     

       222
       222
       -
       let test_error_handling () =

     

       223
       223
       -
         printf "=== Test 8: Error handling ===\n";

     

       224
       224
       -
         

     

       225
       225
       -
         (* Test invalid hash parsing *)

     

       226
       226
       -
         (try

     

       227
       227
       -
           let _ = Hash.of_string "invalid_hash_format:xyz" in

     

       228
       228
       -
           printf "Hash parsing with invalid format (should work or fail gracefully)\n"

     

       229
       229
       -
         with _ -> printf "✓ Invalid hash format properly handled\n");

     

       230
       230
       -
         

     

       231
       231
       -
         (* Test malformed registry parsing *)

     

       232
       232
       -
         let malformed_registry = "invalid registry content without proper format" in

     

       233
       233
       -
         let parsed = Registry.of_string malformed_registry in

     

       234
       234
       -
         (* Should create empty registry or handle gracefully *)

     

       235
       235
       -
         printf "Malformed registry size: %d (expected 0)\n" (Registry.size parsed);

     

       236
       236
       -
         

     

       237
       237
       -
         (* Test empty registry operations *)

     

       238
       238
       -
         let empty_reg = Registry.empty in

     

       239
       239
       -
         assert (Registry.size empty_reg = 0);

     

       240
       240
       -
         assert (Registry.find "any_file.txt" empty_reg = None);

     

       241
       241
       -
         printf "✓ Empty registry operations work correctly\n";

     

       242
       242
       -
         

     

       243
       243
       -
         printf "✓ Error handling test passed\n\n"

     

       244
       244
       -
       

     

       245
       245
       -
       (* Main test runner *)

     

       246
       246
       -
       let run_tests () =

     

       247
       247
       -
         printf "Starting Tessera-Manifests Integration Tests\n";

     

       248
       248
       -
         printf "============================================\n\n";

     

       249
       249
       -
         

     

       250
       250
       -
         Random.self_init ();

     

       251
       251
       -
         

     

       252
       252
       -
         try

     

       253
       253
       -
           (* Run tests sequentially *)

     

       254
       254
       -
           let (embeddings_registry, landmasks_registry) = test_registry_loading () in

     

       255
       255
       -
           test_registry_structure embeddings_registry landmasks_registry;

     

       256
       256
       -
           test_hash_validation embeddings_registry landmasks_registry;

     

       257
       257
       -
           test_registry_queries embeddings_registry landmasks_registry;

     

       258
       258
       -
           test_toru_workflow embeddings_registry;

     

       259
       259
       -
           test_geographic_parsing landmasks_registry;

     

       260
       260
       -
           test_registry_operations embeddings_registry landmasks_registry;

     

       261
       261
       -
           test_error_handling ();

     

       262
       262
       -
           

     

       263
       263
       -
           printf "============================================\n";

     

       264
       264
       -
           printf "All Tessera Integration Tests PASSED! ✅\n";

     

       265
       265
       -
           printf "============================================\n\n";

     

       266
       266
       -
           

     

       267
       267
       -
         with 

     

       268
       268
       -
         | Failure msg -> 

     

       269
       269
       -
             printf "❌ TEST FAILED: %s\n" msg;

     

       270
       270
       -
             exit 1

     

       271
       271
       -
         | e -> 

     

       272
       272
       -
             printf "❌ UNEXPECTED ERROR: %s\n" (Printexc.to_string e);

     

       273
       273
       -
             exit 1

     

       274
       274
       -
       

     

       275
       275
       -
       let () = run_tests ()

-36

stack/toru/test/test_tessera_load.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let test_tessera_file_load () =

     

       4
       4
       -
         Printf.printf "Testing Registry with actual tessera-manifests file...\n";

     

       5
       5
       -
         

     

       6
       6
       -
         let manifest_path = "/Users/avsm/src/git/ucam-eo/tessera-manifests/registry/embeddings/embeddings_2024_lon-10_lat50.txt" in

     

       7
       7
       -
         

     

       8
       8
       -
         if Sys.file_exists manifest_path then (

     

       9
       9
       -
           let ic = open_in manifest_path in

     

       10
       10
       -
           let content = really_input_string ic (in_channel_length ic) in

     

       11
       11
       -
           close_in ic;

     

       12
       12
       -
           

     

       13
       13
       -
           let registry = Registry.of_string content in

     

       14
       14
       -
           

     

       15
       15
       -
           Printf.printf "Loaded tessera registry with %d entries\n" (Registry.size registry);

     

       16
       16
       -
           

     

       17
       17
       -
           (* Test a few specific entries *)

     

       18
       18
       -
           let test_files = [

     

       19
       19
       -
             "2024/grid_-5.05_50.05/grid_-5.05_50.05.npy";

     

       20
       20
       -
             "2024/grid_-5.05_50.15/grid_-5.05_50.15.npy";

     

       21
       21
       -
           ] in

     

       22
       22
       -
           

     

       23
       23
       -
           List.iter (fun filename ->

     

       24
       24
       -
             match Registry.find filename registry with

     

       25
       25
       -
             | Some entry -> 

     

       26
       26
       -
                 Printf.printf "Found %s -> %s\n" filename 

     

       27
       27
       -
                   (Hash.to_string (Registry.hash entry))

     

       28
       28
       -
             | None -> Printf.printf "Not found: %s\n" filename

     

       29
       29
       -
           ) test_files;

     

       30
       30
       -
           

     

       31
       31
       -
           Printf.printf "Tessera file load test completed successfully\n"

     

       32
       32
       -
         ) else (

     

       33
       33
       -
           Printf.printf "Tessera manifest file not found at %s\n" manifest_path

     

       34
       34
       -
         )

     

       35
       35
       -
       

     

       36
       36
       -
       let () = test_tessera_file_load ()

-52

stack/toru/test/test_toru.ml

···

       1
       1
       -
       open Toru

     

       2
       2
       -
       

     

       3
       3
       -
       let test_hash () =

     

       4
       4
       -
         Printf.printf "Testing Hash module...\n";

     

       5
       5
       -
         let hash1 = Hash.of_string "sha256:abc123def456" in

     

       6
       6
       -
         let hash2 = Hash.create Hash.SHA256 "abc123def456" in

     

       7
       7
       -
         Printf.printf "Hash 1: %s\n" (Hash.to_string hash1);

     

       8
       8
       -
         Printf.printf "Hash 2: %s\n" (Hash.to_string hash2);

     

       9
       9
       -
         Printf.printf "Equal: %b\n" (Hash.equal hash1 hash2);

     

       10
       10
       -
         Format.printf "Algorithm: %a\n" Hash.pp_algorithm (Hash.algorithm hash1);

     

       11
       11
       -
         Printf.printf "Value: %s\n" (Hash.value hash1)

     

       12
       12
       -
       

     

       13
       13
       -
       let test_registry () =

     

       14
       14
       -
         Printf.printf "\nTesting Registry module...\n";

     

       15
       15
       -
         let hash = Hash.of_string "sha256:deadbeef" in

     

       16
       16
       -
         let entry = Registry.create_entry ~filename:"test.txt" ~hash () in

     

       17
       17
       -
         Printf.printf "Entry filename: %s\n" (Registry.filename entry);

     

       18
       18
       -
         Printf.printf "Entry hash: %s\n" (Hash.to_string (Registry.hash entry));

     

       19
       19
       -
         

     

       20
       20
       -
         let registry = Registry.empty |> Registry.add entry in

     

       21
       21
       -
         Printf.printf "Registry size: %d\n" (Registry.size registry);

     

       22
       22
       -
         

     

       23
       23
       -
         match Registry.find "test.txt" registry with

     

       24
       24
       -
         | Some found -> Printf.printf "Found entry: %s\n" (Registry.filename found)

     

       25
       25
       -
         | None -> Printf.printf "Entry not found\n"

     

       26
       26
       -
       

     

       27
       27
       -
       let test_cache () =

     

       28
       28
       -
         Printf.printf "\nTesting Cache module...\n";

     

       29
       29
       -
         let cache_path = Cache.default_cache_path ~app_name:"test-toru" () in

     

       30
       30
       -
         Printf.printf "Default cache path: %s\n" cache_path

     

       31
       31
       -
       

     

       32
       32
       -
       let test_downloader () =

     

       33
       33
       -
         Printf.printf "\nTesting Downloader module...\n";

     

       34
       34
       -
         Eio_main.run @@ fun env ->

     

       35
       35
       -
         Eio.Switch.run @@ fun sw ->

     

       36
       36
       -
           let (module D : Downloader.DOWNLOADER) = Downloader.Downloaders.wget () in

     

       37
       37
       -
           let downloader = D.create ~sw ~env () in

     

       38
       38
       -
           Printf.printf "Downloader name: %s\n" (D.name downloader);

     

       39
       39
       -
           Printf.printf "Supports resume: %b\n" (D.supports_resume downloader)

     

       40
       40
       -
       

     

       41
       41
       -
       let main () =

     

       42
       42
       -
         Printf.printf "Toru Library Test\n";

     

       43
       43
       -
         Printf.printf "=================\n";

     

       44
       44
       -
         

     

       45
       45
       -
         test_hash ();

     

       46
       46
       -
         test_registry ();

     

       47
       47
       -
         test_cache ();

     

       48
       48
       -
         test_downloader ();

     

       49
       49
       -
         

     

       50
       50
       -
         Printf.printf "\nAll basic tests completed!\n"

     

       51
       51
       -
       

     

       52
       52
       -
       let () = main ()

-112

stack/toru/test/test_xdg_integration.ml

···

       1
       1
       -
       open Printf

     

       2
       2
       -
       

     

       3
       3
       -
       (** Test XDG integration to verify we're using the official xdg package *)

     

       4
       4
       -
       

     

       5
       5
       -
       let test_xdg_paths () =

     

       6
       6
       -
         printf "Testing XDG Base Directory integration...\n";

     

       7
       7
       -
         

     

       8
       8
       -
         (* Test with default app name *)

     

       9
       9
       -
         let default_path = Toru.Cache.default_cache_path ~app_name:"test-app" () in

     

       10
       10
       -
         printf "  - Default cache path: %s\n" default_path;

     

       11
       11
       -
         

     

       12
       12
       -
         (* Test with custom environment *)

     

       13
       13
       -
         let custom_home = "/tmp/test-home" in

     

       14
       14
       -
         let custom_cache = "/tmp/test-cache" in

     

       15
       15
       -
         

     

       16
       16
       -
         (* Create a custom XDG configuration *)

     

       17
       17
       -
         let custom_env var =

     

       18
       18
       -
           match var with

     

       19
       19
       -
           | "HOME" -> Some custom_home

     

       20
       20
       -
           | "XDG_CACHE_HOME" -> Some custom_cache

     

       21
       21
       -
           | _ -> None

     

       22
       22
       -
         in

     

       23
       23
       -
         

     

       24
       24
       -
         let xdg_dirs = Xdg.create ~env:custom_env () in

     

       25
       25
       -
         let cache_dir = Xdg.cache_dir xdg_dirs in

     

       26
       26
       -
         let custom_path = Filename.concat cache_dir "test-app" in

     

       27
       27
       -
         

     

       28
       28
       -
         printf "  - Custom XDG cache dir: %s\n" cache_dir;

     

       29
       29
       -
         printf "  - Custom app cache path: %s\n" custom_path;

     

       30
       30
       -
         

     

       31
       31
       -
         (* Verify the paths make sense *)

     

       32
       32
       -
         assert (String.contains default_path '.');  (* Should contain .cache or similar *)

     

       33
       33
       -
         assert (custom_path = "/tmp/test-cache/test-app");

     

       34
       34
       -
         

     

       35
       35
       -
         printf "✓ XDG integration test passed\n\n"

     

       36
       36
       -
       

     

       37
       37
       -
       let test_cross_platform_support () =

     

       38
       38
       -
         printf "Testing cross-platform XDG support...\n";

     

       39
       39
       -
         

     

       40
       40
       -
         (* Test different platform configurations *)

     

       41
       41
       -
         let test_configs = [

     

       42
       42
       -
           ("Unix HOME only", fun var -> if var = "HOME" then Some "/home/user" else None);

     

       43
       43
       -
           ("Unix with XDG", fun var -> 

     

       44
       44
       -
             match var with 

     

       45
       45
       -
             | "HOME" -> Some "/home/user"

     

       46
       46
       -
             | "XDG_CACHE_HOME" -> Some "/home/user/.cache"

     

       47
       47
       -
             | _ -> None);

     

       48
       48
       -
           ("Windows", fun var ->

     

       49
       49
       -
             match var with

     

       50
       50
       -
             | "USERPROFILE" -> Some "C:\\Users\\User"

     

       51
       51
       -
             | "LOCALAPPDATA" -> Some "C:\\Users\\User\\AppData\\Local"

     

       52
       52
       -
             | _ -> None);

     

       53
       53
       -
         ] in

     

       54
       54
       -
         

     

       55
       55
       -
         List.iter (fun (name, env_fn) ->

     

       56
       56
       -
           let xdg_dirs = Xdg.create ~env:env_fn () in

     

       57
       57
       -
           let cache_dir = Xdg.cache_dir xdg_dirs in

     

       58
       58
       -
           printf "  - %s: %s\n" name cache_dir;

     

       59
       59
       -
           

     

       60
       60
       -
           (* Basic sanity check *)

     

       61
       61
       -
           assert (String.length cache_dir > 0);

     

       62
       62
       -
         ) test_configs;

     

       63
       63
       -
         

     

       64
       64
       -
         printf "✓ Cross-platform support test passed\n\n"

     

       65
       65
       -
       

     

       66
       66
       -
       let test_cache_creation_with_xdg () =

     

       67
       67
       -
         printf "Testing Cache creation with XDG...\n";

     

       68
       68
       -
         

     

       69
       69
       -
         Eio_main.run @@ fun env ->

     

       70
       70
       -
         Eio.Switch.run @@ fun sw ->

     

       71
       71
       -
           (* Create cache using default XDG paths *)

     

       72
       72
       -
           let base_path = Toru.Cache.default_cache_path ~app_name:"xdg-test" () in

     

       73
       73
       -
           let cache = Toru.Cache.create ~sw ~fs:env#fs base_path in

     

       74
       74
       -
           let base_path = Toru.Cache.base_path cache in

     

       75
       75
       -
           let path_str = Eio.Path.native_exn base_path in

     

       76
       76
       -
           

     

       77
       77
       -
           printf "  - Cache base path: %s\n" path_str;

     

       78
       78
       -
           

     

       79
       79
       -
           (* Verify it contains expected XDG components *)

     

       80
       80
       -
           assert (String.contains path_str '/');  (* Should be a proper path *)

     

       81
       81
       -
           assert (String.length path_str > 10);   (* Should be reasonably long *)

     

       82
       82
       -
           

     

       83
       83
       -
           (* Test file operations work *)

     

       84
       84
       -
           Toru.Cache.ensure_dir cache;

     

       85
       85
       -
           printf "  - ✓ Cache directory creation works\n";

     

       86
       86
       -
           

     

       87
       87
       -
           let test_file = "xdg-test-file.txt" in

     

       88
       88
       -
           let exists_before = Toru.Cache.exists cache test_file in

     

       89
       89
       -
           assert (not exists_before);

     

       90
       90
       -
           printf "  - ✓ File existence check works\n";

     

       91
       91
       -
           

     

       92
       92
       -
           printf "✓ Cache creation with XDG test passed\n\n"

     

       93
       93
       -
       

     

       94
       94
       -
       let run_tests () =

     

       95
       95
       -
         printf "=== XDG Integration Tests ===\n\n";

     

       96
       96
       -
         

     

       97
       97
       -
         try

     

       98
       98
       -
           test_xdg_paths ();

     

       99
       99
       -
           test_cross_platform_support ();

     

       100
       100
       -
           test_cache_creation_with_xdg ();

     

       101
       101
       -
           

     

       102
       102
       -
           printf "🎉 All XDG integration tests passed!\n";

     

       103
       103
       -
           printf "✅ Successfully replaced homebrew XDG code with official xdg package\n\n";

     

       104
       104
       -
         with

     

       105
       105
       -
         | Failure msg ->

     

       106
       106
       -
             printf "❌ Test failed: %s\n" msg;

     

       107
       107
       -
             exit 1

     

       108
       108
       -
         | exn ->

     

       109
       109
       -
             printf "❌ Unexpected error: %s\n" (Printexc.to_string exn);

     

       110
       110
       -
             exit 1

     

       111
       111
       -
       

     

       112
       112
       -
       let () = run_tests ()