commit ddb92c2db319156e541b47a33b8fa7ec9d18d1ce · anil.recoil.org/slop

-33

stack/bushel/bin/bushel_main.ml

···

       4
       4
        
       

     

       5
       5
        
       (* Import actual command implementations from submodules *)

     

       6
       6
        
       

     

       7
       7
       -
       (* Faces command *)

     

       8
       8
       -
       let faces_cmd =

     

       9
       9
       -
         let doc = "Retrieve face thumbnails from Immich photo service" in

     

       10
       10
       -
         let info = Cmd.info "faces" ~version ~doc in

     

       11
       11
       -
         Cmd.v info Bushel_faces.term

     

       12
       12
       -
       

     

       13
       13
       -
       (* Links command - uses group structure *)

     

       14
       14
       -
       let links_cmd = Bushel_links.cmd

     

       15
       15
       -
       

     

       16
       7
        
       (* Obsidian command *)

     

       17
       8
        
       let obsidian_cmd =

     

       18
       9
        
         let doc = "Convert Bushel entries to Obsidian format" in

     

       19
       10
        
         let info = Cmd.info "obsidian" ~version ~doc in

     

       20
       11
        
         Cmd.v info Bushel_obsidian.term

     

       21
       12
        
       

     

       22
       22
       -
       (* Paper command *)

     

       23
       23
       -
       let paper_cmd =

     

       24
       24
       -
         let doc = "Fetch paper metadata from DOI" in

     

       25
       25
       -
         let info = Cmd.info "paper" ~version ~doc in

     

       26
       26
       -
         Cmd.v info Bushel_paper.term

     

       27
       27
       -
       

     

       28
       13
        
       (* Paper classify command *)

     

       29
       14
        
       let paper_classify_cmd = Bushel_paper_classify.cmd

     

       30
       15
        
       

     
···

       37
       22
        
         let info = Cmd.info "thumbs" ~version ~doc in

     

       38
       23
        
         Cmd.v info Bushel_thumbs.term

     

       39
       24
        
       

     

       40
       40
       -
       (* Video command *)

     

       41
       41
       -
       let video_cmd =

     

       42
       42
       -
         let doc = "Fetch videos from PeerTube instances" in

     

       43
       43
       -
         let info = Cmd.info "video" ~version ~doc in

     

       44
       44
       -
         Cmd.v info Bushel_video.term

     

       45
       45
       -
       

     

       46
       46
       -
       (* Video thumbs command *)

     

       47
       47
       -
       let video_thumbs_cmd = Bushel_video_thumbs.cmd

     

       48
       48
       -
       

     

       49
       25
        
       (* Query command *)

     

       50
       26
        
       let query_cmd =

     

       51
       27
        
         let doc = "Query Bushel collections using multisearch" in

     
···

       70
       46
        
       (* Note DOI command *)

     

       71
       47
        
       let note_doi_cmd = Bushel_note_doi.cmd

     

       72
       48
        
       

     

       73
       73
       -
       (* DOI resolve command *)

     

       74
       74
       -
       let doi_cmd = Bushel_doi.cmd

     

       75
       75
       -
       

     

       76
       49
        
       (* Main command *)

     

       77
       50
        
       let bushel_cmd =

     

       78
       51
        
         let doc = "Bushel content management toolkit" in

     
···

       95
       68
        
         let info = Cmd.info "bushel" ~version ~doc ~sdocs ~man in

     

       96
       69
        
         Cmd.group info [

     

       97
       70
        
           bibtex_cmd;

     

       98
       98
       -
           doi_cmd;

     

       99
       99
       -
           faces_cmd;

     

       100
       71
        
           ideas_cmd;

     

       101
       72
        
           info_cmd;

     

       102
       102
       -
           links_cmd;

     

       103
       73
        
           missing_cmd;

     

       104
       74
        
           note_doi_cmd;

     

       105
       75
        
           obsidian_cmd;

     

       106
       106
       -
           paper_cmd;

     

       107
       76
        
           paper_classify_cmd;

     

       108
       77
        
           paper_tex_cmd;

     

       109
       78
        
           query_cmd;

     

       110
       79
        
           thumbs_cmd;

     

       111
       111
       -
           video_cmd;

     

       112
       112
       -
           video_thumbs_cmd;

     

       113
       80
        
         ]

     

       114
       81
        
       

     

       115
       82
        
       let () = exit (Cmd.eval' bushel_cmd)

+32 -52

stack/bushel/bin/bushel_search.ml

···

       1
       1
        
       open Cmdliner

     

       2
       2
       -
       open Lwt.Syntax

     

       3
       3
       -
       

     

       4
       4
       -
       (** TODO:claude Bushel search command for integration with main CLI *)

     

       5
       2
        
       

     

       6
       6
       -
       let endpoint =

     

       7
       7
       -
         let doc = "Typesense server endpoint URL" in

     

       8
       8
       -
         Arg.(value & opt string "" & info ["endpoint"; "e"] ~doc)

     

       9
       9
       -
       

     

       10
       10
       -
       let api_key =

     

       11
       11
       -
         let doc = "Typesense API key for authentication" in

     

       12
       12
       -
         Arg.(value & opt string "" & info ["api-key"; "k"] ~doc)

     

       13
       13
       -
       

     

       3
       3
       +
       (** Bushel search command for integration with main CLI *)

     

       14
       4
        
       

     

       15
       5
        
       let limit =

     

       16
       6
        
         let doc = "Maximum number of results to return" in

     
···

       24
       14
        
         let doc = "Search query text" in

     

       25
       15
        
         Arg.(required & pos 0 (some string) None & info [] ~docv:"QUERY" ~doc)

     

       26
       16
        
       

     

       27
       27
       -
       (** TODO:claude Search function using multisearch *)

     

       28
       28
       -
       let search endpoint api_key query_text limit offset =

     

       29
       29
       -
         let base_config = Bushel.Typesense.load_config_from_files () in

     

       30
       30
       -
         let config = { 

     

       31
       31
       -
           Bushel.Typesense.endpoint = if endpoint = "" then base_config.endpoint else endpoint;

     

       32
       32
       -
           api_key = if api_key = "" then base_config.api_key else api_key;

     

       33
       33
       -
           openai_key = base_config.openai_key;

     

       34
       34
       -
         } in

     

       35
       35
       -
         

     

       17
       17
       +
       (** Search function using multisearch *)

     

       18
       18
       +
       let search query_text limit offset () =

     

       19
       19
       +
         let config = Bushel.Typesense.load_config_from_files () in

     

       20
       20
       +
       

     

       36
       21
        
         if config.api_key = "" then (

     

       37
       37
       -
           Printf.eprintf "Error: API key is required. Use --api-key, set TYPESENSE_API_KEY environment variable, or create .typesense-key file.\n";

     

       38
       38
       -
           exit 1

     

       39
       39
       -
         );

     

       40
       40
       -
         

     

       41
       41
       -
         Printf.printf "Searching Typesense at %s\n" config.endpoint;

     

       42
       42
       -
         Printf.printf "Query: \"%s\"\n" query_text;

     

       43
       43
       -
         Printf.printf "Limit: %d, Offset: %d\n" limit offset;

     

       44
       44
       -
         Printf.printf "\n";

     

       45
       45
       -
         

     

       46
       46
       -
         Lwt_main.run (

     

       47
       47
       -
           Lwt.catch (fun () ->

     

       48
       48
       -
             let* result = Bushel.Typesense.multisearch config query_text ~limit:50 () in

     

       49
       49
       -
             match result with

     

       50
       50
       -
             | Ok multisearch_resp ->

     

       51
       51
       -
               let combined_response = Bushel.Typesense.combine_multisearch_results multisearch_resp ~limit ~offset () in

     

       52
       52
       -
               Printf.printf "Found %d results (%.2fms)\n\n" combined_response.total combined_response.query_time;

     

       53
       53
       -
               

     

       54
       54
       -
               List.iteri (fun i (hit : Bushel.Typesense.search_result) ->

     

       55
       55
       -
                 Printf.printf "%d. %s (score: %.2f)\n" (i + 1) (Bushel.Typesense.pp_search_result_oneline hit) hit.Bushel.Typesense.score

     

       56
       56
       -
               ) combined_response.hits;

     

       57
       57
       -
               Lwt.return_unit

     

       58
       58
       -
             | Error err ->

     

       59
       59
       -
               Format.eprintf "Search error: %a\n" Bushel.Typesense.pp_error err;

     

       60
       60
       -
               exit 1

     

       61
       61
       -
           ) (fun exn ->

     

       62
       62
       -
             Printf.eprintf "Error: %s\n" (Printexc.to_string exn);

     

       63
       63
       -
             exit 1

     

       64
       64
       -
           )

     

       65
       65
       -
         );

     

       66
       66
       -
         0

     

       22
       22
       +
           Printf.eprintf "Error: API key is required. Use TYPESENSE_API_KEY environment variable or create .typesense-key file.\n";

     

       23
       23
       +
           1

     

       24
       24
       +
         ) else (

     

       25
       25
       +
           Printf.printf "Searching Typesense at %s\n" config.endpoint;

     

       26
       26
       +
           Printf.printf "Query: \"%s\"\n" query_text;

     

       27
       27
       +
           Printf.printf "Limit: %d, Offset: %d\n\n" limit offset;

     

       67
       28
        
       

     

       68
       68
       -
       (** TODO:claude Command line term *)

     

       69
       69
       -
       let term = Term.(const search $ endpoint $ api_key $ query_text $ limit $ offset)
     

       29
       29
       +
           Eio_main.run (fun env ->

     

       30
       30
       +
             Eio.Switch.run (fun sw ->

     

       31
       31
       +
               let result = Bushel.Typesense.multisearch ~sw ~env config query_text ~limit:50 () in

     

       32
       32
       +
               match result with

     

       33
       33
       +
               | Ok multisearch_resp ->

     

       34
       34
       +
                 let combined_response = Bushel.Typesense.combine_multisearch_results multisearch_resp ~limit ~offset () in

     

       35
       35
       +
                 Printf.printf "Found %d results (%.2fms)\n\n" combined_response.total combined_response.query_time;

     

       36
       36
       +
       

     

       37
       37
       +
                 List.iteri (fun i (hit : Bushel.Typesense.search_result) ->

     

       38
       38
       +
                   Printf.printf "%d. %s (score: %.2f)\n" (i + 1) (Bushel.Typesense.pp_search_result_oneline hit) hit.Bushel.Typesense.score

     

       39
       39
       +
                 ) combined_response.hits

     

       40
       40
       +
               | Error err ->

     

       41
       41
       +
                 Format.eprintf "Search error: %a\n" Bushel.Typesense.pp_error err;

     

       42
       42
       +
                 exit 1

     

       43
       43
       +
             )

     

       44
       44
       +
           );

     

       45
       45
       +
           0

     

       46
       46
       +
         )

     

       47
       47
       +
       

     

       48
       48
       +
       (** Command line term *)

     

       49
       49
       +
       let term = Term.(const search $ query_text $ limit $ offset $ Bushel_common.setup_term)

+93 -153

stack/bushel/bin/bushel_typesense.ml

···

       1
       1
        
       open Cmdliner

     

       2
       2
       -
       open Lwt.Syntax

     

       3
       2
        
       

     

       4
       4
       -
       (** TODO:claude Bushel Typesense binary with upload and query functionality *)

     

       5
       5
       -
       

     

       6
       6
       -
       let endpoint =

     

       7
       7
       -
         let doc = "Typesense server endpoint URL" in

     

       8
       8
       -
         Arg.(value & opt string "http://localhost:8108" & info ["endpoint"; "e"] ~doc)

     

       9
       9
       -
       

     

       10
       10
       -
       let api_key =

     

       11
       11
       -
         let doc = "Typesense API key for authentication" in

     

       12
       12
       -
         Arg.(value & opt string "" & info ["api-key"; "k"] ~doc)

     

       13
       13
       -
       

     

       14
       14
       -
       let openai_key =

     

       15
       15
       -
         let doc = "OpenAI API key for embeddings" in

     

       16
       16
       -
         Arg.(value & opt string "" & info ["openai-key"; "oa"] ~doc)

     

       3
       3
       +
       (** Bushel Typesense binary with upload and query functionality *)

     

       17
       4
        
       

     

       18
       5
        
       let data_dir =

     

       19
       6
        
         let doc = "Directory containing bushel data files" in

     

       20
       7
        
         Arg.(value & opt string "." & info ["data-dir"; "d"] ~doc)

     

       21
       8
        
       

     

       22
       22
       -
       (** TODO:claude Main upload function *)

     

       23
       23
       -
       let upload endpoint api_key openai_key data_dir =

     

       24
       24
       -
         if api_key = "" then (

     

       25
       25
       -
           Printf.eprintf "Error: API key is required. Use --api-key or set TYPESENSE_API_KEY environment variable.\n";

     

       26
       26
       -
           exit 1

     

       27
       27
       -
         );

     

       9
       9
       +
       (** Main upload function *)

     

       10
       10
       +
       let upload env _xdg _profile data_dir openai_key =

     

       11
       11
       +
         let config = Bushel.Typesense.load_config_from_files () in

     

       28
       12
        
       

     

       29
       29
       -
         if openai_key = "" then (

     

       30
       30
       -
           Printf.eprintf "Error: OpenAI API key is required for embeddings. Use --openai-key or set OPENAI_API_KEY environment variable.\n";

     

       31
       31
       -
           exit 1

     

       32
       32
       -
         );

     

       33
       33
       -
       

     

       34
       34
       -
         let config = Bushel.Typesense.{ endpoint; api_key; openai_key } in

     

       13
       13
       +
         let config = { config with

     

       14
       14
       +
           openai_key = if openai_key = "" then config.openai_key else openai_key

     

       15
       15
       +
         } in

     

       35
       16
        
       

     

       36
       36
       -
         Printf.printf "Loading bushel data from %s\n" data_dir;

     

       37
       37
       -
         let entries = Bushel.load data_dir in

     

       17
       17
       +
         if config.api_key = "" then (

     

       18
       18
       +
           Printf.eprintf "Error: API key is required. Use TYPESENSE_API_KEY environment variable or create .typesense-key file.\n";

     

       19
       19
       +
           1

     

       20
       20
       +
         ) else if config.openai_key = "" then (

     

       21
       21
       +
           Printf.eprintf "Error: OpenAI API key is required for embeddings. Use OPENAI_API_KEY environment variable or create .openrouter-api file.\n";

     

       22
       22
       +
           1

     

       23
       23
       +
         ) else (

     

       24
       24
       +
           Printf.printf "Loading bushel data from %s\n%!" data_dir;

     

       25
       25
       +
           let entries = Bushel.load data_dir in

     

       38
       26
        
       

     

       39
       39
       -
         Printf.printf "Uploading bushel data to Typesense at %s\n" endpoint;

     

       27
       27
       +
           Printf.printf "Uploading bushel data to Typesense at %s\n%!" config.endpoint;

     

       40
       28
        
       

     

       41
       41
       -
         Lwt_main.run (

     

       42
       42
       -
           Lwt.catch (fun () ->

     

       43
       43
       -
             Bushel.Typesense.upload_all config entries

     

       44
       44
       -
           ) (fun exn ->

     

       45
       45
       -
             Printf.eprintf "Error: %s\n" (Printexc.to_string exn);

     

       46
       46
       -
             exit 1

     

       47
       47
       -
           )

     

       29
       29
       +
           Eio.Switch.run (fun sw ->

     

       30
       30
       +
             Bushel.Typesense.upload_all ~sw ~env config entries

     

       31
       31
       +
           );

     

       32
       32
       +
           0

     

       48
       33
        
         )

     

       49
       34
        
       

     

       50
       35
        
       

     

       51
       51
       -
       (** TODO:claude Query function *)

     

       52
       52
       -
       let query endpoint api_key query_text collection limit offset =

     

       53
       53
       -
         let base_config = Bushel.Typesense.load_config_from_files () in

     

       54
       54
       -
         let config = { 

     

       55
       55
       -
           Bushel.Typesense.endpoint = if endpoint = "" then base_config.endpoint else endpoint;

     

       56
       56
       -
           api_key = if api_key = "" then base_config.api_key else api_key;

     

       57
       57
       -
           openai_key = base_config.openai_key;

     

       58
       58
       -
         } in

     

       59
       59
       -
         

     

       36
       36
       +
       (** Query function *)

     

       37
       37
       +
       let query env _xdg _profile query_text collection limit offset =

     

       38
       38
       +
         let config = Bushel.Typesense.load_config_from_files () in

     

       39
       39
       +
       

     

       60
       40
        
         if config.api_key = "" then (

     

       61
       61
       -
           Printf.eprintf "Error: API key is required. Use --api-key or set TYPESENSE_API_KEY environment variable.\n";

     

       62
       62
       -
           exit 1

     

       63
       63
       -
         );

     

       64
       64
       -
         

     

       65
       65
       -
         Printf.printf "Searching Typesense at %s\n" config.endpoint;

     

       66
       66
       -
         Printf.printf "Query: \"%s\"\n" query_text;

     

       67
       67
       -
         if collection <> "" then Printf.printf "Collection: %s\n" collection;

     

       68
       68
       -
         Printf.printf "Limit: %d, Offset: %d\n" limit offset;

     

       69
       69
       -
         Printf.printf "\n";

     

       70
       70
       -
         

     

       71
       71
       -
         Lwt_main.run (

     

       72
       72
       -
           Lwt.catch (fun () ->

     

       73
       73
       -
             let search_fn = if collection = "" then 

     

       74
       74
       -
               Bushel.Typesense.search_all config query_text ~limit ~offset

     

       41
       41
       +
           Printf.eprintf "Error: API key is required. Use TYPESENSE_API_KEY environment variable or create .typesense-key file.\n";

     

       42
       42
       +
           1

     

       43
       43
       +
         ) else (

     

       44
       44
       +
           Printf.printf "Searching Typesense at %s\n%!" config.endpoint;

     

       45
       45
       +
           Printf.printf "Query: \"%s\"\n%!" query_text;

     

       46
       46
       +
           if collection <> "" then Printf.printf "Collection: %s\n%!" collection;

     

       47
       47
       +
           Printf.printf "Limit: %d, Offset: %d\n\n%!" limit offset;

     

       48
       48
       +
       

     

       49
       49
       +
           Eio.Switch.run (fun sw ->

     

       50
       50
       +
             let search_fn = if collection = "" then

     

       51
       51
       +
               Bushel.Typesense.search_all ~sw ~env config query_text ~limit ~offset

     

       75
       52
        
             else

     

       76
       76
       -
               Bushel.Typesense.search_collection config collection query_text ~limit ~offset

     

       53
       53
       +
               Bushel.Typesense.search_collection ~sw ~env config collection query_text ~limit ~offset

     

       77
       54
        
             in

     

       78
       78
       -
             let* result = search_fn () in

     

       55
       55
       +
             let result = search_fn () in

     

       79
       56
        
             match result with

     

       80
       57
        
             | Ok response ->

     

       81
       81
       -
               Printf.printf "Found %d results (%.2fms)\n\n" response.total response.query_time;

     

       58
       58
       +
               Printf.printf "Found %d results (%.2fms)\n\n%!" response.total response.query_time;

     

       82
       59
        
               List.iteri (fun i (hit : Bushel.Typesense.search_result) ->

     

       83
       83
       -
                 Printf.printf "%d. [%s] %s (score: %.2f)\n" (i + 1) hit.collection hit.title hit.score;

     

       84
       84
       -
                 if hit.content <> "" then Printf.printf "   %s\n" hit.content;

     

       60
       60
       +
                 Printf.printf "%d. [%s] %s (score: %.2f)\n%!" (i + 1) hit.collection hit.title hit.score;

     

       61
       61
       +
                 if hit.content <> "" then Printf.printf "   %s\n%!" hit.content;

     

       85
       62
        
                 if hit.highlights <> [] then (

     

       86
       86
       -
                   Printf.printf "   Highlights:\n";

     

       63
       63
       +
                   Printf.printf "   Highlights:\n%!";

     

       87
       64
        
                   List.iter (fun (field, snippets) ->

     

       88
       65
        
                     List.iter (fun snippet ->

     

       89
       89
       -
                       Printf.printf "     %s: %s\n" field snippet

     

       66
       66
       +
                       Printf.printf "     %s: %s\n%!" field snippet

     

       90
       67
        
                     ) snippets

     

       91
       68
        
                   ) hit.highlights

     

       92
       69
        
                 );

     

       93
       93
       -
                 Printf.printf "\n"

     

       94
       94
       -
               ) response.hits;

     

       95
       95
       -
               Lwt.return_unit

     

       70
       70
       +
                 Printf.printf "\n%!"

     

       71
       71
       +
               ) response.hits

     

       96
       72
        
             | Error err ->

     

       97
       97
       -
               Format.eprintf "Search error: %a\n" Bushel.Typesense.pp_error err;

     

       73
       73
       +
               Format.eprintf "Search error: %a\n%!" Bushel.Typesense.pp_error err;

     

       98
       74
        
               exit 1

     

       99
       99
       -
           ) (fun exn ->

     

       100
       100
       -
             Printf.eprintf "Error: %s\n" (Printexc.to_string exn);

     

       101
       101
       -
             exit 1

     

       102
       102
       -
           )

     

       75
       75
       +
           );

     

       76
       76
       +
           0

     

       103
       77
        
         )

     

       104
       78
        
       

     

       105
       105
       -
       (** TODO:claude List collections function *)

     

       106
       106
       -
       let list endpoint api_key =

     

       107
       107
       -
         let base_config = Bushel.Typesense.load_config_from_files () in

     

       108
       108
       -
         let config = { 

     

       109
       109
       -
           Bushel.Typesense.endpoint = if endpoint = "" then base_config.endpoint else endpoint;

     

       110
       110
       -
           api_key = if api_key = "" then base_config.api_key else api_key;

     

       111
       111
       -
           openai_key = base_config.openai_key;

     

       112
       112
       -
         } in

     

       113
       113
       -
         

     

       79
       79
       +
       (** List collections function *)

     

       80
       80
       +
       let list env _xdg _profile =

     

       81
       81
       +
         let config = Bushel.Typesense.load_config_from_files () in

     

       82
       82
       +
       

     

       114
       83
        
         if config.api_key = "" then (

     

       115
       115
       -
           Printf.eprintf "Error: API key is required. Use --api-key or set TYPESENSE_API_KEY environment variable.\n";

     

       116
       116
       -
           exit 1

     

       117
       117
       -
         );

     

       118
       118
       -
         

     

       119
       119
       -
         Printf.printf "Listing collections at %s\n\n" config.endpoint;

     

       120
       120
       -
         

     

       121
       121
       -
         Lwt_main.run (

     

       122
       122
       -
           Lwt.catch (fun () ->

     

       123
       123
       -
             let* result = Bushel.Typesense.list_collections config in

     

       84
       84
       +
           Printf.eprintf "Error: API key is required. Use TYPESENSE_API_KEY environment variable or create .typesense-key file.\n";

     

       85
       85
       +
           1

     

       86
       86
       +
         ) else (

     

       87
       87
       +
           Printf.printf "Listing collections at %s\n\n%!" config.endpoint;

     

       88
       88
       +
       

     

       89
       89
       +
           Eio.Switch.run (fun sw ->

     

       90
       90
       +
             let result = Bushel.Typesense.list_collections ~sw ~env config in

     

       124
       91
        
             match result with

     

       125
       92
        
             | Ok collections ->

     

       126
       126
       -
               Printf.printf "Collections:\n";

     

       93
       93
       +
               Printf.printf "Collections:\n%!";

     

       127
       94
        
               List.iter (fun (name, count) ->

     

       128
       128
       -
                 Printf.printf "  %s (%d documents)\n" name count

     

       129
       129
       -
               ) collections;

     

       130
       130
       -
               Lwt.return_unit

     

       95
       95
       +
                 Printf.printf "  %s (%d documents)\n%!" name count

     

       96
       96
       +
               ) collections

     

       131
       97
        
             | Error err ->

     

       132
       132
       -
               Format.eprintf "List error: %a\n" Bushel.Typesense.pp_error err;

     

       98
       98
       +
               Format.eprintf "List error: %a\n%!" Bushel.Typesense.pp_error err;

     

       133
       99
        
               exit 1

     

       134
       134
       -
           ) (fun exn ->

     

       135
       135
       -
             Printf.eprintf "Error: %s\n" (Printexc.to_string exn);

     

       136
       136
       -
             exit 1

     

       137
       137
       -
           )

     

       100
       100
       +
           );

     

       101
       101
       +
           0

     

       138
       102
        
         )

     

       139
       103
        
       

     

       140
       140
       -
       (** TODO:claude Command line arguments for query *)

     

       104
       104
       +
       (** Command line arguments *)

     

       105
       105
       +
       let openai_key =

     

       106
       106
       +
         let doc = "OpenAI API key for embeddings" in

     

       107
       107
       +
         Arg.(value & opt string "" & info ["openai-key"; "oa"] ~doc)

     

       108
       108
       +
       

     

       141
       109
        
       let query_text =

     

       142
       110
        
         let doc = "Search query text" in

     

       143
       111
        
         Arg.(required & pos 0 (some string) None & info [] ~docv:"QUERY" ~doc)

     

       144
       112
        
       

     

       145
       113
        
       let collection =

     

       146
       146
       -
         let doc = "Specific collection to search (contacts, papers, projects, news, videos, notes, ideas)" in

     

       114
       114
       +
         let doc = "Specific collection to search (contacts, papers, projects, notes, videos, ideas)" in

     

       147
       115
        
         Arg.(value & opt string "" & info ["collection"; "c"] ~doc)

     

       148
       116
        
       

     

       149
       117
        
       let limit =

     
···

       154
       122
        
         let doc = "Number of results to skip (for pagination)" in

     

       155
       123
        
         Arg.(value & opt int 0 & info ["offset"; "o"] ~doc)

     

       156
       124
        
       

     

       157
       157
       -
       (** TODO:claude Query command *)

     

       125
       125
       +
       (** Query command - uses direct Eio_main.run instead of eiocmd for simplicity *)

     

       158
       126
        
       let query_cmd =

     

       159
       127
        
         let doc = "Search bushel collections in Typesense" in

     

       160
       128
        
         let man = [

     

       161
       129
        
           `S Manpage.s_description;

     

       162
       130
        
           `P "Search across all or specific bushel collections in Typesense.";

     

       163
       163
       -
           `P "The API key can be provided via --api-key flag or TYPESENSE_API_KEY environment variable.";

     

       164
       164
       -
           `P "If .typesense-url and .typesense-api files exist, they will be used for configuration.";

     

       131
       131
       +
           `P "The API key can be read from .typesense-key file or TYPESENSE_API_KEY environment variable.";

     

       165
       132
        
           `S Manpage.s_examples;

     

       166
       133
        
           `P "Search all collections:";

     

       167
       134
        
           `Pre "  bushel-typesense query \"machine learning\"";

     
···

       171
       138
        
           `Pre "  bushel-typesense query \"AI\" --limit 5 --offset 10";

     

       172
       139
        
         ] in

     

       173
       140
        
         let info = Cmd.info "query" ~doc ~man in

     

       174
       174
       -
         Cmd.v info Term.(const query $ endpoint $ api_key $ query_text $ collection $ limit $ offset)

     

       141
       141
       +
         let run query_text collection limit offset () =

     

       142
       142
       +
           Eio_main.run (fun env -> query env () () query_text collection limit offset)

     

       143
       143
       +
         in

     

       144
       144
       +
         Cmd.v info Term.(const run $ query_text $ collection $ limit $ offset $ Bushel_common.setup_term)

     

       175
       145
        
       

     

       176
       176
       -
       (** TODO:claude List command *)

     

       146
       146
       +
       (** List command *)

     

       177
       147
        
       let list_cmd =

     

       178
       148
        
         let doc = "List all collections in Typesense" in

     

       179
       149
        
         let man = [

     
···

       181
       151
        
           `P "List all available collections and their document counts.";

     

       182
       152
        
         ] in

     

       183
       153
        
         let info = Cmd.info "list" ~doc ~man in

     

       184
       184
       -
         Cmd.v info Term.(const list $ endpoint $ api_key)

     

       154
       154
       +
         let run () =

     

       155
       155
       +
           Eio_main.run (fun env -> list env () ())

     

       156
       156
       +
         in

     

       157
       157
       +
         Cmd.v info Term.(const run $ Bushel_common.setup_term)

     

       185
       158
        
       

     

       186
       186
       -
       (** TODO:claude Updated upload command *)

     

       159
       159
       +
       (** Upload command *)

     

       187
       160
        
       let upload_cmd =

     

       188
       161
        
         let doc = "Upload bushel collections to Typesense search engine" in

     

       189
       162
        
         let man = [

     

       190
       163
        
           `S Manpage.s_description;

     

       191
       191
       -
           `P "Upload all bushel object types (contacts, papers, projects, news, videos, notes, ideas) to a Typesense search engine instance.";

     

       192
       192
       -
           `P "The API key can be provided via --api-key flag or TYPESENSE_API_KEY environment variable.";

     

       164
       164
       +
           `P "Upload all bushel object types (contacts, papers, projects, notes, videos, ideas) to a Typesense search engine instance.";

     

       165
       165
       +
           `P "The API keys can be read from files or environment variables.";

     

       193
       166
        
           `S Manpage.s_examples;

     

       194
       194
       -
           `P "Upload to local Typesense instance:";

     

       195
       195
       -
           `Pre "  bushel-typesense upload --api-key xyz123 --openai-key sk-abc... --data-dir /path/to/data";

     

       196
       196
       -
           `P "Upload to remote Typesense instance:";

     

       197
       197
       -
           `Pre "  bushel-typesense upload --endpoint https://search.example.com --api-key xyz123 --openai-key sk-abc...";

     

       167
       167
       +
           `P "Upload to Typesense instance:";

     

       168
       168
       +
           `Pre "  bushel-typesense upload --data-dir /path/to/data";

     

       198
       169
        
         ] in

     

       199
       170
        
         let info = Cmd.info "upload" ~doc ~man in

     

       200
       200
       -
         Cmd.v info Term.(const upload $ endpoint $ api_key $ openai_key $ data_dir)

     

       171
       171
       +
         let run data_dir openai_key () =

     

       172
       172
       +
           Eio_main.run (fun env -> upload env () () data_dir openai_key)

     

       173
       173
       +
         in

     

       174
       174
       +
         Cmd.v info Term.(const run $ data_dir $ openai_key $ Bushel_common.setup_term)

     

       201
       175
        
       

     

       202
       202
       -
       (** TODO:claude Main command group *)

     

       176
       176
       +
       (** Main command group *)

     

       203
       177
        
       let main_cmd =

     

       204
       178
        
         let doc = "Bushel Typesense client" in

     

       205
       179
        
         let man = [

     
···

       211
       185
        
         let info = Cmd.info "bushel-typesense" ~doc ~man in

     

       212
       186
        
         Cmd.group info [upload_cmd; query_cmd; list_cmd]

     

       213
       187
        
       

     

       214
       214
       -
       let () =

     

       215
       215
       -
         (* Check for API keys in environment if not provided *)

     

       216
       216
       -
         let api_key_env = try Some (Sys.getenv "TYPESENSE_API_KEY") with Not_found -> None in

     

       217
       217
       -
         let openai_key_env = try Some (Sys.getenv "OPENAI_API_KEY") with Not_found -> None in

     

       218
       218
       -
         match api_key_env with

     

       219
       219
       -
         | Some key when key <> "" ->

     

       220
       220
       -
           (* Override the api_key argument with environment variable *)

     

       221
       221
       -
           let api_key = Arg.(value & opt string key & info ["api-key"; "k"] ~doc:"Typesense API key") in

     

       222
       222
       -
           let openai_key = match openai_key_env with

     

       223
       223
       -
             | Some oa_key when oa_key <> "" -> Arg.(value & opt string oa_key & info ["openai-key"; "oa"] ~doc:"OpenAI API key")

     

       224
       224
       -
             | _ -> openai_key

     

       225
       225
       -
           in

     

       226
       226
       -
           let upload_cmd =

     

       227
       227
       -
             let doc = "Upload bushel collections to Typesense search engine" in

     

       228
       228
       -
             let info = Cmd.info "upload" ~doc in

     

       229
       229
       -
             Cmd.v info Term.(const upload $ endpoint $ api_key $ openai_key $ data_dir)

     

       230
       230
       -
           in

     

       231
       231
       -
           let query_cmd =

     

       232
       232
       -
             let doc = "Search bushel collections in Typesense" in

     

       233
       233
       -
             let info = Cmd.info "query" ~doc in

     

       234
       234
       -
             Cmd.v info Term.(const query $ endpoint $ api_key $ query_text $ collection $ limit $ offset)

     

       235
       235
       -
           in

     

       236
       236
       -
           let list_cmd =

     

       237
       237
       -
             let doc = "List all collections in Typesense" in

     

       238
       238
       -
             let info = Cmd.info "list" ~doc in

     

       239
       239
       -
             Cmd.v info Term.(const list $ endpoint $ api_key)

     

       240
       240
       -
           in

     

       241
       241
       -
           let main_cmd =

     

       242
       242
       -
             let doc = "Bushel Typesense client" in

     

       243
       243
       -
             let info = Cmd.info "bushel-typesense" ~doc in

     

       244
       244
       -
             Cmd.group info [upload_cmd; query_cmd; list_cmd]

     

       245
       245
       -
           in

     

       246
       246
       -
           exit (Cmd.eval main_cmd)

     

       247
       247
       -
         | _ ->

     

       248
       248
       -
           exit (Cmd.eval main_cmd)
     

       188
       188
       +
       let () = exit (Cmd.eval' main_cmd)

+3 -3

stack/bushel/bin/dune

···

       7
       7
        
        (name bushel_main)

     

       8
       8
        
        (public_name bushel)

     

       9
       9
        
        (package bushel)

     

       10
       10
       -
        (modules bushel_main bushel_bibtex bushel_doi bushel_ideas bushel_info bushel_missing bushel_note_doi bushel_obsidian bushel_paper bushel_paper_classify bushel_paper_tex bushel_video bushel_video_thumbs bushel_thumbs bushel_faces bushel_links bushel_search)

     

       10
       10
       +
        (modules bushel_main bushel_bibtex bushel_ideas bushel_info bushel_missing bushel_note_doi bushel_obsidian bushel_paper_classify bushel_paper_tex bushel_thumbs bushel_search)

     

       11
       11
        
        (flags (:standard -w -69))

     

       12
       12
       -
        (libraries bushel bushel_common cmdliner cohttp-lwt-unix lwt.unix yaml ezjsonm zotero-translation peertube fmt fmt.cli fmt.tty logs logs.cli logs.fmt cmarkit karakeep uri unix ptime.clock.os crockford))

     

       12
       12
       +
        (libraries bushel bushel_common cmdliner eio eio_main yaml ezjsonm zotero-translation fmt fmt.cli fmt.tty logs logs.cli logs.fmt cmarkit uri unix ptime.clock.os crockford))

     

       13
       13
        
       

     

       14
       14
        
       (executable

     

       15
       15
        
        (name bushel_typesense)

     
···

       17
       17
        
        (package bushel)

     

       18
       18
        
        (modules bushel_typesense)

     

       19
       19
        
        (flags (:standard -w -69))

     

       20
       20
       -
        (libraries bushel bushel_common cmdliner lwt.unix))

     

       20
       20
       +
        (libraries bushel bushel_common cmdliner eio eio_main eiocmd))

+9 -5

stack/bushel/bushel.opam

···

       18
       18
        
         "bytesrw"

     

       19
       19
        
         "jekyll-format"

     

       20
       20
        
         "yaml"

     

       21
       21
       -
         "lwt"

     

       22
       22
       -
         "cohttp-lwt-unix"

     

       21
       21
       +
         "eio"

     

       22
       22
       +
         "eio_main"

     

       23
       23
       +
         "requests"

     

       23
       24
        
         "fmt"

     

       24
       24
       -
         "peertube"

     

       25
       25
       -
         "karakeep"

     

       26
       26
       -
         "typesense-client"

     

       25
       25
       +
         "peertubee"

     

       26
       26
       +
         "karakeepe"

     

       27
       27
       +
         "typesense-cliente"

     

       27
       28
        
         "cmdliner"

     

       29
       29
       +
         "eiocmd"

     

       30
       30
       +
         "xdge"

     

       31
       31
       +
         "keyeio"

     

       28
       32
        
         "odoc" {with-doc}

     

       29
       33
        
       ]

     

       30
       34
        
       build: [

+10 -43

stack/bushel/dune-project

···

       22
       22
        
         bytesrw

     

       23
       23
        
         jekyll-format

     

       24
       24
        
         yaml

     

       25
       25
       -
         lwt

     

       26
       26
       -
         cohttp-lwt-unix

     

       25
       25
       +
         eio

     

       26
       26
       +
         eio_main

     

       27
       27
       +
         requests

     

       27
       28
        
         fmt

     

       28
       28
       -
         peertube

     

       29
       29
       -
         karakeep

     

       30
       30
       -
         typesense-client

     

       31
       31
       -
         cmdliner))

     

       32
       32
       -
       

     

       33
       33
       -
       (package

     

       34
       34
       -
        (name peertube)

     

       35
       35
       -
        (synopsis "PeerTube API client")

     

       36
       36
       -
        (description "Client for interacting with PeerTube instances")

     

       37
       37
       -
        (depends

     

       38
       38
       -
         (ocaml (>= "5.2.0"))

     

       39
       39
       -
         ezjsonm

     

       40
       40
       -
         lwt

     

       41
       41
       -
         cohttp-lwt-unix

     

       42
       42
       -
         ptime

     

       43
       43
       -
         fmt))

     

       44
       44
       -
       

     

       45
       45
       -
       (package

     

       46
       46
       -
        (name karakeep)

     

       47
       47
       -
        (synopsis "Karakeep API client for Bushel")

     

       48
       48
       -
        (description "Karakeep API client to retrieve bookmarks from Karakeep instances")

     

       49
       49
       -
        (depends

     

       50
       50
       -
         (ocaml (>= "5.2.0"))

     

       51
       51
       -
         ezjsonm

     

       52
       52
       -
         lwt

     

       53
       53
       -
         cohttp-lwt-unix

     

       54
       54
       -
         ptime

     

       55
       55
       -
         fmt))

     

       56
       56
       -
       

     

       57
       57
       -
       (package

     

       58
       58
       -
        (name typesense-client)

     

       59
       59
       -
        (synopsis "Standalone Typesense client for OCaml")

     

       60
       60
       -
        (description "A standalone Typesense client that can be compiled to JavaScript")

     

       61
       61
       -
        (depends

     

       62
       62
       -
         (ocaml (>= "5.2.0"))

     

       63
       63
       -
         ezjsonm

     

       64
       64
       -
         lwt

     

       65
       65
       -
         cohttp-lwt-unix

     

       66
       66
       -
         ptime

     

       67
       67
       -
         fmt

     

       68
       68
       -
         uri))

     

       29
       29
       +
         peertubee

     

       30
       30
       +
         karakeepe

     

       31
       31
       +
         typesense-cliente

     

       32
       32
       +
         cmdliner

     

       33
       33
       +
         eiocmd

     

       34
       34
       +
         xdge

     

       35
       35
       +
         keyeio))

+3 -3

stack/bushel/lib/dune

···

       10
       10
        
         ptime

     

       11
       11
        
         yaml.unix

     

       12
       12
        
         jekyll-format

     

       13
       13
       -
         lwt

     

       14
       14
       -
         cohttp-lwt-unix

     

       13
       13
       +
         eio

     

       14
       14
       +
         requests

     

       15
       15
        
         fmt

     

       16
       16
        
         re

     

       17
       17
        
         ptime.clock

     

       18
       18
        
         ptime.clock.os

     

       19
       19
       -
         typesense-client))

     

       19
       19
       +
         typesense-cliente))

+30 -552

stack/bushel/lib/typesense.ml

···

       1
       1
       -
       <<<<<<< HEAD

     

       2
       2
       -
       (** TODO:claude Typesense API client for Bushel *)

     

       1
       1
       +
       (** Typesense API client for Bushel *)

     

       3
       2
        
       

     

       4
       3
        
       type config = {

     

       5
       4
        
         endpoint : string;

     
···

       17
       16
        
         | Json_error msg -> Fmt.pf fmt "JSON error: %s" msg

     

       18
       17
        
         | Connection_error msg -> Fmt.pf fmt "Connection error: %s" msg

     

       19
       18
        
       

     

       20
       20
       -
       (** TODO:claude Create authentication headers for Typesense API *)

     

       19
       19
       +
       (** Create authentication headers for Typesense API *)

     

       21
       20
        
       let auth_headers api_key =

     

       22
       21
        
         Requests.Headers.empty

     

       23
       22
        
         |> Requests.Headers.set "X-TYPESENSE-API-KEY" api_key

     

       24
       23
        
         |> Requests.Headers.set "Content-Type" "application/json"

     

       25
       24
        
       

     

       26
       26
       -
       (** TODO:claude Make HTTP request to Typesense API *)

     

       25
       25
       +
       (** Make HTTP request to Typesense API *)

     

       27
       26
        
       let make_request ~sw ~env ?(meth=`GET) ?(body="") config path =

     

       28
       27
        
         let uri = Uri.of_string (config.endpoint ^ path) in

     

       29
       28
        
         let headers = auth_headers config.api_key in

     
···

       49
       48
        
         with exn ->

     

       50
       49
        
           Error (Connection_error (Printexc.to_string exn))

     

       51
       50
        
       

     

       52
       52
       -
       (** TODO:claude Create a collection with given schema *)

     

       51
       51
       +
       (** Create a collection with given schema *)

     

       53
       52
        
       let create_collection ~sw ~env config (schema : Ezjsonm.value) =

     

       54
       53
        
         let body = Ezjsonm.value_to_string schema in

     

       55
       54
        
         make_request ~sw ~env ~meth:`POST ~body config "/collections"

     

       56
       55
        
       

     

       57
       57
       -
       (** TODO:claude Check if collection exists *)

     

       56
       56
       +
       (** Check if collection exists *)

     

       58
       57
        
       let collection_exists ~sw ~env config name =

     

       59
       58
        
         let result = make_request ~sw ~env config ("/collections/" ^ name) in

     

       60
       59
        
         match result with

     
···

       62
       61
        
         | Error (Http_error (404, _)) -> false

     

       63
       62
        
         | Error _ -> false

     

       64
       63
        
       

     

       65
       65
       -
       (** TODO:claude Delete a collection *)

     

       64
       64
       +
       (** Delete a collection *)

     

       66
       65
        
       let delete_collection ~sw ~env config name =

     

       67
       66
        
         make_request ~sw ~env ~meth:`DELETE config ("/collections/" ^ name)

     

       68
       67
        
       

     

       69
       69
       -
       (** TODO:claude Upload documents to a collection in batch *)

     

       68
       68
       +
       (** Upload documents to a collection in batch *)

     

       70
       69
        
       let upload_documents ~sw ~env config collection_name (documents : Ezjsonm.value list) =

     

       71
       70
        
         let jsonl_lines = List.map (fun doc -> Ezjsonm.value_to_string doc) documents in

     

       72
       71
        
         let body = String.concat "\n" jsonl_lines in

     
···

       74
       73
        
           (Printf.sprintf "/collections/%s/documents/import?action=upsert" collection_name)

     

       75
       74
        
       

     

       76
       75
        
       

     

       77
       77
       -
       (** TODO:claude Convert Bushel objects to Typesense documents *)

     

       76
       76
       +
       (** Convert Bushel objects to Typesense documents *)

     

       78
       77
        
       

     

       79
       79
       -
       (** TODO:claude Helper function to truncate long strings for embedding *)

     

       78
       78
       +
       (** Helper function to truncate long strings for embedding *)

     

       80
       79
        
       let truncate_for_embedding ?(max_chars=20000) text =

     

       81
       80
        
         if String.length text <= max_chars then text

     

       82
       81
        
         else String.sub text 0 max_chars

     

       83
       82
        
       

     

       84
       84
       -
       (** TODO:claude Helper function to convert Ptime to Unix timestamp *)

     

       83
       83
       +
       (** Helper function to convert Ptime to Unix timestamp *)

     

       85
       84
        
       let ptime_to_timestamp ptime =

     

       86
       85
        
         let span = Ptime.to_span ptime in

     

       87
       86
        
         let seconds = Ptime.Span.to_int_s span in

     
···

       89
       88
        
         | Some s -> Int64.of_int s

     

       90
       89
        
         | None -> 0L

     

       91
       90
        
       

     

       92
       92
       -
       (** TODO:claude Helper function to convert date tuple to Unix timestamp *)

     

       91
       91
       +
       (** Helper function to convert date tuple to Unix timestamp *)

     

       93
       92
        
       let date_to_timestamp (year, month, day) =

     

       94
       93
        
         match Ptime.of_date (year, month, day) with

     

       95
       94
        
         | Some ptime -> ptime_to_timestamp ptime

     
···

       321
       320
        
           ("thumbnail_url", string (Option.value ~default:"" thumbnail_url));

     

       322
       321
        
         ]

     

       323
       322
        
       

     

       324
       324
       -
       (** TODO:claude Helper function to add embedding field to schema *)

     

       323
       323
       +
       (** Helper function to add embedding field to schema *)

     

       325
       324
        
       let add_embedding_field_to_schema schema config embedding_from_fields =

     

       326
       325
        
         let open Ezjsonm in

     

       327
       326
        
         let fields = get_dict schema |> List.assoc "fields" |> get_list (fun f -> f) in

     
···

       337
       336
        
           ]);

     

       338
       337
        
         ] in

     

       339
       338
        
         let updated_fields = fields @ [embedding_field] in

     

       340
       340
       -
         let updated_schema = 

     

       339
       339
       +
         let updated_schema =

     

       341
       340
        
           List.map (fun (k, v) ->

     

       342
       341
        
             if k = "fields" then (k, list (fun f -> f) updated_fields)

     

       343
       342
        
             else (k, v)

     
···

       345
       344
        
         in

     

       346
       345
        
         dict updated_schema

     

       347
       346
        
       

     

       348
       348
       -
       (** TODO:claude Upload all bushel objects to their respective collections *)

     

       347
       347
       +
       (** Upload all bushel objects to their respective collections *)

     

       349
       348
        
       let upload_all ~sw ~env config entries =

     

       350
       349
        
         print_string "Uploading bushel data to Typesense\n";

     

       351
       350
        
       

     
···

       412
       411
        
       

     

       413
       412
        
         List.iter upload_collection collections

     

       414
       413
        
       

     

       415
       415
       -
       (** TODO:claude Re-export search types from Typesense_cliente *)

     

       414
       414
       +
       (** Re-export search types from Typesense_cliente *)

     

       416
       415
        
       type search_result = Typesense_cliente.search_result = {

     

       417
       416
        
         id: string;

     

       418
       417
        
         title: string;

     
···

       429
       428
        
         query_time: float;

     

       430
       429
        
       }

     

       431
       430
        
       

     

       432
       432
       -
       (** TODO:claude Convert bushel config to client config *)

     

       431
       431
       +
       (** Convert bushel config to client config *)

     

       433
       432
        
       let to_client_config (config : config) =

     

       434
       433
        
         Typesense_cliente.{ endpoint = config.endpoint; api_key = config.api_key }

     

       435
       434
        
       

     

       436
       436
       -
       (** TODO:claude Search a single collection *)

     

       435
       435
       +
       (** Search a single collection *)

     

       437
       436
        
       let search_collection ~sw ~env (config : config) collection_name query ?(limit=10) ?(offset=0) () =

     

       438
       437
        
         let client_config = to_client_config config in

     

       439
       438
        
         let requests_session = Requests.create ~sw env in

     
···

       445
       444
        
         | Error (Typesense_cliente.Json_error msg) -> Error (Json_error msg)

     

       446
       445
        
         | Error (Typesense_cliente.Connection_error msg) -> Error (Connection_error msg)

     

       447
       446
        
       

     

       448
       448
       -
       (** TODO:claude Search across all collections - use client multisearch *)

     

       447
       447
       +
       (** Search across all collections - use client multisearch *)

     

       449
       448
        
       let search_all ~sw ~env (config : config) query ?(limit=10) ?(offset=0) () =

     

       450
       449
        
         let client_config = to_client_config config in

     

       451
       450
        
         let requests_session = Requests.create ~sw env in

     
···

       459
       458
        
         | Error (Typesense_cliente.Json_error msg) -> Error (Json_error msg)

     

       460
       459
        
         | Error (Typesense_cliente.Connection_error msg) -> Error (Connection_error msg)

     

       461
       460
        
       

     

       462
       462
       -
       (** TODO:claude List all collections *)

     

       461
       461
       +
       (** List all collections *)

     

       463
       462
        
       let list_collections ~sw ~env (config : config) =

     

       464
       463
        
         let client_config = to_client_config config in

     

       465
       464
        
         let requests_session = Requests.create ~sw env in

     
···

       471
       470
        
         | Error (Typesense_cliente.Json_error msg) -> Error (Json_error msg)

     

       472
       471
        
         | Error (Typesense_cliente.Connection_error msg) -> Error (Connection_error msg)

     

       473
       472
        
       

     

       474
       474
       -
       (** TODO:claude Re-export multisearch types from Typesense_cliente *)

     

       473
       473
       +
       (** Re-export multisearch types from Typesense_cliente *)

     

       475
       474
        
       type multisearch_response = Typesense_cliente.multisearch_response = {

     

       476
       475
        
         results: search_response list;

     

       477
       476
        
       }

     

       478
       477
        
       

     

       479
       479
       -
       (** TODO:claude Perform multisearch across all collections *)

     

       478
       478
       +
       (** Perform multisearch across all collections *)

     

       480
       479
        
       let multisearch ~sw ~env (config : config) query ?(limit=10) () =

     

       481
       480
        
         let client_config = to_client_config config in

     

       482
       481
        
         let requests_session = Requests.create ~sw env in

     
···

       488
       487
        
         | Error (Typesense_cliente.Json_error msg) -> Error (Json_error msg)

     

       489
       488
        
         | Error (Typesense_cliente.Connection_error msg) -> Error (Connection_error msg)

     

       490
       489
        
       

     

       491
       491
       -
       (** TODO:claude Combine multisearch results into single result set *)

     

       490
       490
       +
       (** Combine multisearch results into single result set *)

     

       492
       491
        
       let combine_multisearch_results (multisearch_resp : multisearch_response) ?(limit=10) ?(offset=0) () =

     

       493
       492
        
         Typesense_cliente.combine_multisearch_results multisearch_resp ~limit ~offset ()

     

       494
       493
        
       

     

       495
       495
       -
       (** TODO:claude Load configuration from files *)

     

       494
       494
       +
       (** Load configuration from files *)

     

       496
       495
        
       let load_config_from_files () =

     

       497
       496
        
         let read_file_if_exists filename =

     

       498
       497
        
           if Sys.file_exists filename then

     
···

       502
       501
        
             Some (String.trim content)

     

       503
       502
        
           else None

     

       504
       503
        
         in

     

       505
       505
       -
         

     

       506
       506
       -
         let endpoint = match read_file_if_exists ".typesense-url" with

     

       507
       507
       -
           | Some url -> url

     

       508
       508
       -
           | None -> "http://localhost:8108"

     

       509
       509
       -
         in

     

       510
       510
       -
         

     

       511
       511
       -
         let api_key = match read_file_if_exists ".typesense-key" with

     

       512
       512
       -
           | Some key -> key

     

       513
       513
       -
           | None -> 

     

       514
       514
       -
             try Sys.getenv "TYPESENSE_API_KEY"

     

       515
       515
       -
             with Not_found -> ""

     

       516
       516
       -
         in

     

       517
       517
       -
         

     

       518
       518
       -
         let openai_key = match read_file_if_exists ".openrouter-api" with

     

       519
       519
       -
           | Some key -> key

     

       520
       520
       -
           | None -> 

     

       521
       521
       -
             try Sys.getenv "OPENAI_API_KEY"

     

       522
       522
       -
             with Not_found -> ""

     

       523
       523
       -
         in

     

       524
       524
       -
         

     

       525
       525
       -
         { endpoint; api_key; openai_key }

     

       526
       504
        
       

     

       527
       527
       -
       (** TODO:claude Re-export pretty printer from Typesense_cliente *)

     

       528
       528
       -
       let pp_search_result_oneline = Typesense_cliente.pp_search_result_oneline

     

       529
       529
       -
       ||||||| parent of 40c9549 (bushel)

     

       530
       530
       -
       =======

     

       531
       531
       -
       open Lwt.Syntax

     

       532
       532
       -
       open Cohttp_lwt_unix

     

       533
       533
       -
       

     

       534
       534
       -
       (** TODO:claude Typesense API client for Bushel *)

     

       535
       535
       -
       

     

       536
       536
       -
       type config = {

     

       537
       537
       -
         endpoint : string;

     

       538
       538
       -
         api_key : string;

     

       539
       539
       -
         openai_key : string;

     

       540
       540
       -
       }

     

       541
       541
       -
       

     

       542
       542
       -
       type error = 

     

       543
       543
       -
         | Http_error of int * string

     

       544
       544
       -
         | Json_error of string

     

       545
       545
       -
         | Connection_error of string

     

       546
       546
       -
       

     

       547
       547
       -
       let pp_error fmt = function

     

       548
       548
       -
         | Http_error (code, msg) -> Fmt.pf fmt "HTTP %d: %s" code msg

     

       549
       549
       -
         | Json_error msg -> Fmt.pf fmt "JSON error: %s" msg

     

       550
       550
       -
         | Connection_error msg -> Fmt.pf fmt "Connection error: %s" msg

     

       551
       551
       -
       

     

       552
       552
       -
       (** TODO:claude Create authentication headers for Typesense API *)

     

       553
       553
       -
       let auth_headers api_key =

     

       554
       554
       -
         Cohttp.Header.of_list [

     

       555
       555
       -
           ("X-TYPESENSE-API-KEY", api_key);

     

       556
       556
       -
           ("Content-Type", "application/json");

     

       557
       557
       -
         ]

     

       558
       558
       -
       

     

       559
       559
       -
       (** TODO:claude Make HTTP request to Typesense API *)

     

       560
       560
       -
       let make_request ?(meth=`GET) ?(body="") config path =

     

       561
       561
       -
         let uri = Uri.of_string (config.endpoint ^ path) in

     

       562
       562
       -
         let headers = auth_headers config.api_key in

     

       563
       563
       -
         let body = if body = "" then `Empty else `String body in

     

       564
       564
       -
         Lwt.catch (fun () ->

     

       565
       565
       -
           let* resp, body = Client.call ~headers ~body meth uri in

     

       566
       566
       -
           let status = Cohttp.Code.code_of_status (Response.status resp) in

     

       567
       567
       -
           let* body_str = Cohttp_lwt.Body.to_string body in

     

       568
       568
       -
           if status >= 200 && status < 300 then

     

       569
       569
       -
             Lwt.return_ok body_str

     

       570
       570
       -
           else

     

       571
       571
       -
             Lwt.return_error (Http_error (status, body_str))

     

       572
       572
       -
         ) (fun exn ->

     

       573
       573
       -
           Lwt.return_error (Connection_error (Printexc.to_string exn))

     

       574
       574
       -
         )

     

       575
       575
       -
       

     

       576
       576
       -
       (** TODO:claude Create a collection with given schema *)

     

       577
       577
       -
       let create_collection config (schema : Ezjsonm.value) =

     

       578
       578
       -
         let body = Ezjsonm.value_to_string schema in

     

       579
       579
       -
         make_request ~meth:`POST ~body config "/collections"

     

       580
       580
       -
       

     

       581
       581
       -
       (** TODO:claude Check if collection exists *)

     

       582
       582
       -
       let collection_exists config name =

     

       583
       583
       -
         let* result = make_request config ("/collections/" ^ name) in

     

       584
       584
       -
         match result with

     

       585
       585
       -
         | Ok _ -> Lwt.return true

     

       586
       586
       -
         | Error (Http_error (404, _)) -> Lwt.return false

     

       587
       587
       -
         | Error _ -> Lwt.return false

     

       588
       588
       -
       

     

       589
       589
       -
       (** TODO:claude Delete a collection *)

     

       590
       590
       -
       let delete_collection config name =

     

       591
       591
       -
         make_request ~meth:`DELETE config ("/collections/" ^ name)

     

       592
       592
       -
       

     

       593
       593
       -
       (** TODO:claude Upload documents to a collection in batch *)

     

       594
       594
       -
       let upload_documents config collection_name (documents : Ezjsonm.value list) =

     

       595
       595
       -
         let jsonl_lines = List.map (fun doc -> Ezjsonm.value_to_string doc) documents in

     

       596
       596
       -
         let body = String.concat "\n" jsonl_lines in

     

       597
       597
       -
         make_request ~meth:`POST ~body config 

     

       598
       598
       -
           (Printf.sprintf "/collections/%s/documents/import?action=upsert" collection_name)

     

       599
       599
       -
       

     

       600
       600
       -
       

     

       601
       601
       -
       (** TODO:claude Convert Bushel objects to Typesense documents *)

     

       602
       602
       -
       

     

       603
       603
       -
       (** TODO:claude Helper function to truncate long strings for embedding *)

     

       604
       604
       -
       let truncate_for_embedding ?(max_chars=20000) text =

     

       605
       605
       -
         if String.length text <= max_chars then text

     

       606
       606
       -
         else String.sub text 0 max_chars

     

       607
       607
       -
       

     

       608
       608
       -
       (** TODO:claude Helper function to convert Ptime to Unix timestamp *)

     

       609
       609
       -
       let ptime_to_timestamp ptime =

     

       610
       610
       -
         let span = Ptime.to_span ptime in

     

       611
       611
       -
         let seconds = Ptime.Span.to_int_s span in

     

       612
       612
       -
         match seconds with

     

       613
       613
       -
         | Some s -> Int64.of_int s

     

       614
       614
       -
         | None -> 0L

     

       615
       615
       -
       

     

       616
       616
       -
       (** TODO:claude Helper function to convert date tuple to Unix timestamp *)

     

       617
       617
       -
       let date_to_timestamp (year, month, day) =

     

       618
       618
       -
         match Ptime.of_date (year, month, day) with

     

       619
       619
       -
         | Some ptime -> ptime_to_timestamp ptime

     

       620
       620
       -
         | None -> 0L

     

       621
       621
       -
       

     

       622
       622
       -
       (** Resolve author handles to full names in a list *)

     

       623
       623
       -
       let resolve_author_list contacts authors =

     

       624
       624
       -
         List.map (fun author ->

     

       625
       625
       -
           (* Strip '@' prefix if present *)

     

       626
       626
       -
           let handle =

     

       627
       627
       -
             if String.length author > 0 && author.[0] = '@' then

     

       628
       628
       -
               String.sub author 1 (String.length author - 1)

     

       629
       629
       -
             else

     

       630
       630
       -
               author

     

       631
       631
       -
           in

     

       632
       632
       -
           (* Try to look up as a contact handle *)

     

       633
       633
       -
           match Contact.find_by_handle contacts handle with

     

       634
       634
       -
           | Some contact -> Contact.name contact

     

       635
       635
       -
           | None -> author (* Keep original if not found *)

     

       636
       636
       -
         ) authors

     

       637
       637
       -
       

     

       638
       638
       -
       let contact_to_document (contact : Contact.t) =

     

       639
       639
       -
         let open Ezjsonm in

     

       640
       640
       -
         let safe_string_list_from_opt = function

     

       641
       641
       -
           | Some s -> [s]

     

       642
       642
       -
           | None -> []

     

       643
       643
       -
         in

     

       644
       644
       -
         dict [

     

       645
       645
       -
           ("id", string (Contact.handle contact));

     

       646
       646
       -
           ("handle", string (Contact.handle contact));

     

       647
       647
       -
           ("name", string (Contact.name contact));

     

       648
       648
       -
           ("names", list string (Contact.names contact));

     

       649
       649
       -
           ("email", list string (safe_string_list_from_opt (Contact.email contact)));

     

       650
       650
       -
           ("icon", list string (safe_string_list_from_opt (Contact.icon contact)));

     

       651
       651
       -
           ("github", list string (safe_string_list_from_opt (Contact.github contact)));

     

       652
       652
       -
           ("twitter", list string (safe_string_list_from_opt (Contact.twitter contact)));

     

       653
       653
       -
           ("url", list string (safe_string_list_from_opt (Contact.url contact)));

     

       654
       654
       -
         ]

     

       655
       655
       -
       

     

       656
       656
       -
       let paper_to_document entries (paper : Paper.t) =

     

       657
       657
       -
         let date_tuple = Paper.date paper in

     

       658
       658
       -
         let contacts = Entry.contacts entries in

     

       659
       659
       -
       

     

       660
       660
       -
         (* Helper to extract string arrays from JSON, handling both single strings and arrays *)

     

       661
       661
       -
         let extract_string_array_from_json json_field_name =

     

       662
       662
       -
           try

     

       663
       663
       -
             (* Access the raw JSON from the paper record *)

     

       664
       664
       -
             let paper_json = Paper.raw_json paper in

     

       665
       665
       -
             let value = Ezjsonm.get_dict paper_json |> List.assoc json_field_name in

     

       666
       666
       -
             match value with

     

       667
       667
       -
             | `String s -> [s]

     

       668
       668
       -
             | `A l -> List.filter_map (function `String s -> Some s | _ -> None) l

     

       669
       669
       -
             | _ -> []

     

       670
       670
       -
           with _ -> []

     

       671
       671
       -
         in

     

       672
       672
       -
       

     

       673
       673
       -
         (* Resolve author handles to full names *)

     

       674
       674
       -
         let authors = resolve_author_list contacts (Paper.authors paper) in

     

       675
       675
       -
       

     

       676
       676
       -
         (* Convert abstract markdown to plain text *)

     

       677
       677
       -
         let abstract = Md.markdown_to_plaintext entries (Paper.abstract paper) |> truncate_for_embedding in

     

       678
       678
       -
       

     

       679
       679
       -
         (* Extract publication metadata *)

     

       680
       680
       -
         let bibtype = Paper.bibtype paper in

     

       681
       681
       -
         let metadata =

     

       682
       682
       -
           try

     

       683
       683
       -
             match bibtype with

     

       684
       684
       -
             | "article" -> Printf.sprintf "Journal: %s" (Paper.journal paper)

     

       685
       685
       -
             | "inproceedings" -> Printf.sprintf "Proceedings: %s" (Paper.journal paper)

     

       686
       686
       -
             | "misc" | "techreport" -> Printf.sprintf "Preprint: %s" (Paper.journal paper)

     

       687
       687
       -
             | _ -> Printf.sprintf "%s: %s" (String.capitalize_ascii bibtype) (Paper.journal paper)

     

       688
       688
       -
           with _ -> bibtype

     

       689
       689
       -
         in

     

       690
       690
       -
       

     

       691
       691
       -
         (* Get bibtex from raw JSON *)

     

       692
       692
       -
         let bibtex =

     

       693
       693
       -
           try

     

       694
       694
       -
             let paper_json = Paper.raw_json paper in

     

       695
       695
       -
             Ezjsonm.get_dict paper_json

     

       696
       696
       -
             |> List.assoc "bibtex"

     

       697
       697
       -
             |> Ezjsonm.get_string

     

       698
       698
       -
           with _ -> ""

     

       699
       699
       -
         in

     

       700
       700
       -
       

     

       701
       701
       -
         let thumbnail_url = Entry.thumbnail entries (`Paper paper) in

     

       702
       702
       -
         Ezjsonm.dict [

     

       703
       703
       -
           ("id", Ezjsonm.string (Paper.slug paper));

     

       704
       704
       -
           ("title", Ezjsonm.string (Paper.title paper));

     

       705
       705
       -
           ("authors", Ezjsonm.list Ezjsonm.string authors);

     

       706
       706
       -
           ("abstract", Ezjsonm.string abstract);

     

       707
       707
       -
           ("metadata", Ezjsonm.string metadata);

     

       708
       708
       -
           ("bibtex", Ezjsonm.string bibtex);

     

       709
       709
       -
           ("date", Ezjsonm.string (let y, m, d = date_tuple in Printf.sprintf "%04d-%02d-%02d" y m d));

     

       710
       710
       -
           ("date_timestamp", Ezjsonm.int64 (date_to_timestamp date_tuple));

     

       711
       711
       -
           ("tags", Ezjsonm.list Ezjsonm.string (Paper.tags paper));

     

       712
       712
       -
           ("doi", Ezjsonm.list Ezjsonm.string (extract_string_array_from_json "doi"));

     

       713
       713
       -
           ("pdf_url", Ezjsonm.list Ezjsonm.string (extract_string_array_from_json "pdf_url"));

     

       714
       714
       -
           ("journal", Ezjsonm.list Ezjsonm.string (extract_string_array_from_json "journal"));

     

       715
       715
       -
           ("related_projects", Ezjsonm.list Ezjsonm.string (Paper.project_slugs paper));

     

       716
       716
       -
           ("thumbnail_url", Ezjsonm.string (Option.value ~default:"" thumbnail_url));

     

       717
       717
       -
         ]

     

       718
       718
       -
       

     

       719
       719
       -
       let project_to_document entries (project : Project.t) =

     

       720
       720
       -
         let open Ezjsonm in

     

       721
       721
       -
         (* Use January 1st of start year as the date for sorting *)

     

       722
       722
       -
         let date_timestamp = date_to_timestamp (project.start, 1, 1) in

     

       723
       723
       -
       

     

       724
       724
       -
         (* Convert body markdown to plain text *)

     

       725
       725
       -
         let description = Md.markdown_to_plaintext entries (Project.body project) |> truncate_for_embedding in

     

       726
       726
       -
       

     

       727
       727
       -
         let thumbnail_url = Entry.thumbnail entries (`Project project) in

     

       728
       728
       -
         dict [

     

       729
       729
       -
           ("id", string project.slug);

     

       730
       730
       -
           ("title", string (Project.title project));

     

       731
       731
       -
           ("description", string description);

     

       732
       732
       -
           ("start", int project.start);

     

       733
       733
       -
           ("finish", option int project.finish);

     

       734
       734
       -
           ("start_year", int project.start);

     

       735
       735
       -
           ("date", string (Printf.sprintf "%04d-01-01" project.start));

     

       736
       736
       -
           ("date_timestamp", int64 date_timestamp);

     

       737
       737
       -
           ("tags", list string (Project.tags project));

     

       738
       738
       -
           ("thumbnail_url", string (Option.value ~default:"" thumbnail_url));

     

       739
       739
       -
         ]

     

       740
       740
       -
       

     

       741
       741
       -
       let video_to_document entries (video : Video.t) =

     

       742
       742
       -
         let open Ezjsonm in

     

       743
       743
       -
         let datetime = Video.datetime video in

     

       744
       744
       -
         let safe_string_list_from_opt = function

     

       745
       745
       -
           | Some s -> [s]

     

       746
       746
       -
           | None -> []

     

       747
       747
       -
         in

     

       748
       748
       -
       

     

       749
       749
       -
         (* Convert body markdown to plain text *)

     

       750
       750
       -
         let description = Md.markdown_to_plaintext entries (Video.body video) |> truncate_for_embedding in

     

       751
       751
       -
       

     

       752
       752
       -
         (* Resolve paper and project slugs to titles *)

     

       753
       753
       -
         let paper_title = match Video.paper video with

     

       754
       754
       -
           | Some slug ->

     

       755
       755
       -
               (match Entry.lookup entries slug with

     

       756
       756
       -
                | Some entry -> Some (Entry.title entry)

     

       757
       757
       -
                | None -> Some slug) (* Fallback to slug if not found *)

     

       758
       758
       -
           | None -> None

     

       759
       759
       -
         in

     

       760
       760
       -
         let project_title = match Video.project video with

     

       761
       761
       -
           | Some slug ->

     

       762
       762
       -
               (match Entry.lookup entries slug with

     

       763
       763
       -
                | Some entry -> Some (Entry.title entry)

     

       764
       764
       -
                | None -> Some slug) (* Fallback to slug if not found *)

     

       765
       765
       -
           | None -> None

     

       766
       766
       -
         in

     

       767
       767
       -
       

     

       768
       768
       -
         let thumbnail_url = Entry.thumbnail entries (`Video video) in

     

       769
       769
       -
         dict [

     

       770
       770
       -
           ("id", string (Video.slug video));

     

       771
       771
       -
           ("title", string (Video.title video));

     

       772
       772
       -
           ("description", string description);

     

       773
       773
       -
           ("published_date", string (Ptime.to_rfc3339 datetime));

     

       774
       774
       -
           ("date", string (Ptime.to_rfc3339 datetime));

     

       775
       775
       -
           ("date_timestamp", int64 (ptime_to_timestamp datetime));

     

       776
       776
       -
           ("url", string (Video.url video));

     

       777
       777
       -
           ("uuid", string (Video.uuid video));

     

       778
       778
       -
           ("is_talk", bool (Video.talk video));

     

       779
       779
       -
           ("paper", list string (safe_string_list_from_opt paper_title));

     

       780
       780
       -
           ("project", list string (safe_string_list_from_opt project_title));

     

       781
       781
       -
           ("tags", list string video.tags);

     

       782
       782
       -
           ("thumbnail_url", string (Option.value ~default:"" thumbnail_url));

     

       783
       783
       -
         ]

     

       784
       784
       -
       

     

       785
       785
       -
       let note_to_document entries (note : Note.t) =

     

       786
       786
       -
         let open Ezjsonm in

     

       787
       787
       -
         let datetime = Note.datetime note in

     

       788
       788
       -
         let safe_string_list_from_opt = function

     

       789
       789
       -
           | Some s -> [s]

     

       790
       790
       -
           | None -> []

     

       791
       791
       -
         in

     

       792
       792
       -
       

     

       793
       793
       -
         (* Convert body markdown to plain text *)

     

       794
       794
       -
         let content = Md.markdown_to_plaintext entries (Note.body note) |> truncate_for_embedding in

     

       795
       795
       -
       

     

       796
       796
       -
         let thumbnail_url = Entry.thumbnail entries (`Note note) in

     

       797
       797
       -
         let word_count = Note.words note in

     

       798
       798
       -
         dict [

     

       799
       799
       -
           ("id", string (Note.slug note));

     

       800
       800
       -
           ("title", string (Note.title note));

     

       801
       801
       -
           ("date", string (Ptime.to_rfc3339 datetime));

     

       802
       802
       -
           ("date_timestamp", int64 (ptime_to_timestamp datetime));

     

       803
       803
       -
           ("content", string content);

     

       804
       804
       -
           ("tags", list string (Note.tags note));

     

       805
       805
       -
           ("draft", bool (Note.draft note));

     

       806
       806
       -
           ("synopsis", list string (safe_string_list_from_opt (Note.synopsis note)));

     

       807
       807
       -
           ("thumbnail_url", string (Option.value ~default:"" thumbnail_url));

     

       808
       808
       -
           ("words", int word_count);

     

       809
       809
       -
         ]

     

       810
       810
       -
       

     

       811
       811
       -
       let idea_to_document entries (idea : Idea.t) =

     

       812
       812
       -
         let open Ezjsonm in

     

       813
       813
       -
         let contacts = Entry.contacts entries in

     

       814
       814
       -
         (* Use January 1st of the year as the date for sorting *)

     

       815
       815
       -
         let date_timestamp = date_to_timestamp (Idea.year idea, 1, 1) in

     

       816
       816
       -
       

     

       817
       817
       -
         (* Convert body markdown to plain text *)

     

       818
       818
       -
         let description = Md.markdown_to_plaintext entries (Idea.body idea) |> truncate_for_embedding in

     

       819
       819
       -
       

     

       820
       820
       -
         (* Resolve supervisor and student handles to full names *)

     

       821
       821
       -
         let supervisors = resolve_author_list contacts (Idea.supervisors idea) in

     

       822
       822
       -
         let students = resolve_author_list contacts (Idea.students idea) in

     

       823
       823
       -
       

     

       824
       824
       -
         (* Resolve project slug to project title *)

     

       825
       825
       -
         let project_title =

     

       826
       826
       -
           match Entry.lookup entries (Idea.project idea) with

     

       827
       827
       -
           | Some entry -> Entry.title entry

     

       828
       828
       -
           | None -> Idea.project idea (* Fallback to slug if not found *)

     

       829
       829
       -
         in

     

       830
       830
       -
       

     

       831
       831
       -
         let thumbnail_url = Entry.thumbnail entries (`Idea idea) in

     

       832
       832
       -
         dict [

     

       833
       833
       -
           ("id", string idea.slug);

     

       834
       834
       -
           ("title", string (Idea.title idea));

     

       835
       835
       -
           ("description", string description);

     

       836
       836
       -
           ("level", string (Idea.level_to_string (Idea.level idea)));

     

       837
       837
       -
           ("project", string project_title);

     

       838
       838
       -
           ("status", string (Idea.status_to_string (Idea.status idea)));

     

       839
       839
       -
           ("year", int (Idea.year idea));

     

       840
       840
       -
           ("date", string (Printf.sprintf "%04d-01-01" (Idea.year idea)));

     

       841
       841
       -
           ("date_timestamp", int64 date_timestamp);

     

       842
       842
       -
           ("supervisors", list string supervisors);

     

       843
       843
       -
           ("students", list string students);

     

       844
       844
       -
           ("tags", list string idea.tags);

     

       845
       845
       -
           ("thumbnail_url", string (Option.value ~default:"" thumbnail_url));

     

       846
       846
       -
         ]

     

       847
       847
       -
       

     

       848
       848
       -
       (** TODO:claude Helper function to add embedding field to schema *)

     

       849
       849
       -
       let add_embedding_field_to_schema schema config embedding_from_fields =

     

       850
       850
       -
         let open Ezjsonm in

     

       851
       851
       -
         let fields = get_dict schema |> List.assoc "fields" |> get_list (fun f -> f) in

     

       852
       852
       -
         let embedding_field = dict [

     

       853
       853
       -
           ("name", string "embedding");

     

       854
       854
       -
           ("type", string "float[]");

     

       855
       855
       -
           ("embed", dict [

     

       856
       856
       -
             ("from", list string embedding_from_fields);

     

       857
       857
       -
             ("model_config", dict [

     

       858
       858
       -
               ("model_name", string "openai/text-embedding-3-small");

     

       859
       859
       -
               ("api_key", string config.openai_key);

     

       860
       860
       -
             ]);

     

       861
       861
       -
           ]);

     

       862
       862
       -
         ] in

     

       863
       863
       -
         let updated_fields = fields @ [embedding_field] in

     

       864
       864
       -
         let updated_schema = 

     

       865
       865
       -
           List.map (fun (k, v) ->

     

       866
       866
       -
             if k = "fields" then (k, list (fun f -> f) updated_fields)

     

       867
       867
       -
             else (k, v)

     

       868
       868
       -
           ) (get_dict schema)

     

       869
       869
       -
         in

     

       870
       870
       -
         dict updated_schema

     

       871
       871
       -
       

     

       872
       872
       -
       (** TODO:claude Upload all bushel objects to their respective collections *)

     

       873
       873
       -
       let upload_all config entries =

     

       874
       874
       -
         let* () = Lwt_io.write Lwt_io.stdout "Uploading bushel data to Typesense\n" in

     

       875
       875
       -
       

     

       876
       876
       -
         let contacts = Entry.contacts entries in

     

       877
       877
       -
         let papers = Entry.papers entries in

     

       878
       878
       -
         let projects = Entry.projects entries in

     

       879
       879
       -
         let notes = Entry.notes entries in

     

       880
       880
       -
         let videos = Entry.videos entries in

     

       881
       881
       -
         let ideas = Entry.ideas entries in

     

       882
       882
       -
       

     

       883
       883
       -
         let collections = [

     

       884
       884
       -
           ("contacts", add_embedding_field_to_schema Contact.typesense_schema config ["name"; "names"], (List.map contact_to_document contacts : Ezjsonm.value list));

     

       885
       885
       -
           ("papers", add_embedding_field_to_schema Paper.typesense_schema config ["title"; "abstract"; "authors"], (List.map (paper_to_document entries) papers : Ezjsonm.value list));

     

       886
       886
       -
           ("videos", add_embedding_field_to_schema Video.typesense_schema config ["title"; "description"], (List.map (video_to_document entries) videos : Ezjsonm.value list));

     

       887
       887
       -
           ("projects", add_embedding_field_to_schema Project.typesense_schema config ["title"; "description"; "tags"], (List.map (project_to_document entries) projects : Ezjsonm.value list));

     

       888
       888
       -
           ("notes", add_embedding_field_to_schema Note.typesense_schema config ["title"; "content"; "tags"], (List.map (note_to_document entries) notes : Ezjsonm.value list));

     

       889
       889
       -
           ("ideas", add_embedding_field_to_schema Idea.typesense_schema config ["title"; "description"; "tags"], (List.map (idea_to_document entries) ideas : Ezjsonm.value list));

     

       890
       890
       -
         ] in

     

       891
       891
       -
       

     

       892
       892
       -
         let upload_collection ((name, schema, documents) : string * Ezjsonm.value * Ezjsonm.value list) =

     

       893
       893
       -
           let* () = Lwt_io.write Lwt_io.stdout (Fmt.str "Processing collection: %s\n" name) in

     

       894
       894
       -
           let* exists = collection_exists config name in

     

       895
       895
       -
           let* () = 

     

       896
       896
       -
             if exists then (

     

       897
       897
       -
               let* () = Lwt_io.write Lwt_io.stdout (Fmt.str "Collection %s exists, deleting...\n" name) in

     

       898
       898
       -
               let* result = delete_collection config name in

     

       899
       899
       -
               match result with

     

       900
       900
       -
               | Ok _ -> Lwt_io.write Lwt_io.stdout (Fmt.str "Deleted collection %s\n" name)

     

       901
       901
       -
               | Error err -> 

     

       902
       902
       -
                 let err_str = Fmt.str "%a" pp_error err in

     

       903
       903
       -
                 Lwt_io.write Lwt_io.stdout (Fmt.str "Failed to delete collection %s: %s\n" name err_str)

     

       904
       904
       -
             ) else

     

       905
       905
       -
               Lwt.return_unit

     

       906
       906
       -
           in

     

       907
       907
       -
           let* () = Lwt_io.write Lwt_io.stdout (Fmt.str "Creating collection %s with %d documents\n" name (List.length documents)) in

     

       908
       908
       -
           let* result = create_collection config schema in

     

       909
       909
       -
           match result with

     

       910
       910
       -
           | Ok _ ->

     

       911
       911
       -
             let* () = Lwt_io.write Lwt_io.stdout (Fmt.str "Created collection %s\n" name) in

     

       912
       912
       -
             if documents = [] then

     

       913
       913
       -
               Lwt_io.write Lwt_io.stdout (Fmt.str "No documents to upload for %s\n" name)

     

       914
       914
       -
             else (

     

       915
       915
       -
               let* result = upload_documents config name documents in

     

       916
       916
       -
               match result with

     

       917
       917
       -
               | Ok response -> 

     

       918
       918
       -
                 (* Count successes and failures *)

     

       919
       919
       -
                 let lines = String.split_on_char '\n' response in

     

       920
       920
       -
                 let successes = List.fold_left (fun acc line -> 

     

       921
       921
       -
                   if String.contains line ':' && Str.string_match (Str.regexp ".*success.*true.*") line 0 then acc + 1 else acc) 0 lines in

     

       922
       922
       -
                 let failures = List.fold_left (fun acc line -> 

     

       923
       923
       -
                   if String.contains line ':' && Str.string_match (Str.regexp ".*success.*false.*") line 0 then acc + 1 else acc) 0 lines in

     

       924
       924
       -
                 let* () = Lwt_io.write Lwt_io.stdout (Fmt.str "Upload results for %s: %d successful, %d failed out of %d total\n" 

     

       925
       925
       -
                   name successes failures (List.length documents)) in

     

       926
       926
       -
                 if failures > 0 then

     

       927
       927
       -
                   let* () = Lwt_io.write Lwt_io.stdout (Fmt.str "Failed documents in %s:\n" name) in

     

       928
       928
       -
                   let failed_lines = List.filter (fun line -> Str.string_match (Str.regexp ".*success.*false.*") line 0) lines in

     

       929
       929
       -
                   Lwt_list.iter_s (fun line -> Lwt_io.write Lwt_io.stdout (line ^ "\n")) failed_lines

     

       930
       930
       -
                 else

     

       931
       931
       -
                   Lwt.return_unit

     

       932
       932
       -
               | Error err -> 

     

       933
       933
       -
                 let err_str = Fmt.str "%a" pp_error err in

     

       934
       934
       -
                 Lwt_io.write Lwt_io.stdout (Fmt.str "Failed to upload documents to %s: %s\n" name err_str)

     

       935
       935
       -
             )

     

       936
       936
       -
           | Error err ->

     

       937
       937
       -
             let err_str = Fmt.str "%a" pp_error err in

     

       938
       938
       -
             Lwt_io.write Lwt_io.stdout (Fmt.str "Failed to create collection %s: %s\n" name err_str)

     

       939
       939
       -
         in

     

       940
       940
       -
       

     

       941
       941
       -
         Lwt_list.iter_s upload_collection collections

     

       942
       942
       -
       

     

       943
       943
       -
       (** TODO:claude Re-export search types from Typesense_client *)

     

       944
       944
       -
       type search_result = Typesense_client.search_result = {

     

       945
       945
       -
         id: string;

     

       946
       946
       -
         title: string;

     

       947
       947
       -
         content: string;

     

       948
       948
       -
         score: float;

     

       949
       949
       -
         collection: string;

     

       950
       950
       -
         highlights: (string * string list) list;

     

       951
       951
       -
         document: Ezjsonm.value;

     

       952
       952
       -
       }

     

       953
       953
       -
       

     

       954
       954
       -
       type search_response = Typesense_client.search_response = {

     

       955
       955
       -
         hits: search_result list;

     

       956
       956
       -
         total: int;

     

       957
       957
       -
         query_time: float;

     

       958
       958
       -
       }

     

       959
       959
       -
       

     

       960
       960
       -
       (** TODO:claude Convert bushel config to client config *)

     

       961
       961
       -
       let to_client_config (config : config) =

     

       962
       962
       -
         Typesense_client.{ endpoint = config.endpoint; api_key = config.api_key }

     

       963
       963
       -
       

     

       964
       964
       -
       (** TODO:claude Search a single collection *)

     

       965
       965
       -
       let search_collection (config : config) collection_name query ?(limit=10) ?(offset=0) () =

     

       966
       966
       -
         let client_config = to_client_config config in

     

       967
       967
       -
         let* result = Typesense_client.search_collection client_config collection_name query ~limit ~offset () in

     

       968
       968
       -
         match result with

     

       969
       969
       -
         | Ok response -> Lwt.return_ok response

     

       970
       970
       -
         | Error (Typesense_client.Http_error (code, msg)) -> Lwt.return_error (Http_error (code, msg))

     

       971
       971
       -
         | Error (Typesense_client.Json_error msg) -> Lwt.return_error (Json_error msg)

     

       972
       972
       -
         | Error (Typesense_client.Connection_error msg) -> Lwt.return_error (Connection_error msg)

     

       973
       973
       -
       

     

       974
       974
       -
       (** TODO:claude Search across all collections - use client multisearch *)

     

       975
       975
       -
       let search_all (config : config) query ?(limit=10) ?(offset=0) () =

     

       976
       976
       -
         let client_config = to_client_config config in

     

       977
       977
       -
         let* result = Typesense_client.multisearch client_config query ~limit:50 () in

     

       978
       978
       -
         match result with

     

       979
       979
       -
         | Ok multisearch_resp ->

     

       980
       980
       -
           let combined_response = Typesense_client.combine_multisearch_results multisearch_resp ~limit ~offset () in

     

       981
       981
       -
           Lwt.return_ok combined_response

     

       982
       982
       -
         | Error (Typesense_client.Http_error (code, msg)) -> Lwt.return_error (Http_error (code, msg))

     

       983
       983
       -
         | Error (Typesense_client.Json_error msg) -> Lwt.return_error (Json_error msg)

     

       984
       984
       -
         | Error (Typesense_client.Connection_error msg) -> Lwt.return_error (Connection_error msg)

     

       985
       985
       -
       

     

       986
       986
       -
       (** TODO:claude List all collections *)

     

       987
       987
       -
       let list_collections (config : config) =

     

       988
       988
       -
         let client_config = to_client_config config in

     

       989
       989
       -
         let* result = Typesense_client.list_collections client_config in

     

       990
       990
       -
         match result with

     

       991
       991
       -
         | Ok collections -> Lwt.return_ok collections

     

       992
       992
       -
         | Error (Typesense_client.Http_error (code, msg)) -> Lwt.return_error (Http_error (code, msg))

     

       993
       993
       -
         | Error (Typesense_client.Json_error msg) -> Lwt.return_error (Json_error msg)

     

       994
       994
       -
         | Error (Typesense_client.Connection_error msg) -> Lwt.return_error (Connection_error msg)

     

       995
       995
       -
       

     

       996
       996
       -
       (** TODO:claude Re-export multisearch types from Typesense_client *)

     

       997
       997
       -
       type multisearch_response = Typesense_client.multisearch_response = {

     

       998
       998
       -
         results: search_response list;

     

       999
       999
       -
       }

     

       1000
       1000
       -
       

     

       1001
       1001
       -
       (** TODO:claude Perform multisearch across all collections *)

     

       1002
       1002
       -
       let multisearch (config : config) query ?(limit=10) () =

     

       1003
       1003
       -
         let client_config = to_client_config config in

     

       1004
       1004
       -
         let* result = Typesense_client.multisearch client_config query ~limit () in

     

       1005
       1005
       -
         match result with

     

       1006
       1006
       -
         | Ok multisearch_resp -> Lwt.return_ok multisearch_resp

     

       1007
       1007
       -
         | Error (Typesense_client.Http_error (code, msg)) -> Lwt.return_error (Http_error (code, msg))

     

       1008
       1008
       -
         | Error (Typesense_client.Json_error msg) -> Lwt.return_error (Json_error msg)

     

       1009
       1009
       -
         | Error (Typesense_client.Connection_error msg) -> Lwt.return_error (Connection_error msg)

     

       1010
       1010
       -
       

     

       1011
       1011
       -
       (** TODO:claude Combine multisearch results into single result set *)

     

       1012
       1012
       -
       let combine_multisearch_results (multisearch_resp : multisearch_response) ?(limit=10) ?(offset=0) () =

     

       1013
       1013
       -
         Typesense_client.combine_multisearch_results multisearch_resp ~limit ~offset ()

     

       1014
       1014
       -
       

     

       1015
       1015
       -
       (** TODO:claude Load configuration from files *)

     

       1016
       1016
       -
       let load_config_from_files () =

     

       1017
       1017
       -
         let read_file_if_exists filename =

     

       1018
       1018
       -
           if Sys.file_exists filename then

     

       1019
       1019
       -
             let ic = open_in filename in

     

       1020
       1020
       -
             let content = really_input_string ic (in_channel_length ic) in

     

       1021
       1021
       -
             close_in ic;

     

       1022
       1022
       -
             Some (String.trim content)

     

       1023
       1023
       -
           else None

     

       1024
       1024
       -
         in

     

       1025
       1025
       -
         

     

       1026
       505
        
         let endpoint = match read_file_if_exists ".typesense-url" with

     

       1027
       506
        
           | Some url -> url

     

       1028
       507
        
           | None -> "http://localhost:8108"

     

       1029
       508
        
         in

     

       1030
       1030
       -
         

     

       509
       509
       +
       

     

       1031
       510
        
         let api_key = match read_file_if_exists ".typesense-key" with

     

       1032
       511
        
           | Some key -> key

     

       1033
       1033
       -
           | None -> 

     

       512
       512
       +
           | None ->

     

       1034
       513
        
             try Sys.getenv "TYPESENSE_API_KEY"

     

       1035
       514
        
             with Not_found -> ""

     

       1036
       515
        
         in

     

       1037
       1037
       -
         

     

       516
       516
       +
       

     

       1038
       517
        
         let openai_key = match read_file_if_exists ".openrouter-api" with

     

       1039
       518
        
           | Some key -> key

     

       1040
       1040
       -
           | None -> 

     

       519
       519
       +
           | None ->

     

       1041
       520
        
             try Sys.getenv "OPENAI_API_KEY"

     

       1042
       521
        
             with Not_found -> ""

     

       1043
       522
        
         in

     

       1044
       1044
       -
         

     

       523
       523
       +
       

     

       1045
       524
        
         { endpoint; api_key; openai_key }

     

       1046
       525
        
       

     

       1047
       1047
       -
       (** TODO:claude Re-export pretty printer from Typesense_client *)

     

       1048
       1048
       -
       let pp_search_result_oneline = Typesense_client.pp_search_result_oneline

     

       1049
       1049
       -
       >>>>>>> 40c9549 (bushel)

     

       526
       526
       +
       (** Re-export pretty printer from Typesense_cliente *)

     

       527
       527
       +
       let pp_search_result_oneline = Typesense_cliente.pp_search_result_oneline

+11 -129

stack/bushel/lib/typesense.mli

···

       1
       1
        
       (** Typesense API client for Bushel

     

       2
       2
       -
       <<<<<<< HEAD

     

       3
       2
        
       

     

       4
       3
        
           This module provides an OCaml client for the Typesense search engine API.

     

       5
       4
        
           It handles collection management and document indexing for all Bushel object

     
···

       12
       11
        
               Eio.Switch.run (fun sw ->

     

       13
       12
        
                 Typesense.upload_all ~sw ~env config entries))

     

       14
       13
        
           ]}

     

       15
       15
       -
       

     

       16
       16
       -
           TODO:claude *)

     

       14
       14
       +
       *)

     

       17
       15
        
       

     

       18
       16
        
       (** Configuration for connecting to a Typesense server *)

     

       19
       17
        
       type config = {

     
···

       32
       30
        
       val pp_error : Format.formatter -> error -> unit

     

       33
       31
        
       

     

       34
       32
        
       (** Create a collection with the given schema.

     

       35
       35
       -
           The schema should follow Typesense's collection schema format.

     

       36
       36
       -
           TODO:claude *)

     

       33
       33
       +
           The schema should follow Typesense's collection schema format. *)

     

       37
       34
        
       val create_collection :

     

       38
       35
        
         sw:Eio.Switch.t ->

     

       39
       36
        
         env:< clock: [> float Eio.Time.clock_ty ] Eio.Resource.t; net: [> [> `Generic ] Eio.Net.ty ] Eio.Resource.t; .. > ->

     
···

       42
       39
        
         (string, error) result

     

       43
       40
        
       

     

       44
       41
        
       (** Check if a collection exists by name.

     

       45
       45
       -
           Returns true if the collection exists, false otherwise.

     

       46
       46
       -
           TODO:claude *)

     

       42
       42
       +
           Returns true if the collection exists, false otherwise. *)

     

       47
       43
        
       val collection_exists :

     

       48
       44
        
         sw:Eio.Switch.t ->

     

       49
       45
        
         env:< clock: [> float Eio.Time.clock_ty ] Eio.Resource.t; net: [> [> `Generic ] Eio.Net.ty ] Eio.Resource.t; .. > ->

     
···

       51
       47
        
         string ->

     

       52
       48
        
         bool

     

       53
       49
        
       

     

       54
       54
       -
       (** Delete a collection by name.

     

       55
       55
       -
           TODO:claude *)

     

       50
       50
       +
       (** Delete a collection by name. *)

     

       56
       51
        
       val delete_collection :

     

       57
       52
        
         sw:Eio.Switch.t ->

     

       58
       53
        
         env:< clock: [> float Eio.Time.clock_ty ] Eio.Resource.t; net: [> [> `Generic ] Eio.Net.ty ] Eio.Resource.t; .. > ->

     
···

       61
       56
        
         (string, error) result

     

       62
       57
        
       

     

       63
       58
        
       (** Upload documents to a collection in batch using JSONL format.

     

       64
       64
       -
           More efficient than uploading documents one by one.

     

       65
       65
       -
           TODO:claude *)

     

       59
       59
       +
           More efficient than uploading documents one by one. *)

     

       66
       60
        
       val upload_documents :

     

       67
       61
        
         sw:Eio.Switch.t ->

     

       68
       62
        
         env:< clock: [> float Eio.Time.clock_ty ] Eio.Resource.t; net: [> [> `Generic ] Eio.Net.ty ] Eio.Resource.t; .. > ->

     
···

       76
       70
        
           - Extract all bushel data types from the Entry.t

     

       77
       71
        
           - Create or recreate collections for each type

     

       78
       72
        
           - Upload all documents in batches

     

       79
       79
       -
           - Report progress to stdout

     

       80
       80
       -
           TODO:claude *)

     

       73
       73
       +
           - Report progress to stdout *)

     

       81
       74
        
       val upload_all :

     

       82
       75
        
         sw:Eio.Switch.t ->

     

       83
       76
        
         env:< clock: [> float Eio.Time.clock_ty ] Eio.Resource.t; net: [> [> `Generic ] Eio.Net.ty ] Eio.Resource.t; .. > ->

     
···

       103
       96
        
         query_time: float;                        (** Query execution time in milliseconds *)

     

       104
       97
        
       }

     

       105
       98
        
       

     

       106
       106
       -
       (** Search a specific collection.

     

       107
       107
       -
           TODO:claude *)

     

       99
       99
       +
       (** Search a specific collection. *)

     

       108
       100
        
       val search_collection :

     

       109
       101
        
         sw:Eio.Switch.t ->

     

       110
       102
        
         env:< clock: float Eio.Time.clock_ty Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; net: [`Generic | `Unix] Eio.Net.ty Eio.Resource.t; .. > ->

     
···

       117
       109
        
         (search_response, error) result

     

       118
       110
        
       

     

       119
       111
        
       (** Search across all bushel collections.

     

       120
       120
       -
           Results are sorted by relevance score and paginated.

     

       121
       121
       -
           TODO:claude *)

     

       112
       112
       +
           Results are sorted by relevance score and paginated. *)

     

       122
       113
        
       val search_all :

     

       123
       114
        
         sw:Eio.Switch.t ->

     

       124
       115
        
         env:< clock: float Eio.Time.clock_ty Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; net: [`Generic | `Unix] Eio.Net.ty Eio.Resource.t; .. > ->

     
···

       135
       126
        
       }

     

       136
       127
        
       

     

       137
       128
        
       (** Perform multisearch across all collections using Typesense's multi_search endpoint.

     

       138
       138
       -
           More efficient than individual searches as it's done in a single request.

     

       139
       139
       -
           TODO:claude *)

     

       129
       129
       +
           More efficient than individual searches as it's done in a single request. *)

     

       140
       130
        
       val multisearch :

     

       141
       131
        
         sw:Eio.Switch.t ->

     

       142
       132
        
         env:< clock: float Eio.Time.clock_ty Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; net: [`Generic | `Unix] Eio.Net.ty Eio.Resource.t; .. > ->

     
···

       147
       137
        
         (multisearch_response, error) result

     

       148
       138
        
       

     

       149
       139
        
       (** Combine multisearch results into a single result set.

     

       150
       150
       -
           Results are sorted by relevance score and paginated.

     

       151
       151
       -
           TODO:claude *)

     

       140
       140
       +
           Results are sorted by relevance score and paginated. *)

     

       152
       141
        
       val combine_multisearch_results : multisearch_response -> ?limit:int -> ?offset:int -> unit -> search_response

     

       153
       142
        
       

     

       154
       143
        
       (** List all collections with document counts.

     

       155
       155
       -
           Returns a list of (collection_name, document_count) pairs.

     

       156
       156
       -
           TODO:claude *)

     

       144
       144
       +
           Returns a list of (collection_name, document_count) pairs. *)

     

       157
       145
        
       val list_collections :

     

       158
       146
        
         sw:Eio.Switch.t ->

     

       159
       147
        
         env:< clock: float Eio.Time.clock_ty Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; net: [`Generic | `Unix] Eio.Net.ty Eio.Resource.t; .. > ->

     

       160
       148
        
         config ->

     

       161
       149
        
         ((string * int) list, error) result

     

       162
       162
       -
       ||||||| parent of 40c9549 (bushel)

     

       163
       163
       -
       =======

     

       164
       164
       -
           

     

       165
       165
       -
           This module provides an OCaml client for the Typesense search engine API.

     

       166
       166
       -
           It handles collection management and document indexing for all Bushel object

     

       167
       167
       -
           types including contacts, papers, projects, news, videos, notes, and ideas.

     

       168
       168
       -
           

     

       169
       169
       -
           Example usage:

     

       170
       170
       -
           {[

     

       171
       171
       -
             let config = { endpoint = "https://search.example.com"; api_key = "xyz123" } in

     

       172
       172
       -
             Lwt_main.run (Typesense.upload_all config "/path/to/bushel/data")

     

       173
       173
       -
           ]}

     

       174
       174
       -
           

     

       175
       175
       -
           TODO:claude *)

     

       176
       176
       -
       

     

       177
       177
       -
       (** Configuration for connecting to a Typesense server *)

     

       178
       178
       -
       type config = {

     

       179
       179
       -
         endpoint : string;    (** Typesense server URL (e.g., "https://search.example.com") *)

     

       180
       180
       -
         api_key : string;     (** API key for authentication *)

     

       181
       181
       -
         openai_key : string;  (** OpenAI API key for embeddings *)

     

       182
       182
       -
       }

     

       183
       183
       -
       

     

       184
       184
       -
       (** Possible errors that can occur during Typesense operations *)

     

       185
       185
       -
       type error = 

     

       186
       186
       -
         | Http_error of int * string        (** HTTP error with status code and message *)

     

       187
       187
       -
         | Json_error of string              (** JSON parsing or encoding error *)

     

       188
       188
       -
         | Connection_error of string        (** Network connection error *)

     

       189
       189
       -
       

     

       190
       190
       -
       (** Pretty-printer for error types *)

     

       191
       191
       -
       val pp_error : Format.formatter -> error -> unit

     

       192
       192
       -
       

     

       193
       193
       -
       (** Create a collection with the given schema. 

     

       194
       194
       -
           The schema should follow Typesense's collection schema format.

     

       195
       195
       -
           TODO:claude *)

     

       196
       196
       -
       val create_collection : config -> Ezjsonm.value -> (string, error) result Lwt.t

     

       197
       197
       -
       

     

       198
       198
       -
       (** Check if a collection exists by name. 

     

       199
       199
       -
           Returns true if the collection exists, false otherwise.

     

       200
       200
       -
           TODO:claude *)

     

       201
       201
       -
       val collection_exists : config -> string -> bool Lwt.t

     

       202
       202
       -
       

     

       203
       203
       -
       (** Delete a collection by name.

     

       204
       204
       -
           TODO:claude *)

     

       205
       205
       -
       val delete_collection : config -> string -> (string, error) result Lwt.t

     

       206
       206
       -
       

     

       207
       207
       -
       (** Upload documents to a collection in batch using JSONL format.

     

       208
       208
       -
           More efficient than uploading documents one by one.

     

       209
       209
       -
           TODO:claude *)

     

       210
       210
       -
       val upload_documents : config -> string -> Ezjsonm.value list -> (string, error) result Lwt.t

     

       211
       211
       -
       

     

       212
       212
       -
       (** Upload all bushel objects to Typesense.

     

       213
       213
       -
           This function will:

     

       214
       214
       -
           - Extract all bushel data types from the Entry.t

     

       215
       215
       -
           - Create or recreate collections for each type

     

       216
       216
       -
           - Upload all documents in batches

     

       217
       217
       -
           - Report progress to stdout

     

       218
       218
       -
           TODO:claude *)

     

       219
       219
       -
       val upload_all : config -> Entry.t -> unit Lwt.t

     

       220
       220
       -
       

     

       221
       221
       -
       (** Search result structure containing document information and relevance score *)

     

       222
       222
       -
       type search_result = {

     

       223
       223
       -
         id: string;                               (** Document ID *)

     

       224
       224
       -
         title: string;                            (** Document title *)

     

       225
       225
       -
         content: string;                          (** Document content/description *)

     

       226
       226
       -
         score: float;                             (** Relevance score *)

     

       227
       227
       -
         collection: string;                       (** Collection name *)

     

       228
       228
       -
         highlights: (string * string list) list; (** Highlighted search terms by field *)

     

       229
       229
       -
         document: Ezjsonm.value;                  (** Raw document for flexible field access *)

     

       230
       230
       -
       }

     

       231
       231
       -
       

     

       232
       232
       -
       (** Search response containing results and metadata *)

     

       233
       233
       -
       type search_response = {

     

       234
       234
       -
         hits: search_result list;                 (** List of matching documents *)

     

       235
       235
       -
         total: int;                               (** Total number of matches *)

     

       236
       236
       -
         query_time: float;                        (** Query execution time in milliseconds *)

     

       237
       237
       -
       }

     

       238
       238
       -
       

     

       239
       239
       -
       (** Search a specific collection.

     

       240
       240
       -
           TODO:claude *)

     

       241
       241
       -
       val search_collection : config -> string -> string -> ?limit:int -> ?offset:int -> unit -> (search_response, error) result Lwt.t

     

       242
       242
       -
       

     

       243
       243
       -
       (** Search across all bushel collections.

     

       244
       244
       -
           Results are sorted by relevance score and paginated.

     

       245
       245
       -
           TODO:claude *)

     

       246
       246
       -
       val search_all : config -> string -> ?limit:int -> ?offset:int -> unit -> (search_response, error) result Lwt.t

     

       247
       247
       -
       

     

       248
       248
       -
       (** Multisearch response containing results from multiple collections *)

     

       249
       249
       -
       type multisearch_response = {

     

       250
       250
       -
         results: search_response list;                (** Results from each collection *)

     

       251
       251
       -
       }

     

       252
       252
       -
       

     

       253
       253
       -
       (** Perform multisearch across all collections using Typesense's multi_search endpoint.

     

       254
       254
       -
           More efficient than individual searches as it's done in a single request.

     

       255
       255
       -
           TODO:claude *)

     

       256
       256
       -
       val multisearch : config -> string -> ?limit:int -> unit -> (multisearch_response, error) result Lwt.t

     

       257
       257
       -
       

     

       258
       258
       -
       (** Combine multisearch results into a single result set.

     

       259
       259
       -
           Results are sorted by relevance score and paginated.

     

       260
       260
       -
           TODO:claude *)

     

       261
       261
       -
       val combine_multisearch_results : multisearch_response -> ?limit:int -> ?offset:int -> unit -> search_response

     

       262
       262
       -
       

     

       263
       263
       -
       (** List all collections with document counts.

     

       264
       264
       -
           Returns a list of (collection_name, document_count) pairs.

     

       265
       265
       -
           TODO:claude *)

     

       266
       266
       -
       val list_collections : config -> ((string * int) list, error) result Lwt.t

     

       267
       267
       -
       >>>>>>> 40c9549 (bushel)

     

       268
       150
        
       

     

       269
       151
        
       (** Load configuration from .typesense-url and .typesense-api files.

     

       270
       152
        
           Falls back to environment variables and defaults.