···

       
1
       
1
       
-
       
_build

     

       
2
       
2
       
-
       
.*.swp

     

       
3
       
3
       
-
       
**/.claude/settings.local.json

     

       
4
       
4
       
-
       
.photos-api

     

       
5
       
5
       
-
       
.karakeep-api

     

       
6
       
6
       
-
       
KARAKEEP.md

     

       
7
       
7
       
-
       
karakeep-src

     

       
8
       
8
       
-
       
.DS_Store

     

       
9
       
9
       
-
       
.openapi-key

     

···

       
1
       
1
       
-
       
profile=janestreet

     

···

       
1
       
1
       
-
       
open Cmdliner

     

       
2
       
2
       
-
       
open Printf

     

       
3
       
3
       
-
       


     

       
4
       
4
       
-
       
(** TODO:claude Generate bibtex entry from paper data *)

     

       
5
       
5
       
-
       
let generate_bibtex_entry paper =

     

       
6
       
6
       
-
       
  let open Bushel.Paper in

     

       
7
       
7
       
-
       
  (* Use slug as the bibtex key/label *)

     

       
8
       
8
       
-
       
  let bibkey = slug paper in

     

       
9
       
9
       
-
       
  let bibtype = try bibtype paper with _ -> "misc" in

     

       
10
       
10
       
-
       
  let title = try title paper with _ -> "Untitled" in

     

       
11
       
11
       
-
       
  let authors =

     

       
12
       
12
       
-
       
    let auth_list = try authors paper with _ -> [] in

     

       
13
       
13
       
-
       
    String.concat " and " auth_list

     

       
14
       
14
       
-
       
  in

     

       
15
       
15
       
-
       
  let year = try year paper with _ -> 0 in

     

       
16
       
16
       
-
       


     

       
17
       
17
       
-
       
  (* Build the bibtex entry *)

     

       
18
       
18
       
-
       
  let buf = Buffer.create 1024 in

     

       
19
       
19
       
-
       
  Buffer.add_string buf (sprintf "@%s{%s,\n" bibtype bibkey);

     

       
20
       
20
       
-
       
  Buffer.add_string buf (sprintf "  title = {%s},\n" title);

     

       
21
       
21
       
-
       
  Buffer.add_string buf (sprintf "  author = {%s},\n" authors);

     

       
22
       
22
       
-
       
  Buffer.add_string buf (sprintf "  year = {%d}" year);

     

       
23
       
23
       
-
       


     

       
24
       
24
       
-
       
  (* Add optional fields *)

     

       
25
       
25
       
-
       
  (match String.lowercase_ascii bibtype with

     

       
26
       
26
       
-
       
  | "article" ->

     

       
27
       
27
       
-
       
    (try

     

       
28
       
28
       
-
       
      Buffer.add_string buf (sprintf ",\n  journal = {%s}" (journal paper))

     

       
29
       
29
       
-
       
    with _ -> ());

     

       
30
       
30
       
-
       
    (match volume paper with

     

       
31
       
31
       
-
       
    | Some v -> Buffer.add_string buf (sprintf ",\n  volume = {%s}" v)

     

       
32
       
32
       
-
       
    | None -> ());

     

       
33
       
33
       
-
       
    (match issue paper with

     

       
34
       
34
       
-
       
    | Some i -> Buffer.add_string buf (sprintf ",\n  number = {%s}" i)

     

       
35
       
35
       
-
       
    | None -> ());

     

       
36
       
36
       
-
       
    (match pages paper with

     

       
37
       
37
       
-
       
    | "" -> ()

     

       
38
       
38
       
-
       
    | p -> Buffer.add_string buf (sprintf ",\n  pages = {%s}" p))

     

       
39
       
39
       
-
       
  | "inproceedings" ->

     

       
40
       
40
       
-
       
    (try

     

       
41
       
41
       
-
       
      Buffer.add_string buf (sprintf ",\n  booktitle = {%s}" (booktitle paper))

     

       
42
       
42
       
-
       
    with _ -> ());

     

       
43
       
43
       
-
       
    (match pages paper with

     

       
44
       
44
       
-
       
    | "" -> ()

     

       
45
       
45
       
-
       
    | p -> Buffer.add_string buf (sprintf ",\n  pages = {%s}" p));

     

       
46
       
46
       
-
       
    (match publisher paper with

     

       
47
       
47
       
-
       
    | "" -> ()

     

       
48
       
48
       
-
       
    | p -> Buffer.add_string buf (sprintf ",\n  publisher = {%s}" p))

     

       
49
       
49
       
-
       
  | "techreport" ->

     

       
50
       
50
       
-
       
    (try

     

       
51
       
51
       
-
       
      Buffer.add_string buf (sprintf ",\n  institution = {%s}" (institution paper))

     

       
52
       
52
       
-
       
    with _ -> ());

     

       
53
       
53
       
-
       
    (match number paper with

     

       
54
       
54
       
-
       
    | Some n -> Buffer.add_string buf (sprintf ",\n  number = {%s}" n)

     

       
55
       
55
       
-
       
    | None -> ())

     

       
56
       
56
       
-
       
  | "book" ->

     

       
57
       
57
       
-
       
    (match publisher paper with

     

       
58
       
58
       
-
       
    | "" -> ()

     

       
59
       
59
       
-
       
    | p -> Buffer.add_string buf (sprintf ",\n  publisher = {%s}" p));

     

       
60
       
60
       
-
       
    (try

     

       
61
       
61
       
-
       
      Buffer.add_string buf (sprintf ",\n  isbn = {%s}" (isbn paper))

     

       
62
       
62
       
-
       
    with _ -> ())

     

       
63
       
63
       
-
       
  | _ -> ());

     

       
64
       
64
       
-
       


     

       
65
       
65
       
-
       
  (* Add DOI if available *)

     

       
66
       
66
       
-
       
  (match doi paper with

     

       
67
       
67
       
-
       
  | Some d -> Buffer.add_string buf (sprintf ",\n  doi = {%s}" d)

     

       
68
       
68
       
-
       
  | None -> ());

     

       
69
       
69
       
-
       


     

       
70
       
70
       
-
       
  (* Add URL if available *)

     

       
71
       
71
       
-
       
  (match url paper with

     

       
72
       
72
       
-
       
  | Some u -> Buffer.add_string buf (sprintf ",\n  url = {%s}" u)

     

       
73
       
73
       
-
       
  | None -> ());

     

       
74
       
74
       
-
       


     

       
75
       
75
       
-
       
  Buffer.add_string buf "\n}\n";

     

       
76
       
76
       
-
       
  Buffer.contents buf

     

       
77
       
77
       
-
       


     

       
78
       
78
       
-
       
(** TODO:claude Main function to export bibtex for all papers *)

     

       
79
       
79
       
-
       
let export_bibtex base_dir output_file latest_only _env _xdg _profile =

     

       
80
       
80
       
-
       
  (* Load all papers *)

     

       
81
       
81
       
-
       
  let bushel = Bushel.load base_dir in

     

       
82
       
82
       
-
       
  let papers = Bushel.Entry.papers bushel in

     

       
83
       
83
       
-
       


     

       
84
       
84
       
-
       
  (* Filter to only latest versions if requested *)

     

       
85
       
85
       
-
       
  let papers =

     

       
86
       
86
       
-
       
    if latest_only then

     

       
87
       
87
       
-
       
      List.filter (fun p -> p.Bushel.Paper.latest) papers

     

       
88
       
88
       
-
       
    else

     

       
89
       
89
       
-
       
      papers

     

       
90
       
90
       
-
       
  in

     

       
91
       
91
       
-
       


     

       
92
       
92
       
-
       
  (* Sort papers by year (most recent first) *)

     

       
93
       
93
       
-
       
  let papers = List.sort Bushel.Paper.compare papers in

     

       
94
       
94
       
-
       


     

       
95
       
95
       
-
       
  (* Generate bibtex for each paper *)

     

       
96
       
96
       
-
       
  let bibtex_entries = List.map generate_bibtex_entry papers in

     

       
97
       
97
       
-
       
  let bibtex_content = String.concat "\n" bibtex_entries in

     

       
98
       
98
       
-
       


     

       
99
       
99
       
-
       
  (* Output to file or stdout *)

     

       
100
       
100
       
-
       
  match output_file with

     

       
101
       
101
       
-
       
  | None ->

     

       
102
       
102
       
-
       
    print_string bibtex_content;

     

       
103
       
103
       
-
       
    0

     

       
104
       
104
       
-
       
  | Some file ->

     

       
105
       
105
       
-
       
    let oc = open_out file in

     

       
106
       
106
       
-
       
    output_string oc bibtex_content;

     

       
107
       
107
       
-
       
    close_out oc;

     

       
108
       
108
       
-
       
    printf "Bibtex exported to %s (%d entries)\n" file (List.length papers);

     

       
109
       
109
       
-
       
    0

     

       
110
       
110
       
-
       


     

       
111
       
111
       
-
       
(** TODO:claude Command line arguments *)

     

       
112
       
112
       
-
       
let output_file_arg =

     

       
113
       
113
       
-
       
  let doc = "Output file for bibtex (defaults to stdout)" in

     

       
114
       
114
       
-
       
  Arg.(value & opt (some string) None & info ["o"; "output"] ~docv:"FILE" ~doc)

     

       
115
       
115
       
-
       


     

       
116
       
116
       
-
       
let latest_only_arg =

     

       
117
       
117
       
-
       
  let doc = "Export only the latest version of each paper" in

     

       
118
       
118
       
-
       
  Arg.(value & flag & info ["latest"] ~doc)

     

       
119
       
119
       
-
       


     

       
120
       
120
       
-
       
(** TODO:claude Command term *)

     

       
121
       
121
       
-
       
let term : (Eio_unix.Stdenv.base -> Xdge.t -> Keyeio.Profile.t -> int) Cmdliner.Term.t =

     

       
122
       
122
       
-
       
  Term.(const export_bibtex $ Bushel_common.base_dir $ output_file_arg $ latest_only_arg)

     

       
123
       
123
       
-
       


     

       
124
       
124
       
-
       
let cmd =

     

       
125
       
125
       
-
       
  let doc = "Export bibtex for all papers" in

     

       
126
       
126
       
-
       
  let info = Cmd.info "bibtex" ~doc in

     

       
127
       
127
       
-
       
  Cmd.v info term
     

···

       
1
       
1
       
-
       
open Cmdliner

     

       
2
       
2
       
-
       


     

       
3
       
3
       
-
       
(** TODO:claude Get default base directory from BUSHEL_DATA env variable or current directory *)

     

       
4
       
4
       
-
       
let get_default_base_dir () =

     

       
5
       
5
       
-
       
  match Sys.getenv_opt "BUSHEL_DATA" with

     

       
6
       
6
       
-
       
  | Some dir -> dir

     

       
7
       
7
       
-
       
  | None -> "."

     

       
8
       
8
       
-
       


     

       
9
       
9
       
-
       
(** TODO:claude Optional base directory term with BUSHEL_DATA env variable support *)

     

       
10
       
10
       
-
       
let base_dir =

     

       
11
       
11
       
-
       
  let doc = "Base directory containing Bushel data (defaults to BUSHEL_DATA env var or current directory)" in

     

       
12
       
12
       
-
       
  Arg.(value & opt dir (get_default_base_dir ()) & info ["d"; "dir"] ~docv:"DIR" ~doc)

     

       
13
       
13
       
-
       


     

       
14
       
14
       
-
       
(** TODO:claude Output directory as option *)

     

       
15
       
15
       
-
       
let output_dir ~default =

     

       
16
       
16
       
-
       
  let doc = "Output directory for generated files" in

     

       
17
       
17
       
-
       
  Arg.(value & opt string default & info ["o"; "output"] ~docv:"DIR" ~doc)

     

       
18
       
18
       
-
       


     

       
19
       
19
       
-
       
(** TODO:claude URL term with custom default *)

     

       
20
       
20
       
-
       
let url_term ~default ~doc =

     

       
21
       
21
       
-
       
  Arg.(value & opt string default & info ["u"; "url"] ~docv:"URL" ~doc)

     

       
22
       
22
       
-
       


     

       
23
       
23
       
-
       
(** TODO:claude API key file term *)

     

       
24
       
24
       
-
       
let api_key_file ~default =

     

       
25
       
25
       
-
       
  let doc = "File containing API key" in

     

       
26
       
26
       
-
       
  Arg.(value & opt string default & info ["k"; "key-file"] ~docv:"FILE" ~doc)

     

       
27
       
27
       
-
       


     

       
28
       
28
       
-
       
(** TODO:claude API key term *)

     

       
29
       
29
       
-
       
let api_key =

     

       
30
       
30
       
-
       
  let doc = "API key for authentication" in

     

       
31
       
31
       
-
       
  Arg.(value & opt (some string) None & info ["api-key"] ~docv:"KEY" ~doc)

     

       
32
       
32
       
-
       


     

       
33
       
33
       
-
       
(** TODO:claude Overwrite flag *)

     

       
34
       
34
       
-
       
let overwrite =

     

       
35
       
35
       
-
       
  let doc = "Overwrite existing files" in

     

       
36
       
36
       
-
       
  Arg.(value & flag & info ["overwrite"] ~doc)

     

       
37
       
37
       
-
       


     

       
38
       
38
       
-
       
(** TODO:claude Verbose flag *)

     

       
39
       
39
       
-
       
let verbose =

     

       
40
       
40
       
-
       
  let doc = "Enable verbose output" in

     

       
41
       
41
       
-
       
  Arg.(value & flag & info ["v"; "verbose"] ~doc)

     

       
42
       
42
       
-
       


     

       
43
       
43
       
-
       
(** TODO:claude File path term *)

     

       
44
       
44
       
-
       
let file_term ~default ~doc =

     

       
45
       
45
       
-
       
  Arg.(value & opt string default & info ["f"; "file"] ~docv:"FILE" ~doc)

     

       
46
       
46
       
-
       


     

       
47
       
47
       
-
       
(** TODO:claude Channel/handle term *)

     

       
48
       
48
       
-
       
let channel ~default =

     

       
49
       
49
       
-
       
  let doc = "Channel or handle name" in

     

       
50
       
50
       
-
       
  Arg.(value & opt string default & info ["c"; "channel"] ~docv:"CHANNEL" ~doc)

     

       
51
       
51
       
-
       


     

       
52
       
52
       
-
       
(** TODO:claude Optional handle term *)

     

       
53
       
53
       
-
       
let handle_opt =

     

       
54
       
54
       
-
       
  let doc = "Process specific handle" in

     

       
55
       
55
       
-
       
  Arg.(value & opt (some string) None & info ["h"; "handle"] ~docv:"HANDLE" ~doc)

     

       
56
       
56
       
-
       


     

       
57
       
57
       
-
       
(** TODO:claude Tag term for filtering *)

     

       
58
       
58
       
-
       
let tag =

     

       
59
       
59
       
-
       
  let doc = "Tag to filter or apply" in

     

       
60
       
60
       
-
       
  Arg.(value & opt (some string) None & info ["t"; "tag"] ~docv:"TAG" ~doc)

     

       
61
       
61
       
-
       


     

       
62
       
62
       
-
       
(** TODO:claude Limit term *)

     

       
63
       
63
       
-
       
let limit =

     

       
64
       
64
       
-
       
  let doc = "Limit number of items to process" in

     

       
65
       
65
       
-
       
  Arg.(value & opt (some int) None & info ["l"; "limit"] ~docv:"N" ~doc)

     

       
66
       
66
       
-
       


     

       
67
       
67
       
-
       
(* Note: Logging setup is now handled by eiocmd for all bushel binaries *)
     

···

       
1
       
1
       
-
       
module ZT = Zotero_translation

     

       
2
       
2
       
-
       
open Lwt.Infix

     

       
3
       
3
       
-
       
module J = Ezjsonm

     

       
4
       
4
       
-
       
open Cmdliner

     

       
5
       
5
       
-
       


     

       
6
       
6
       
-
       
(* Extract all DOIs from notes by scanning for doi.org URLs *)

     

       
7
       
7
       
-
       
let extract_dois_from_notes notes =

     

       
8
       
8
       
-
       
  let doi_url_pattern = Re.Perl.compile_pat "https?://(?:dx\\.)?doi\\.org/([^)\\s\"'>]+)" in

     

       
9
       
9
       
-
       
  let dois = ref [] in

     

       
10
       
10
       
-
       
  List.iter (fun note ->

     

       
11
       
11
       
-
       
    let body = Bushel.Note.body note in

     

       
12
       
12
       
-
       
    let matches = Re.all doi_url_pattern body in

     

       
13
       
13
       
-
       
    List.iter (fun group ->

     

       
14
       
14
       
-
       
      try

     

       
15
       
15
       
-
       
        let encoded_doi = Re.Group.get group 1 in

     

       
16
       
16
       
-
       
        let doi = Uri.pct_decode encoded_doi in

     

       
17
       
17
       
-
       
        if not (List.mem doi !dois) then

     

       
18
       
18
       
-
       
          dois := doi :: !dois

     

       
19
       
19
       
-
       
      with _ -> ()

     

       
20
       
20
       
-
       
    ) matches

     

       
21
       
21
       
-
       
  ) notes;

     

       
22
       
22
       
-
       
  !dois

     

       
23
       
23
       
-
       


     

       
24
       
24
       
-
       
(* Extract publisher URLs from notes (Elsevier, ScienceDirect, IEEE, Nature, ACM, Sage, UPenn, Springer, Taylor & Francis, OUP) *)

     

       
25
       
25
       
-
       
let extract_publisher_urls_from_notes notes =

     

       
26
       
26
       
-
       
  (* Matches publisher URLs: linkinghub.elsevier.com, sciencedirect.com/science/article, ieeexplore.ieee.org, academic.oup.com, nature.com, journals.sagepub.com, garfield.library.upenn.edu, link.springer.com, tandfonline.com/doi, and dl.acm.org/doi/10.* URLs *)

     

       
27
       
27
       
-
       
  let publisher_pattern = Re.Perl.compile_pat "https?://(?:(?:www\\.)?(?:linkinghub\\.elsevier\\.com|(?:www\\.)?sciencedirect\\.com/science/article|ieeexplore\\.ieee\\.org|academic\\.oup\\.com|nature\\.com|journals\\.sagepub\\.com|garfield\\.library\\.upenn\\.edu|link\\.springer\\.com)/[^)\\s\"'>]+|(?:dl\\.acm\\.org|(?:www\\.)?tandfonline\\.com)/doi(?:/pdf)?/10\\.[^)\\s\"'>]+)" in

     

       
28
       
28
       
-
       
  let urls = ref [] in

     

       
29
       
29
       
-
       
  List.iter (fun note ->

     

       
30
       
30
       
-
       
    let body = Bushel.Note.body note in

     

       
31
       
31
       
-
       
    let matches = Re.all publisher_pattern body in

     

       
32
       
32
       
-
       
    List.iter (fun group ->

     

       
33
       
33
       
-
       
      try

     

       
34
       
34
       
-
       
        let url = Re.Group.get group 0 in

     

       
35
       
35
       
-
       
        if not (List.mem url !urls) then

     

       
36
       
36
       
-
       
          urls := url :: !urls

     

       
37
       
37
       
-
       
      with _ -> ()

     

       
38
       
38
       
-
       
    ) matches

     

       
39
       
39
       
-
       
  ) notes;

     

       
40
       
40
       
-
       
  !urls

     

       
41
       
41
       
-
       


     

       
42
       
42
       
-
       
(* Resolve a single DOI via Zotero and convert to doi_entry *)

     

       
43
       
43
       
-
       
let resolve_doi zt ~verbose doi =

     

       
44
       
44
       
-
       
  Printf.printf "Resolving DOI: %s\n%!" doi;

     

       
45
       
45
       
-
       
  let doi_url = Printf.sprintf "https://doi.org/%s" doi in

     

       
46
       
46
       
-
       
  Lwt.catch

     

       
47
       
47
       
-
       
    (fun () ->

     

       
48
       
48
       
-
       
      ZT.json_of_doi zt ~slug:"temp" doi >>= fun json ->

     

       
49
       
49
       
-
       
      if verbose then begin

     

       
50
       
50
       
-
       
        Printf.printf "  Raw Zotero response:\n%s\n%!" (J.value_to_string json)

     

       
51
       
51
       
-
       
      end;

     

       
52
       
52
       
-
       
      try

     

       
53
       
53
       
-
       
        let keys = J.get_dict (json :> J.value) in

     

       
54
       
54
       
-
       
        let title = J.find json ["title"] |> J.get_string in

     

       
55
       
55
       
-
       
        let authors = J.find json ["author"] |> J.get_list J.get_string in

     

       
56
       
56
       
-
       
        let year = J.find json ["year"] |> J.get_string |> int_of_string in

     

       
57
       
57
       
-
       
        let bibtype = J.find json ["bibtype"] |> J.get_string in

     

       
58
       
58
       
-
       
        let publisher =

     

       
59
       
59
       
-
       
          try

     

       
60
       
60
       
-
       
            (* Try journal first, then booktitle, then proceedingsTitle, then publisher *)

     

       
61
       
61
       
-
       
            match List.assoc_opt "journal" keys with

     

       
62
       
62
       
-
       
            | Some j -> J.get_string j

     

       
63
       
63
       
-
       
            | None ->

     

       
64
       
64
       
-
       
              match List.assoc_opt "booktitle" keys with

     

       
65
       
65
       
-
       
              | Some b -> J.get_string b

     

       
66
       
66
       
-
       
              | None ->

     

       
67
       
67
       
-
       
                match List.assoc_opt "proceedingsTitle" keys with

     

       
68
       
68
       
-
       
                | Some pt -> J.get_string pt

     

       
69
       
69
       
-
       
                | None ->

     

       
70
       
70
       
-
       
                  match List.assoc_opt "publisher" keys with

     

       
71
       
71
       
-
       
                  | Some p -> J.get_string p

     

       
72
       
72
       
-
       
                  | None -> ""

     

       
73
       
73
       
-
       
          with _ -> ""

     

       
74
       
74
       
-
       
        in

     

       
75
       
75
       
-
       
        let entry = Bushel.Doi_entry.create_resolved ~doi ~title ~authors ~year ~bibtype ~publisher ~source_urls:[doi_url] () in

     

       
76
       
76
       
-
       
        Printf.printf "  ✓ Resolved: %s (%d)\n%!" title year;

     

       
77
       
77
       
-
       
        Lwt.return entry

     

       
78
       
78
       
-
       
      with e ->

     

       
79
       
79
       
-
       
        Printf.eprintf "  ✗ Failed to parse response for %s: %s\n%!" doi (Printexc.to_string e);

     

       
80
       
80
       
-
       
        Lwt.return (Bushel.Doi_entry.create_failed ~doi ~error:(Printexc.to_string e) ~source_urls:[doi_url] ())

     

       
81
       
81
       
-
       
    )

     

       
82
       
82
       
-
       
    (fun exn ->

     

       
83
       
83
       
-
       
      Printf.eprintf "  ✗ Failed to resolve %s: %s\n%!" doi (Printexc.to_string exn);

     

       
84
       
84
       
-
       
      Lwt.return (Bushel.Doi_entry.create_failed ~doi ~error:(Printexc.to_string exn) ~source_urls:[doi_url] ())

     

       
85
       
85
       
-
       
    )

     

       
86
       
86
       
-
       


     

       
87
       
87
       
-
       
(* Resolve a publisher URL via Zotero /web endpoint *)

     

       
88
       
88
       
-
       
let resolve_url zt ~verbose url =

     

       
89
       
89
       
-
       
  Printf.printf "Resolving URL: %s\n%!" url;

     

       
90
       
90
       
-
       
  Lwt.catch

     

       
91
       
91
       
-
       
    (fun () ->

     

       
92
       
92
       
-
       
      (* Use Zotero's resolve_url which calls /web endpoint with the URL directly *)

     

       
93
       
93
       
-
       
      ZT.resolve_url zt url >>= function

     

       
94
       
94
       
-
       
      | Error (`Msg err) ->

     

       
95
       
95
       
-
       
        Printf.eprintf "  ✗ Failed to resolve URL: %s\n%!" err;

     

       
96
       
96
       
-
       
        Lwt.return (Bushel.Doi_entry.create_failed ~doi:url ~error:err ~source_urls:[url] ())

     

       
97
       
97
       
-
       
      | Ok json ->

     

       
98
       
98
       
-
       
        if verbose then begin

     

       
99
       
99
       
-
       
          Printf.printf "  Raw Zotero response:\n%s\n%!" (J.value_to_string json)

     

       
100
       
100
       
-
       
        end;

     

       
101
       
101
       
-
       
        try

     

       
102
       
102
       
-
       
          (* Extract metadata from the JSON response *)

     

       
103
       
103
       
-
       
          let json_list = match json with

     

       
104
       
104
       
-
       
            | `A lst -> lst

     

       
105
       
105
       
-
       
            | single -> [single]

     

       
106
       
106
       
-
       
          in

     

       
107
       
107
       
-
       
          match json_list with

     

       
108
       
108
       
-
       
          | [] ->

     

       
109
       
109
       
-
       
            Printf.eprintf "  ✗ Empty response\n%!";

     

       
110
       
110
       
-
       
            Lwt.return (Bushel.Doi_entry.create_failed ~doi:url ~error:"Empty response" ~source_urls:[url] ())

     

       
111
       
111
       
-
       
          | item :: _ ->

     

       
112
       
112
       
-
       
            (* Extract DOI if present, otherwise use URL *)

     

       
113
       
113
       
-
       
            let doi = try J.find item ["DOI"] |> J.get_string with _ ->

     

       
114
       
114
       
-
       
              try J.find item ["doi"] |> J.get_string with _ -> url

     

       
115
       
115
       
-
       
            in

     

       
116
       
116
       
-
       
            let title = try J.find item ["title"] |> J.get_string with _ ->

     

       
117
       
117
       
-
       
              "Unknown Title"

     

       
118
       
118
       
-
       
            in

     

       
119
       
119
       
-
       
            (* Extract authors from Zotero's "creators" field *)

     

       
120
       
120
       
-
       
            let authors = try

     

       
121
       
121
       
-
       
              J.find item ["creators"] |> J.get_list (fun creator_obj ->

     

       
122
       
122
       
-
       
                try

     

       
123
       
123
       
-
       
                  let last_name = J.find creator_obj ["lastName"] |> J.get_string in

     

       
124
       
124
       
-
       
                  let first_name = try J.find creator_obj ["firstName"] |> J.get_string with _ -> "" in

     

       
125
       
125
       
-
       
                  if first_name = "" then last_name else first_name ^ " " ^ last_name

     

       
126
       
126
       
-
       
                with _ -> "Unknown Author"

     

       
127
       
127
       
-
       
              )

     

       
128
       
128
       
-
       
            with _ -> []

     

       
129
       
129
       
-
       
            in

     

       
130
       
130
       
-
       
            (* Extract year from Zotero's "date" field *)

     

       
131
       
131
       
-
       
            (* Handles both ISO format "2025-07" and text format "November 28, 2023" *)

     

       
132
       
132
       
-
       
            let year = try

     

       
133
       
133
       
-
       
              let date_str = J.find item ["date"] |> J.get_string in

     

       
134
       
134
       
-
       
              (* First try splitting on '-' for ISO dates like "2025-07" or "2024-11-04" *)

     

       
135
       
135
       
-
       
              let parts = String.split_on_char '-' date_str in

     

       
136
       
136
       
-
       
              match parts with

     

       
137
       
137
       
-
       
              | year_str :: _ when String.length year_str = 4 ->

     

       
138
       
138
       
-
       
                (try int_of_string year_str with _ -> 0)

     

       
139
       
139
       
-
       
              | _ ->

     

       
140
       
140
       
-
       
                (* Try splitting on space and comma for dates like "November 28, 2023" *)

     

       
141
       
141
       
-
       
                let space_parts = String.split_on_char ' ' date_str in

     

       
142
       
142
       
-
       
                let year_candidate = List.find_opt (fun s ->

     

       
143
       
143
       
-
       
                  let s = String.trim (String.map (fun c -> if c = ',' then ' ' else c) s) in

     

       
144
       
144
       
-
       
                  String.length s = 4 && String.for_all (function '0'..'9' -> true | _ -> false) s

     

       
145
       
145
       
-
       
                ) space_parts in

     

       
146
       
146
       
-
       
                (match year_candidate with

     

       
147
       
147
       
-
       
                 | Some year_str -> int_of_string (String.trim year_str)

     

       
148
       
148
       
-
       
                 | None -> 0)

     

       
149
       
149
       
-
       
            with _ -> 0

     

       
150
       
150
       
-
       
            in

     

       
151
       
151
       
-
       
            (* Extract type/bibtype from Zotero's "itemType" field *)

     

       
152
       
152
       
-
       
            let bibtype = try J.find item ["itemType"] |> J.get_string with _ -> "article" in

     

       
153
       
153
       
-
       
            (* Extract publisher/journal from Zotero's "publicationTitle" or "proceedingsTitle" field *)

     

       
154
       
154
       
-
       
            let publisher = try

     

       
155
       
155
       
-
       
              J.find item ["publicationTitle"] |> J.get_string

     

       
156
       
156
       
-
       
            with _ ->

     

       
157
       
157
       
-
       
              try J.find item ["proceedingsTitle"] |> J.get_string

     

       
158
       
158
       
-
       
              with _ -> ""

     

       
159
       
159
       
-
       
            in

     

       
160
       
160
       
-
       
            (* Include both the original URL and the DOI URL in source_urls *)

     

       
161
       
161
       
-
       
            let doi_url = if doi = url then [] else [Printf.sprintf "https://doi.org/%s" doi] in

     

       
162
       
162
       
-
       
            let source_urls = url :: doi_url in

     

       
163
       
163
       
-
       
            let entry = Bushel.Doi_entry.create_resolved ~doi ~title ~authors ~year ~bibtype ~publisher ~source_urls () in

     

       
164
       
164
       
-
       
            Printf.printf "  ✓ Resolved: %s (%d) [DOI: %s]\n%!" title year doi;

     

       
165
       
165
       
-
       
            Lwt.return entry

     

       
166
       
166
       
-
       
        with e ->

     

       
167
       
167
       
-
       
          Printf.eprintf "  ✗ Failed to parse response: %s\n%!" (Printexc.to_string e);

     

       
168
       
168
       
-
       
          Lwt.return (Bushel.Doi_entry.create_failed ~doi:url ~error:(Printexc.to_string e) ~source_urls:[url] ())

     

       
169
       
169
       
-
       
    )

     

       
170
       
170
       
-
       
    (fun exn ->

     

       
171
       
171
       
-
       
      Printf.eprintf "  ✗ Failed to resolve %s: %s\n%!" url (Printexc.to_string exn);

     

       
172
       
172
       
-
       
      Lwt.return (Bushel.Doi_entry.create_failed ~doi:url ~error:(Printexc.to_string exn) ~source_urls:[url] ())

     

       
173
       
173
       
-
       
    )

     

       
174
       
174
       
-
       


     

       
175
       
175
       
-
       
let run base_dir force verbose =

     

       
176
       
176
       
-
       
  Printf.printf "Loading bushel database...\n%!";

     

       
177
       
177
       
-
       
  let entries = Bushel.load base_dir in

     

       
178
       
178
       
-
       
  let notes = Bushel.Entry.notes entries in

     

       
179
       
179
       
-
       


     

       
180
       
180
       
-
       
  Printf.printf "Scanning %d notes for DOI URLs...\n%!" (List.length notes);

     

       
181
       
181
       
-
       
  let found_dois = extract_dois_from_notes notes in

     

       
182
       
182
       
-
       
  Printf.printf "Found %d unique DOIs\n%!" (List.length found_dois);

     

       
183
       
183
       
-
       


     

       
184
       
184
       
-
       
  Printf.printf "Scanning %d notes for publisher URLs...\n%!" (List.length notes);

     

       
185
       
185
       
-
       
  let found_urls = extract_publisher_urls_from_notes notes in

     

       
186
       
186
       
-
       
  Printf.printf "Found %d unique publisher URLs\n%!" (List.length found_urls);

     

       
187
       
187
       
-
       


     

       
188
       
188
       
-
       
  let data_dir = Bushel.Entry.data_dir entries in

     

       
189
       
189
       
-
       
  let doi_yml_path = Filename.concat data_dir "doi.yml" in

     

       
190
       
190
       
-
       
  Printf.printf "Loading existing DOI cache from %s...\n%!" doi_yml_path;

     

       
191
       
191
       
-
       
  let existing_entries = Bushel.Doi_entry.load doi_yml_path in

     

       
192
       
192
       
-
       
  Printf.printf "Loaded %d cached DOI entries\n%!" (List.length existing_entries);

     

       
193
       
193
       
-
       


     

       
194
       
194
       
-
       
  (* Filter DOIs that need resolution *)

     

       
195
       
195
       
-
       
  let dois_to_resolve =

     

       
196
       
196
       
-
       
    List.filter (fun doi ->

     

       
197
       
197
       
-
       
      match Bushel.Doi_entry.find_by_doi_including_ignored existing_entries doi with

     

       
198
       
198
       
-
       
      | Some _ when not force ->

     

       
199
       
199
       
-
       
        Printf.printf "Skipping DOI %s (already cached)\n%!" doi;

     

       
200
       
200
       
-
       
        false

     

       
201
       
201
       
-
       
      | Some _ when force ->

     

       
202
       
202
       
-
       
        Printf.printf "Re-resolving DOI %s (--force)\n%!" doi;

     

       
203
       
203
       
-
       
        true

     

       
204
       
204
       
-
       
      | Some _ -> false  (* Catch-all for Some case *)

     

       
205
       
205
       
-
       
      | None -> true

     

       
206
       
206
       
-
       
    ) found_dois

     

       
207
       
207
       
-
       
  in

     

       
208
       
208
       
-
       


     

       
209
       
209
       
-
       
  (* Filter URLs that need resolution *)

     

       
210
       
210
       
-
       
  let urls_to_resolve =

     

       
211
       
211
       
-
       
    List.filter (fun url ->

     

       
212
       
212
       
-
       
      match Bushel.Doi_entry.find_by_url_including_ignored existing_entries url with

     

       
213
       
213
       
-
       
      | Some _ when not force ->

     

       
214
       
214
       
-
       
        Printf.printf "Skipping URL %s (already cached)\n%!" url;

     

       
215
       
215
       
-
       
        false

     

       
216
       
216
       
-
       
      | Some _ when force ->

     

       
217
       
217
       
-
       
        Printf.printf "Re-resolving URL %s (--force)\n%!" url;

     

       
218
       
218
       
-
       
        true

     

       
219
       
219
       
-
       
      | Some _ -> false  (* Catch-all for Some case *)

     

       
220
       
220
       
-
       
      | None -> true

     

       
221
       
221
       
-
       
    ) found_urls

     

       
222
       
222
       
-
       
  in

     

       
223
       
223
       
-
       


     

       
224
       
224
       
-
       
  if List.length dois_to_resolve = 0 && List.length urls_to_resolve = 0 then begin

     

       
225
       
225
       
-
       
    Printf.printf "No DOIs or URLs to resolve!\n%!";

     

       
226
       
226
       
-
       
    0

     

       
227
       
227
       
-
       
  end else begin

     

       
228
       
228
       
-
       
    Printf.printf "Resolving %d DOI(s) and %d URL(s)...\n%!" (List.length dois_to_resolve) (List.length urls_to_resolve);

     

       
229
       
229
       
-
       


     

       
230
       
230
       
-
       
    let zt = ZT.v "http://svr-avsm2-eeg-ce:1969" in

     

       
231
       
231
       
-
       


     

       
232
       
232
       
-
       
    (* Resolve all DOIs *)

     

       
233
       
233
       
-
       
    let resolved_doi_entries_lwt =

     

       
234
       
234
       
-
       
      Lwt_list.map_s (resolve_doi zt ~verbose) dois_to_resolve

     

       
235
       
235
       
-
       
    in

     

       
236
       
236
       
-
       


     

       
237
       
237
       
-
       
    (* Resolve all publisher URLs *)

     

       
238
       
238
       
-
       
    let resolved_url_entries_lwt =

     

       
239
       
239
       
-
       
      Lwt_list.map_s (resolve_url zt ~verbose) urls_to_resolve

     

       
240
       
240
       
-
       
    in

     

       
241
       
241
       
-
       


     

       
242
       
242
       
-
       
    let new_doi_entries = Lwt_main.run resolved_doi_entries_lwt in

     

       
243
       
243
       
-
       
    let new_url_entries = Lwt_main.run resolved_url_entries_lwt in

     

       
244
       
244
       
-
       
    let new_entries = new_doi_entries @ new_url_entries in

     

       
245
       
245
       
-
       


     

       
246
       
246
       
-
       
    (* Merge with existing entries, combining source_urls for entries with the same DOI *)

     

       
247
       
247
       
-
       
    let all_entries =

     

       
248
       
248
       
-
       
      if force then

     

       
249
       
249
       
-
       
        (* Replace existing entries with new ones - match by DOI *)

     

       
250
       
250
       
-
       
        let is_updated entry =

     

       
251
       
251
       
-
       
          List.exists (fun new_e ->

     

       
252
       
252
       
-
       
            new_e.Bushel.Doi_entry.doi = entry.Bushel.Doi_entry.doi

     

       
253
       
253
       
-
       
          ) new_entries

     

       
254
       
254
       
-
       
        in

     

       
255
       
255
       
-
       
        let kept_existing = List.filter (fun e -> not (is_updated e)) existing_entries in

     

       
256
       
256
       
-
       
        kept_existing @ new_entries

     

       
257
       
257
       
-
       
      else

     

       
258
       
258
       
-
       
        (* Merge new entries with existing ones, combining source_urls *)

     

       
259
       
259
       
-
       
        let merged = ref existing_entries in

     

       
260
       
260
       
-
       
        List.iter (fun new_entry ->

     

       
261
       
261
       
-
       
          match Bushel.Doi_entry.find_by_doi_including_ignored !merged new_entry.Bushel.Doi_entry.doi with

     

       
262
       
262
       
-
       
          | Some existing_entry ->

     

       
263
       
263
       
-
       
            (* DOI already exists - merge the entries by combining source_urls and preserving ignore flag *)

     

       
264
       
264
       
-
       
            let combined = Bushel.Doi_entry.merge_entries existing_entry new_entry in

     

       
265
       
265
       
-
       
            merged := combined :: (List.filter (fun e -> e.Bushel.Doi_entry.doi <> new_entry.Bushel.Doi_entry.doi) !merged)

     

       
266
       
266
       
-
       
          | None ->

     

       
267
       
267
       
-
       
            (* New DOI - add it *)

     

       
268
       
268
       
-
       
            merged := new_entry :: !merged

     

       
269
       
269
       
-
       
        ) new_entries;

     

       
270
       
270
       
-
       
        !merged

     

       
271
       
271
       
-
       
    in

     

       
272
       
272
       
-
       


     

       
273
       
273
       
-
       
    (* Save updated cache *)

     

       
274
       
274
       
-
       
    Printf.printf "Saving %d total entries to %s...\n%!" (List.length all_entries) doi_yml_path;

     

       
275
       
275
       
-
       
    Bushel.Doi_entry.save doi_yml_path all_entries;

     

       
276
       
276
       
-
       


     

       
277
       
277
       
-
       
    Printf.printf "Done!\n%!";

     

       
278
       
278
       
-
       
    0

     

       
279
       
279
       
-
       
  end

     

       
280
       
280
       
-
       


     

       
281
       
281
       
-
       
let force_flag =

     

       
282
       
282
       
-
       
  let doc = "Force re-resolution of already cached DOIs" in

     

       
283
       
283
       
-
       
  Arg.(value & flag & info ["force"; "f"] ~doc)

     

       
284
       
284
       
-
       


     

       
285
       
285
       
-
       
let verbose_flag =

     

       
286
       
286
       
-
       
  let doc = "Show raw Zotero API responses for debugging" in

     

       
287
       
287
       
-
       
  Arg.(value & flag & info ["verbose"; "v"] ~doc)

     

       
288
       
288
       
-
       


     

       
289
       
289
       
-
       
let term =

     

       
290
       
290
       
-
       
  Term.(const run $ Bushel_common.base_dir $ force_flag $ verbose_flag)

     

       
291
       
291
       
-
       


     

       
292
       
292
       
-
       
let cmd =

     

       
293
       
293
       
-
       
  let doc = "Resolve DOIs found in notes via Zotero Translation Server" in

     

       
294
       
294
       
-
       
  let info = Cmd.info "doi-resolve" ~doc in

     

       
295
       
295
       
-
       
  Cmd.v info term

     

···

       
1
       
1
       
-
       
open Cmdliner

     

       
2
       
2
       
-
       
open Lwt.Infix

     

       
3
       
3
       
-
       
open Printf

     

       
4
       
4
       
-
       


     

       
5
       
5
       
-
       
(* Type for person response *)

     

       
6
       
6
       
-
       
type person = {

     

       
7
       
7
       
-
       
  id: string;

     

       
8
       
8
       
-
       
  name: string;

     

       
9
       
9
       
-
       
  thumbnailPath: string option;

     

       
10
       
10
       
-
       
}

     

       
11
       
11
       
-
       


     

       
12
       
12
       
-
       
(* Parse a person from JSON *)

     

       
13
       
13
       
-
       
let parse_person json =

     

       
14
       
14
       
-
       
  let open Ezjsonm in

     

       
15
       
15
       
-
       
  let id = find json ["id"] |> get_string in

     

       
16
       
16
       
-
       
  let name = find json ["name"] |> get_string in

     

       
17
       
17
       
-
       
  let thumbnailPath = 

     

       
18
       
18
       
-
       
    try Some (find json ["thumbnailPath"] |> get_string)

     

       
19
       
19
       
-
       
    with _ -> None

     

       
20
       
20
       
-
       
  in

     

       
21
       
21
       
-
       
  { id; name; thumbnailPath }

     

       
22
       
22
       
-
       


     

       
23
       
23
       
-
       
(* Parse a list of people from JSON response *)

     

       
24
       
24
       
-
       
let parse_people_response json =

     

       
25
       
25
       
-
       
  let open Ezjsonm in

     

       
26
       
26
       
-
       
  get_list parse_person json

     

       
27
       
27
       
-
       


     

       
28
       
28
       
-
       
(* Read API key from file *)

     

       
29
       
29
       
-
       
let read_api_key file =

     

       
30
       
30
       
-
       
  let ic = open_in file in

     

       
31
       
31
       
-
       
  let key = input_line ic in

     

       
32
       
32
       
-
       
  close_in ic;

     

       
33
       
33
       
-
       
  key

     

       
34
       
34
       
-
       


     

       
35
       
35
       
-
       
(* Search for a person by name *)

     

       
36
       
36
       
-
       
let search_person base_url api_key name =

     

       
37
       
37
       
-
       
  let open Cohttp_lwt_unix in

     

       
38
       
38
       
-
       
  let headers = Cohttp.Header.init_with "X-Api-Key" api_key in

     

       
39
       
39
       
-
       
  let encoded_name = Uri.pct_encode name in

     

       
40
       
40
       
-
       
  let url = Printf.sprintf "%s/api/search/person?name=%s" base_url encoded_name in

     

       
41
       
41
       
-
       
  

     

       
42
       
42
       
-
       
  Client.get ~headers (Uri.of_string url) >>= fun (resp, body) ->

     

       
43
       
43
       
-
       
  if resp.status = `OK then

     

       
44
       
44
       
-
       
    Cohttp_lwt.Body.to_string body >>= fun body_str ->

     

       
45
       
45
       
-
       
    let json = Ezjsonm.from_string body_str in

     

       
46
       
46
       
-
       
    Lwt.return (parse_people_response json)

     

       
47
       
47
       
-
       
  else

     

       
48
       
48
       
-
       
    let status_code = Cohttp.Code.code_of_status resp.status in

     

       
49
       
49
       
-
       
    Lwt.fail_with (Printf.sprintf "HTTP error: %d" status_code)

     

       
50
       
50
       
-
       


     

       
51
       
51
       
-
       
(* Download thumbnail for a person *)

     

       
52
       
52
       
-
       
let download_thumbnail base_url api_key person_id output_path =

     

       
53
       
53
       
-
       
  let open Cohttp_lwt_unix in

     

       
54
       
54
       
-
       
  let headers = Cohttp.Header.init_with "X-Api-Key" api_key in

     

       
55
       
55
       
-
       
  let url = Printf.sprintf "%s/api/people/%s/thumbnail" base_url person_id in

     

       
56
       
56
       
-
       
  

     

       
57
       
57
       
-
       
  Client.get ~headers (Uri.of_string url) >>= fun (resp, body) ->

     

       
58
       
58
       
-
       
  match resp.status with

     

       
59
       
59
       
-
       
  | `OK ->

     

       
60
       
60
       
-
       
    Cohttp_lwt.Body.to_string body >>= fun img_data ->

     

       
61
       
61
       
-
       
    (* Ensure output directory exists *)

     

       
62
       
62
       
-
       
    (try

     

       
63
       
63
       
-
       
       let dir = Filename.dirname output_path in

     

       
64
       
64
       
-
       
       if not (Sys.file_exists dir) then Unix.mkdir dir 0o755;

     

       
65
       
65
       
-
       
       Lwt.return_unit

     

       
66
       
66
       
-
       
     with _ -> Lwt.return_unit) >>= fun () ->

     

       
67
       
67
       
-
       
    Lwt_io.with_file ~mode:Lwt_io.output output_path

     

       
68
       
68
       
-
       
      (fun oc -> Lwt_io.write oc img_data) >>= fun () ->

     

       
69
       
69
       
-
       
    Lwt.return_ok output_path

     

       
70
       
70
       
-
       
  | _ ->

     

       
71
       
71
       
-
       
    let status_code = Cohttp.Code.code_of_status resp.status in

     

       
72
       
72
       
-
       
    Lwt.return_error (Printf.sprintf "HTTP error: %d" status_code)

     

       
73
       
73
       
-
       


     

       
74
       
74
       
-
       
(* Get face for a single contact *)

     

       
75
       
75
       
-
       
(* TODO:claude *)

     

       
76
       
76
       
-
       
let get_face_for_contact base_url api_key output_dir contact =

     

       
77
       
77
       
-
       
  let names = Bushel.Contact.names contact in

     

       
78
       
78
       
-
       
  let handle = Bushel.Contact.handle contact in

     

       
79
       
79
       
-
       
  let output_path = Filename.concat output_dir (handle ^ ".jpg") in

     

       
80
       
80
       
-
       


     

       
81
       
81
       
-
       
  (* Skip if file already exists *)

     

       
82
       
82
       
-
       
  if Sys.file_exists output_path then

     

       
83
       
83
       
-
       
    Lwt.return (`Skipped (sprintf "Thumbnail for '%s' already exists at %s" (List.hd names) output_path))

     

       
84
       
84
       
-
       
  else begin

     

       
85
       
85
       
-
       
    printf "Processing contact: %s (handle: %s)\n%!" (List.hd names) handle;

     

       
86
       
86
       
-
       


     

       
87
       
87
       
-
       
    (* Try each name in the list until we find a match *)

     

       
88
       
88
       
-
       
    let rec try_names = function

     

       
89
       
89
       
-
       
      | [] ->

     

       
90
       
90
       
-
       
          Lwt.return (`Error (sprintf "No person found with any name for contact '%s'" handle))

     

       
91
       
91
       
-
       
      | name :: rest_names ->

     

       
92
       
92
       
-
       
          printf "  Trying name: %s\n%!" name;

     

       
93
       
93
       
-
       
          search_person base_url api_key name >>= function

     

       
94
       
94
       
-
       
          | [] ->

     

       
95
       
95
       
-
       
              printf "  No results for '%s', trying next name...\n%!" name;

     

       
96
       
96
       
-
       
              try_names rest_names

     

       
97
       
97
       
-
       
          | person :: _ ->

     

       
98
       
98
       
-
       
              printf "  Found match for '%s'\n%!" name;

     

       
99
       
99
       
-
       
              download_thumbnail base_url api_key person.id output_path >>= function

     

       
100
       
100
       
-
       
              | Ok path ->

     

       
101
       
101
       
-
       
                  Lwt.return (`Ok (sprintf "Saved thumbnail for '%s' to %s" name path))

     

       
102
       
102
       
-
       
              | Error err ->

     

       
103
       
103
       
-
       
                  Lwt.return (`Error (sprintf "Error for '%s': %s" name err))

     

       
104
       
104
       
-
       
    in

     

       
105
       
105
       
-
       
    try_names names

     

       
106
       
106
       
-
       
  end

     

       
107
       
107
       
-
       


     

       
108
       
108
       
-
       
(* Process all contacts or a specific one *)

     

       
109
       
109
       
-
       
let process_contacts base_dir output_dir specific_handle api_key base_url =

     

       
110
       
110
       
-
       
  printf "Loading Bushel database from %s\n%!" base_dir;

     

       
111
       
111
       
-
       
  let db = Bushel.load base_dir in

     

       
112
       
112
       
-
       
  let contacts = Bushel.Entry.contacts db in

     

       
113
       
113
       
-
       
  printf "Found %d contacts\n%!" (List.length contacts);

     

       
114
       
114
       
-
       
  

     

       
115
       
115
       
-
       
  (* Ensure output directory exists *)

     

       
116
       
116
       
-
       
  if not (Sys.file_exists output_dir) then Unix.mkdir output_dir 0o755;

     

       
117
       
117
       
-
       
  

     

       
118
       
118
       
-
       
  (* Filter contacts based on specific_handle if provided *)

     

       
119
       
119
       
-
       
  let contacts_to_process = 

     

       
120
       
120
       
-
       
    match specific_handle with

     

       
121
       
121
       
-
       
    | Some handle -> 

     

       
122
       
122
       
-
       
        begin match Bushel.Contact.find_by_handle contacts handle with

     

       
123
       
123
       
-
       
        | Some contact -> [contact]

     

       
124
       
124
       
-
       
        | None -> 

     

       
125
       
125
       
-
       
            eprintf "No contact found with handle '%s'\n%!" handle;

     

       
126
       
126
       
-
       
            []

     

       
127
       
127
       
-
       
        end

     

       
128
       
128
       
-
       
    | None -> contacts

     

       
129
       
129
       
-
       
  in

     

       
130
       
130
       
-
       
  

     

       
131
       
131
       
-
       
  (* Process each contact *)

     

       
132
       
132
       
-
       
  let results = Lwt_main.run begin

     

       
133
       
133
       
-
       
    Lwt_list.map_s 

     

       
134
       
134
       
-
       
      (fun contact -> 

     

       
135
       
135
       
-
       
         get_face_for_contact base_url api_key output_dir contact >>= fun result ->

     

       
136
       
136
       
-
       
         Lwt.return (Bushel.Contact.handle contact, result))

     

       
137
       
137
       
-
       
      contacts_to_process

     

       
138
       
138
       
-
       
  end in

     

       
139
       
139
       
-
       
  

     

       
140
       
140
       
-
       
  (* Print summary *)

     

       
141
       
141
       
-
       
  let ok_count = List.length (List.filter (fun (_, r) -> match r with `Ok _ -> true | _ -> false) results) in

     

       
142
       
142
       
-
       
  let error_count = List.length (List.filter (fun (_, r) -> match r with `Error _ -> true | _ -> false) results) in

     

       
143
       
143
       
-
       
  let skipped_count = List.length (List.filter (fun (_, r) -> match r with `Skipped _ -> true | _ -> false) results) in

     

       
144
       
144
       
-
       
  

     

       
145
       
145
       
-
       
  printf "\nSummary:\n";

     

       
146
       
146
       
-
       
  printf "  Successfully processed: %d\n" ok_count;

     

       
147
       
147
       
-
       
  printf "  Errors: %d\n" error_count;

     

       
148
       
148
       
-
       
  printf "  Skipped (already exist): %d\n" skipped_count;

     

       
149
       
149
       
-
       
  

     

       
150
       
150
       
-
       
  (* Print detailed results *)

     

       
151
       
151
       
-
       
  if error_count > 0 then begin

     

       
152
       
152
       
-
       
    printf "\nError details:\n";

     

       
153
       
153
       
-
       
    List.iter (fun (handle, result) ->

     

       
154
       
154
       
-
       
      match result with

     

       
155
       
155
       
-
       
      | `Error msg -> printf "  %s: %s\n" handle msg

     

       
156
       
156
       
-
       
      | _ -> ())

     

       
157
       
157
       
-
       
      results;

     

       
158
       
158
       
-
       
  end;

     

       
159
       
159
       
-
       
  

     

       
160
       
160
       
-
       
  if ok_count > 0 || skipped_count > 0 then 0 else 1

     

       
161
       
161
       
-
       


     

       
162
       
162
       
-
       
(* Command line interface *)

     

       
163
       
163
       
-
       


     

       
164
       
164
       
-
       
(* Export the term for use in main bushel.ml *)

     

       
165
       
165
       
-
       
let term =

     

       
166
       
166
       
-
       
  Term.(

     

       
167
       
167
       
-
       
    const (fun base_dir output_dir handle api_key_file base_url ->

     

       
168
       
168
       
-
       
      try

     

       
169
       
169
       
-
       
        let api_key = read_api_key api_key_file in

     

       
170
       
170
       
-
       
        process_contacts base_dir output_dir handle api_key base_url

     

       
171
       
171
       
-
       
      with e -> 

     

       
172
       
172
       
-
       
        eprintf "Error: %s\n%!" (Printexc.to_string e);

     

       
173
       
173
       
-
       
        1

     

       
174
       
174
       
-
       
    ) $ Bushel_common.base_dir $ Bushel_common.output_dir ~default:"." $ Bushel_common.handle_opt $ 

     

       
175
       
175
       
-
       
      Bushel_common.api_key_file ~default:".photos-api" $ 

     

       
176
       
176
       
-
       
      Bushel_common.url_term ~default:"https://photos.recoil.org" ~doc:"Base URL of the Immich instance")

     

       
177
       
177
       
-
       


     

       
178
       
178
       
-
       
let cmd =

     

       
179
       
179
       
-
       
  let info = Cmd.info "faces" ~doc:"Retrieve face thumbnails for Bushel contacts from Immich" in

     

       
180
       
180
       
-
       
  Cmd.v info term

     

       
181
       
181
       
-
       


     

       
182
       
182
       
-
       
(* Main entry point removed - accessed through bushel_main.ml *)

     

···

       
1
       
1
       
-
       
open Cmdliner

     

       
2
       
2
       
-
       


     

       
3
       
3
       
-
       
(** TODO:claude List completed ideas as markdown bullet list *)

     

       
4
       
4
       
-
       
let list_ideas_md base_dir =

     

       
5
       
5
       
-
       
  let ideas_dir = Printf.sprintf "%s/ideas" base_dir in

     

       
6
       
6
       
-
       
  let contacts_dir = Printf.sprintf "%s/contacts" base_dir in

     

       
7
       
7
       
-
       


     

       
8
       
8
       
-
       
  if not (Sys.file_exists ideas_dir) then (

     

       
9
       
9
       
-
       
    Printf.eprintf "Ideas directory not found: %s\n" ideas_dir;

     

       
10
       
10
       
-
       
    1

     

       
11
       
11
       
-
       
  ) else (

     

       
12
       
12
       
-
       
    (* Load all contacts *)

     

       
13
       
13
       
-
       
    let contacts =

     

       
14
       
14
       
-
       
      if Sys.file_exists contacts_dir then

     

       
15
       
15
       
-
       
        Sys.readdir contacts_dir

     

       
16
       
16
       
-
       
        |> Array.to_list

     

       
17
       
17
       
-
       
        |> List.filter (String.ends_with ~suffix:".md")

     

       
18
       
18
       
-
       
        |> List.filter_map (fun contact_file ->

     

       
19
       
19
       
-
       
          let filepath = Filename.concat contacts_dir contact_file in

     

       
20
       
20
       
-
       
          try Some (Bushel.Contact.of_md filepath)

     

       
21
       
21
       
-
       
          with e ->

     

       
22
       
22
       
-
       
            Printf.eprintf "Error loading contact %s: %s\n" filepath (Printexc.to_string e);

     

       
23
       
23
       
-
       
            None

     

       
24
       
24
       
-
       
        )

     

       
25
       
25
       
-
       
      else []

     

       
26
       
26
       
-
       
    in

     

       
27
       
27
       
-
       


     

       
28
       
28
       
-
       
    let idea_files = Sys.readdir ideas_dir

     

       
29
       
29
       
-
       
                     |> Array.to_list

     

       
30
       
30
       
-
       
                     |> List.filter (String.ends_with ~suffix:".md") in

     

       
31
       
31
       
-
       
    let ideas = List.filter_map (fun idea_file ->

     

       
32
       
32
       
-
       
      let filepath = Filename.concat ideas_dir idea_file in

     

       
33
       
33
       
-
       
      try

     

       
34
       
34
       
-
       
        let idea = Bushel.Idea.of_md filepath in

     

       
35
       
35
       
-
       
        match Bushel.Idea.status idea with

     

       
36
       
36
       
-
       
        | Bushel.Idea.Completed -> Some idea

     

       
37
       
37
       
-
       
        | _ -> None

     

       
38
       
38
       
-
       
      with e ->

     

       
39
       
39
       
-
       
        Printf.eprintf "Error processing %s: %s\n" filepath (Printexc.to_string e);

     

       
40
       
40
       
-
       
        None

     

       
41
       
41
       
-
       
    ) idea_files in

     

       
42
       
42
       
-
       


     

       
43
       
43
       
-
       
    (* Sort by year descending *)

     

       
44
       
44
       
-
       
    let sorted_ideas = List.sort (fun a b ->

     

       
45
       
45
       
-
       
      compare (Bushel.Idea.year b) (Bushel.Idea.year a)

     

       
46
       
46
       
-
       
    ) ideas in

     

       
47
       
47
       
-
       


     

       
48
       
48
       
-
       
    (* Output as markdown bullet list *)

     

       
49
       
49
       
-
       
    List.iter (fun idea ->

     

       
50
       
50
       
-
       
      let student_names =

     

       
51
       
51
       
-
       
        Bushel.Idea.students idea

     

       
52
       
52
       
-
       
        |> List.filter_map (fun handle ->

     

       
53
       
53
       
-
       
          match Bushel.Contact.find_by_handle contacts handle with

     

       
54
       
54
       
-
       
          | Some contact -> Some (Bushel.Contact.name contact)

     

       
55
       
55
       
-
       
          | None ->

     

       
56
       
56
       
-
       
            Printf.eprintf "Warning: contact not found for handle %s\n" handle;

     

       
57
       
57
       
-
       
            Some handle

     

       
58
       
58
       
-
       
        )

     

       
59
       
59
       
-
       
        |> String.concat ", "

     

       
60
       
60
       
-
       
      in

     

       
61
       
61
       
-
       
      let level_str = Bushel.Idea.level_to_string (Bushel.Idea.level idea) in

     

       
62
       
62
       
-
       
      Printf.printf "- %d: \"%s\", %s (%s)\n"

     

       
63
       
63
       
-
       
        (Bushel.Idea.year idea)

     

       
64
       
64
       
-
       
        (Bushel.Idea.title idea)

     

       
65
       
65
       
-
       
        student_names

     

       
66
       
66
       
-
       
        level_str

     

       
67
       
67
       
-
       
    ) sorted_ideas;

     

       
68
       
68
       
-
       
    0

     

       
69
       
69
       
-
       
  )

     

       
70
       
70
       
-
       


     

       
71
       
71
       
-
       
let term =

     

       
72
       
72
       
-
       
  Term.(const list_ideas_md $ Bushel_common.base_dir)

     

       
73
       
73
       
-
       


     

       
74
       
74
       
-
       
let cmd =

     

       
75
       
75
       
-
       
  let doc = "List completed ideas as markdown bullet list" in

     

       
76
       
76
       
-
       
  let info = Cmd.info "ideas-md" ~doc in

     

       
77
       
77
       
-
       
  Cmd.v info term

     

···

       
1
       
1
       
-
       
open Cmdliner

     

       
2
       
2
       
-
       
open Bushel

     

       
3
       
3
       
-
       


     

       
4
       
4
       
-
       
(** Determine the color for a note based on DOI and perma status *)

     

       
5
       
5
       
-
       
let note_color n =

     

       
6
       
6
       
-
       
  match Note.doi n, Note.perma n with

     

       
7
       
7
       
-
       
  | None, false -> `Red          (* No DOI, no perma - red (normal note) *)

     

       
8
       
8
       
-
       
  | None, true -> `Magenta       (* Has perma but no DOI - magenta (needs DOI assignment) *)

     

       
9
       
9
       
-
       
  | Some _, true -> `Green       (* Has DOI with perma:true - green (correct state) *)

     

       
10
       
10
       
-
       
  | Some _, false -> `Yellow     (* Has DOI without perma:true - yellow (bug in metadata) *)

     

       
11
       
11
       
-
       


     

       
12
       
12
       
-
       
(** TODO:claude List all slugs with their types *)

     

       
13
       
13
       
-
       
let list_all_slugs entries ~notes_only =

     

       
14
       
14
       
-
       
  let all = Entry.all_entries entries in

     

       
15
       
15
       
-
       
  (* Filter for notes only if requested *)

     

       
16
       
16
       
-
       
  let filtered = if notes_only then

     

       
17
       
17
       
-
       
    List.filter (fun entry -> match entry with `Note _ -> true | _ -> false) all

     

       
18
       
18
       
-
       
  else all in

     

       
19
       
19
       
-
       
  (* Sort by slug for consistent output *)

     

       
20
       
20
       
-
       
  let sorted = List.sort (fun a b ->

     

       
21
       
21
       
-
       
    String.compare (Entry.slug a) (Entry.slug b)

     

       
22
       
22
       
-
       
  ) filtered in

     

       
23
       
23
       
-
       
  Fmt.pr "@[<v>";

     

       
24
       
24
       
-
       
  Fmt.pr "%a@," (Fmt.styled `Bold Fmt.string) (if notes_only then "Available notes:" else "Available entries:");

     

       
25
       
25
       
-
       
  Fmt.pr "@,";

     

       
26
       
26
       
-
       
  List.iter (fun entry ->

     

       
27
       
27
       
-
       
    let slug = Entry.slug entry in

     

       
28
       
28
       
-
       
    let type_str = Entry.to_type_string entry in

     

       
29
       
29
       
-
       
    let title = Entry.title entry in

     

       
30
       
30
       
-
       
    (* Color code notes based on DOI/perma status *)

     

       
31
       
31
       
-
       
    match entry with

     

       
32
       
32
       
-
       
    | `Note n ->

     

       
33
       
33
       
-
       
      let color = note_color n in

     

       
34
       
34
       
-
       
      Fmt.pr "  %a %a - %a@,"

     

       
35
       
35
       
-
       
        (Fmt.styled color Fmt.string) slug

     

       
36
       
36
       
-
       
        (Fmt.styled `Faint Fmt.string) (Printf.sprintf "(%s)" type_str)

     

       
37
       
37
       
-
       
        Fmt.string title

     

       
38
       
38
       
-
       
    | _ ->

     

       
39
       
39
       
-
       
      Fmt.pr "  %a %a - %a@,"

     

       
40
       
40
       
-
       
        (Fmt.styled `Cyan Fmt.string) slug

     

       
41
       
41
       
-
       
        (Fmt.styled `Faint Fmt.string) (Printf.sprintf "(%s)" type_str)

     

       
42
       
42
       
-
       
        Fmt.string title

     

       
43
       
43
       
-
       
  ) sorted;

     

       
44
       
44
       
-
       
  Fmt.pr "@]@.";

     

       
45
       
45
       
-
       
  0

     

       
46
       
46
       
-
       


     

       
47
       
47
       
-
       
(** TODO:claude Main info command implementation *)

     

       
48
       
48
       
-
       
let info_cmd base_dir notes_only slug_opt _env _xdg _profile =

     

       
49
       
49
       
-
       
  let entries = load base_dir in

     

       
50
       
50
       
-
       
  match slug_opt with

     

       
51
       
51
       
-
       
  | None ->

     

       
52
       
52
       
-
       
    list_all_slugs entries ~notes_only

     

       
53
       
53
       
-
       
  | Some slug ->

     

       
54
       
54
       
-
       
    (* Handle contact handles starting with @ *)

     

       
55
       
55
       
-
       
    if String.starts_with ~prefix:"@" slug then

     

       
56
       
56
       
-
       
      let handle = String.sub slug 1 (String.length slug - 1) in

     

       
57
       
57
       
-
       
      match Contact.find_by_handle (Entry.contacts entries) handle with

     

       
58
       
58
       
-
       
      | None ->

     

       
59
       
59
       
-
       
        Fmt.epr "Error: No contact found with handle '@%s'@." handle;

     

       
60
       
60
       
-
       
        1

     

       
61
       
61
       
-
       
      | Some contact ->

     

       
62
       
62
       
-
       
        Contact.pp Fmt.stdout contact;

     

       
63
       
63
       
-
       
        (* Add thumbnail information for contact *)

     

       
64
       
64
       
-
       
        (match Entry.contact_thumbnail_slug contact with

     

       
65
       
65
       
-
       
         | Some thumb_slug ->

     

       
66
       
66
       
-
       
           Fmt.pr "@.@.";

     

       
67
       
67
       
-
       
           Fmt.pr "@[<v>%a: %s@," (Fmt.styled `Bold Fmt.string) "Thumbnail Slug" thumb_slug;

     

       
68
       
68
       
-
       
           (* Look up the image in srcsetter *)

     

       
69
       
69
       
-
       
           (match Entry.lookup_image entries thumb_slug with

     

       
70
       
70
       
-
       
            | Some img ->

     

       
71
       
71
       
-
       
              let thumbnail_url = Entry.smallest_webp_variant img in

     

       
72
       
72
       
-
       
              Fmt.pr "%a: %s@," (Fmt.styled `Bold Fmt.string) "Thumbnail URL" thumbnail_url;

     

       
73
       
73
       
-
       
              Fmt.pr "%a: %s@," (Fmt.styled `Bold Fmt.string) "Origin" (Srcsetter.origin img);

     

       
74
       
74
       
-
       
              let (w, h) = Srcsetter.dims img in

     

       
75
       
75
       
-
       
              Fmt.pr "%a: %dx%d@," (Fmt.styled `Bold Fmt.string) "Dimensions" w h;

     

       
76
       
76
       
-
       
              let variants = Srcsetter.variants img in

     

       
77
       
77
       
-
       
              if not (Srcsetter.MS.is_empty variants) then begin

     

       
78
       
78
       
-
       
                Fmt.pr "%a:@," (Fmt.styled `Bold Fmt.string) "Variants";

     

       
79
       
79
       
-
       
                Srcsetter.MS.iter (fun name (vw, vh) ->

     

       
80
       
80
       
-
       
                  Fmt.pr "  %s: %dx%d@," name vw vh

     

       
81
       
81
       
-
       
                ) variants

     

       
82
       
82
       
-
       
              end;

     

       
83
       
83
       
-
       
              Fmt.pr "@]"

     

       
84
       
84
       
-
       
            | None ->

     

       
85
       
85
       
-
       
              Fmt.epr "Warning: Contact thumbnail image not in srcsetter: %s@." thumb_slug;

     

       
86
       
86
       
-
       
              Fmt.pr "@]";

     

       
87
       
87
       
-
       
              ())

     

       
88
       
88
       
-
       
         | None -> ());

     

       
89
       
89
       
-
       
        (* Add Typesense JSON *)

     

       
90
       
90
       
-
       
        let doc = Typesense.contact_to_document contact in

     

       
91
       
91
       
-
       
        Fmt.pr "@.@.";

     

       
92
       
92
       
-
       
        Fmt.pr "%a:@," (Fmt.styled `Bold Fmt.string) "Typesense Document";

     

       
93
       
93
       
-
       
        Fmt.pr "%s@," (Ezjsonm.value_to_string ~minify:false doc);

     

       
94
       
94
       
-
       
        (* Add backlinks information for contact *)

     

       
95
       
95
       
-
       
        let backlinks = Bushel.Link_graph.get_backlinks_for_slug handle in

     

       
96
       
96
       
-
       
        if backlinks <> [] then begin

     

       
97
       
97
       
-
       
          Fmt.pr "@.@.";

     

       
98
       
98
       
-
       
          Fmt.pr "%a (%d):@," (Fmt.styled `Bold Fmt.string) "Backlinks" (List.length backlinks);

     

       
99
       
99
       
-
       
          List.iter (fun source_slug ->

     

       
100
       
100
       
-
       
            match Entry.lookup entries source_slug with

     

       
101
       
101
       
-
       
            | Some source_entry ->

     

       
102
       
102
       
-
       
              let source_type = Entry.to_type_string source_entry in

     

       
103
       
103
       
-
       
              let source_title = Entry.title source_entry in

     

       
104
       
104
       
-
       
              Fmt.pr "  %a %a - %a@,"

     

       
105
       
105
       
-
       
                (Fmt.styled `Cyan Fmt.string) source_slug

     

       
106
       
106
       
-
       
                (Fmt.styled `Faint Fmt.string) (Printf.sprintf "(%s)" source_type)

     

       
107
       
107
       
-
       
                Fmt.string source_title

     

       
108
       
108
       
-
       
            | None ->

     

       
109
       
109
       
-
       
              Fmt.pr "  %a %a@,"

     

       
110
       
110
       
-
       
                (Fmt.styled `Cyan Fmt.string) source_slug

     

       
111
       
111
       
-
       
                (Fmt.styled `Red Fmt.string) "(not found)"

     

       
112
       
112
       
-
       
          ) backlinks

     

       
113
       
113
       
-
       
        end;

     

       
114
       
114
       
-
       
        Fmt.pr "@.";

     

       
115
       
115
       
-
       
        0

     

       
116
       
116
       
-
       
    else

     

       
117
       
117
       
-
       
      (* Remove leading ':' if present, as slugs are stored without it *)

     

       
118
       
118
       
-
       
      let normalized_slug =

     

       
119
       
119
       
-
       
        if String.starts_with ~prefix:":" slug

     

       
120
       
120
       
-
       
        then String.sub slug 1 (String.length slug - 1)

     

       
121
       
121
       
-
       
        else slug

     

       
122
       
122
       
-
       
      in

     

       
123
       
123
       
-
       
      match Entry.lookup entries normalized_slug with

     

       
124
       
124
       
-
       
      | None ->

     

       
125
       
125
       
-
       
        Fmt.epr "Error: No entry found with slug '%s'@." slug;

     

       
126
       
126
       
-
       
        1

     

       
127
       
127
       
-
       
      | Some entry ->

     

       
128
       
128
       
-
       
        (match entry with

     

       
129
       
129
       
-
       
         | `Paper p -> Paper.pp Fmt.stdout p

     

       
130
       
130
       
-
       
         | `Project p -> Project.pp Fmt.stdout p

     

       
131
       
131
       
-
       
         | `Idea i -> Idea.pp Fmt.stdout i

     

       
132
       
132
       
-
       
         | `Video v -> Video.pp Fmt.stdout v

     

       
133
       
133
       
-
       
         | `Note n -> Note.pp Fmt.stdout n);

     

       
134
       
134
       
-
       
        (* Add thumbnail information if available *)

     

       
135
       
135
       
-
       
        (match Entry.thumbnail_slug entries entry with

     

       
136
       
136
       
-
       
         | Some thumb_slug ->

     

       
137
       
137
       
-
       
           Fmt.pr "@.@.";

     

       
138
       
138
       
-
       
           Fmt.pr "@[<v>%a: %s@," (Fmt.styled `Bold Fmt.string) "Thumbnail Slug" thumb_slug;

     

       
139
       
139
       
-
       
           (* Look up the image in srcsetter *)

     

       
140
       
140
       
-
       
           (match Entry.lookup_image entries thumb_slug with

     

       
141
       
141
       
-
       
            | Some img ->

     

       
142
       
142
       
-
       
              let thumbnail_url = Entry.smallest_webp_variant img in

     

       
143
       
143
       
-
       
              Fmt.pr "%a: %s@," (Fmt.styled `Bold Fmt.string) "Thumbnail URL" thumbnail_url;

     

       
144
       
144
       
-
       
              Fmt.pr "%a: %s@," (Fmt.styled `Bold Fmt.string) "Origin" (Srcsetter.origin img);

     

       
145
       
145
       
-
       
              let (w, h) = Srcsetter.dims img in

     

       
146
       
146
       
-
       
              Fmt.pr "%a: %dx%d@," (Fmt.styled `Bold Fmt.string) "Dimensions" w h;

     

       
147
       
147
       
-
       
              let variants = Srcsetter.variants img in

     

       
148
       
148
       
-
       
              if not (Srcsetter.MS.is_empty variants) then begin

     

       
149
       
149
       
-
       
                Fmt.pr "%a:@," (Fmt.styled `Bold Fmt.string) "Variants";

     

       
150
       
150
       
-
       
                Srcsetter.MS.iter (fun name (vw, vh) ->

     

       
151
       
151
       
-
       
                  Fmt.pr "  %s: %dx%d@," name vw vh

     

       
152
       
152
       
-
       
                ) variants

     

       
153
       
153
       
-
       
              end;

     

       
154
       
154
       
-
       
              Fmt.pr "@]"

     

       
155
       
155
       
-
       
            | None ->

     

       
156
       
156
       
-
       
              Fmt.epr "Warning: Thumbnail image not in srcsetter: %s@." thumb_slug;

     

       
157
       
157
       
-
       
              Fmt.pr "@]";

     

       
158
       
158
       
-
       
              ())

     

       
159
       
159
       
-
       
         | None -> ());

     

       
160
       
160
       
-
       
        (* Add Typesense JSON *)

     

       
161
       
161
       
-
       
        let doc = match entry with

     

       
162
       
162
       
-
       
          | `Paper p -> Typesense.paper_to_document entries p

     

       
163
       
163
       
-
       
          | `Project p -> Typesense.project_to_document entries p

     

       
164
       
164
       
-
       
          | `Idea i -> Typesense.idea_to_document entries i

     

       
165
       
165
       
-
       
          | `Video v -> Typesense.video_to_document entries v

     

       
166
       
166
       
-
       
          | `Note n -> Typesense.note_to_document entries n

     

       
167
       
167
       
-
       
        in

     

       
168
       
168
       
-
       
        Fmt.pr "@.@.";

     

       
169
       
169
       
-
       
        Fmt.pr "%a:@," (Fmt.styled `Bold Fmt.string) "Typesense Document";

     

       
170
       
170
       
-
       
        Fmt.pr "%s@," (Ezjsonm.value_to_string ~minify:false doc);

     

       
171
       
171
       
-
       
        (* Add backlinks information *)

     

       
172
       
172
       
-
       
        let backlinks = Bushel.Link_graph.get_backlinks_for_slug normalized_slug in

     

       
173
       
173
       
-
       
        if backlinks <> [] then begin

     

       
174
       
174
       
-
       
          Fmt.pr "@.@.";

     

       
175
       
175
       
-
       
          Fmt.pr "%a (%d):@," (Fmt.styled `Bold Fmt.string) "Backlinks" (List.length backlinks);

     

       
176
       
176
       
-
       
          List.iter (fun source_slug ->

     

       
177
       
177
       
-
       
            match Entry.lookup entries source_slug with

     

       
178
       
178
       
-
       
            | Some source_entry ->

     

       
179
       
179
       
-
       
              let source_type = Entry.to_type_string source_entry in

     

       
180
       
180
       
-
       
              let source_title = Entry.title source_entry in

     

       
181
       
181
       
-
       
              Fmt.pr "  %a %a - %a@,"

     

       
182
       
182
       
-
       
                (Fmt.styled `Cyan Fmt.string) source_slug

     

       
183
       
183
       
-
       
                (Fmt.styled `Faint Fmt.string) (Printf.sprintf "(%s)" source_type)

     

       
184
       
184
       
-
       
                Fmt.string source_title

     

       
185
       
185
       
-
       
            | None ->

     

       
186
       
186
       
-
       
              Fmt.pr "  %a %a@,"

     

       
187
       
187
       
-
       
                (Fmt.styled `Cyan Fmt.string) source_slug

     

       
188
       
188
       
-
       
                (Fmt.styled `Red Fmt.string) "(not found)"

     

       
189
       
189
       
-
       
          ) backlinks

     

       
190
       
190
       
-
       
        end;

     

       
191
       
191
       
-
       
        (* Add references information for notes *)

     

       
192
       
192
       
-
       
        (match entry with

     

       
193
       
193
       
-
       
         | `Note n ->

     

       
194
       
194
       
-
       
           let default_author = match Contact.find_by_handle (Entry.contacts entries) "avsm" with

     

       
195
       
195
       
-
       
             | Some c -> c

     

       
196
       
196
       
-
       
             | None -> failwith "Default author 'avsm' not found"

     

       
197
       
197
       
-
       
           in

     

       
198
       
198
       
-
       
           let references = Md.note_references entries default_author n in

     

       
199
       
199
       
-
       
           if references <> [] then begin

     

       
200
       
200
       
-
       
             Fmt.pr "@.@.";

     

       
201
       
201
       
-
       
             Fmt.pr "%a (%d):@," (Fmt.styled `Bold Fmt.string) "References" (List.length references);

     

       
202
       
202
       
-
       
             List.iter (fun (doi, citation, _is_paper) ->

     

       
203
       
203
       
-
       
               Fmt.pr "  %a: %s@,"

     

       
204
       
204
       
-
       
                 (Fmt.styled `Cyan Fmt.string) doi

     

       
205
       
205
       
-
       
                 citation

     

       
206
       
206
       
-
       
             ) references

     

       
207
       
207
       
-
       
           end

     

       
208
       
208
       
-
       
         | _ -> ());

     

       
209
       
209
       
-
       
        Fmt.pr "@.";

     

       
210
       
210
       
-
       
        0

     

       
211
       
211
       
-
       


     

       
212
       
212
       
-
       
(** TODO:claude Command line interface definition *)

     

       
213
       
213
       
-
       
let notes_only_flag =

     

       
214
       
214
       
-
       
  let doc = "Show only notes when listing entries" in

     

       
215
       
215
       
-
       
  Arg.(value & flag & info ["notes-only"; "n"] ~doc)

     

       
216
       
216
       
-
       


     

       
217
       
217
       
-
       
let slug_arg =

     

       
218
       
218
       
-
       
  let doc = "The slug of the entry to display (with or without leading ':'), or contact handle (with '@' prefix). If not provided, lists all available slugs." in

     

       
219
       
219
       
-
       
  Arg.(value & pos 0 (some string) None & info [] ~docv:"SLUG" ~doc)

     

       
220
       
220
       
-
       


     

       
221
       
221
       
-
       
let term : (Eio_unix.Stdenv.base -> Xdge.t -> Keyeio.Profile.t -> int) Cmdliner.Term.t =

     

       
222
       
222
       
-
       
  Term.(const info_cmd $ Bushel_common.base_dir $ notes_only_flag $ slug_arg)

     

       
223
       
223
       
-
       


     

       
224
       
224
       
-
       
let cmd =

     

       
225
       
225
       
-
       
  let doc = "Display all information for a given slug" in

     

       
226
       
226
       
-
       
  let info = Cmd.info "info" ~doc in

     

       
227
       
227
       
-
       
  Cmd.v info term

     

···

       
1
       
1
       
-
       
open Cmdliner

     

       
2
       
2
       
-
       
open Lwt.Infix

     

       
3
       
3
       
-
       


     

       
4
       
4
       
-
       
(* Helper function for logging with proper flushing *)

     

       
5
       
5
       
-
       
let log fmt = Fmt.kstr (fun s -> prerr_string s; flush stderr) fmt

     

       
6
       
6
       
-
       
let log_verbose verbose fmt = 

     

       
7
       
7
       
-
       
  if verbose then Fmt.kstr (fun s -> prerr_string s; flush stderr) fmt 

     

       
8
       
8
       
-
       
  else Fmt.kstr (fun _ -> ()) fmt

     

       
9
       
9
       
-
       


     

       
10
       
10
       
-
       
(* Initialize a new links.yml file or ensure it exists *)

     

       
11
       
11
       
-
       
let init_links_file links_file =

     

       
12
       
12
       
-
       
  if Sys.file_exists links_file then

     

       
13
       
13
       
-
       
    print_endline (Fmt.str "Links file %s already exists" links_file)

     

       
14
       
14
       
-
       
  else begin

     

       
15
       
15
       
-
       
    (* Create an empty links file *)

     

       
16
       
16
       
-
       
    Bushel.Link.save_links_file links_file [];

     

       
17
       
17
       
-
       
    print_endline (Fmt.str "Created empty links file: %s" links_file)

     

       
18
       
18
       
-
       
  end;

     

       
19
       
19
       
-
       
  0

     

       
20
       
20
       
-
       


     

       
21
       
21
       
-
       
(* Update links.yml from Karakeep *)

     

       
22
       
22
       
-
       
let update_from_karakeep base_url api_key_opt tag links_file download_assets =

     

       
23
       
23
       
-
       
  match api_key_opt with

     

       
24
       
24
       
-
       
  | None ->

     

       
25
       
25
       
-
       
      prerr_endline "Error: API key is required.";

     

       
26
       
26
       
-
       
      prerr_endline "Please provide one with --api-key or create a ~/.karakeep-api file.";

     

       
27
       
27
       
-
       
      1

     

       
28
       
28
       
-
       
  | Some api_key ->

     

       
29
       
29
       
-
       
      let assets_dir = "data/assets" in

     

       
30
       
30
       
-
       
      

     

       
31
       
31
       
-
       
      (* Run the Lwt program *)

     

       
32
       
32
       
-
       
      Lwt_main.run (

     

       
33
       
33
       
-
       
        print_endline (Fmt.str "Fetching links from %s with tag '%s'..." base_url tag);

     

       
34
       
34
       
-
       
        

     

       
35
       
35
       
-
       
        (* Prepare tag filter *)

     

       
36
       
36
       
-
       
        let filter_tags = if tag = "" then [] else [tag] in

     

       
37
       
37
       
-
       
        

     

       
38
       
38
       
-
       
        (* Fetch bookmarks from Karakeep with error handling *)

     

       
39
       
39
       
-
       
        Lwt.catch

     

       
40
       
40
       
-
       
          (fun () ->

     

       
41
       
41
       
-
       
            Karakeep.fetch_all_bookmarks ~api_key ~filter_tags base_url >>= fun bookmarks ->

     

       
42
       
42
       
-
       
            

     

       
43
       
43
       
-
       
            print_endline (Fmt.str "Retrieved %d bookmarks from Karakeep" (List.length bookmarks));

     

       
44
       
44
       
-
       
            

     

       
45
       
45
       
-
       
            (* Read existing links if file exists *)

     

       
46
       
46
       
-
       
            let existing_links = Bushel.Link.load_links_file links_file in

     

       
47
       
47
       
-
       
            

     

       
48
       
48
       
-
       
            (* Convert bookmarks to bushel links *)

     

       
49
       
49
       
-
       
            let new_links = List.map (fun bookmark ->

     

       
50
       
50
       
-
       
              Karakeep.to_bushel_link ~base_url bookmark

     

       
51
       
51
       
-
       
            ) bookmarks in

     

       
52
       
52
       
-
       
            

     

       
53
       
53
       
-
       
            (* Merge with existing links - keep existing dates (karakeep dates may be unreliable) *)

     

       
54
       
54
       
-
       
            let merged_links = Bushel.Link.merge_links existing_links new_links in

     

       
55
       
55
       
-
       
            

     

       
56
       
56
       
-
       
            (* Save the updated links file *)

     

       
57
       
57
       
-
       
            Bushel.Link.save_links_file links_file merged_links;

     

       
58
       
58
       
-
       
            

     

       
59
       
59
       
-
       
            print_endline (Fmt.str "Updated %s with %d links" links_file (List.length merged_links));

     

       
60
       
60
       
-
       
            

     

       
61
       
61
       
-
       
            (* Download assets if requested *)

     

       
62
       
62
       
-
       
            if download_assets then begin

     

       
63
       
63
       
-
       
              print_endline "Downloading assets for bookmarks...";

     

       
64
       
64
       
-
       
              

     

       
65
       
65
       
-
       
              (* Ensure the assets directory exists *)

     

       
66
       
66
       
-
       
              (try Unix.mkdir assets_dir 0o755 with Unix.Unix_error (Unix.EEXIST, _, _) -> ());

     

       
67
       
67
       
-
       
              

     

       
68
       
68
       
-
       
              (* Process each bookmark with assets *)

     

       
69
       
69
       
-
       
              Lwt_list.iter_s (fun bookmark ->

     

       
70
       
70
       
-
       
                (* Extract asset IDs from bookmark *)

     

       
71
       
71
       
-
       
                let assets = bookmark.Karakeep.assets in

     

       
72
       
72
       
-
       
                

     

       
73
       
73
       
-
       
                (* Skip if no assets *)

     

       
74
       
74
       
-
       
                if assets = [] then

     

       
75
       
75
       
-
       
                  Lwt.return_unit

     

       
76
       
76
       
-
       
                else

     

       
77
       
77
       
-
       
                  (* Process each asset *)

     

       
78
       
78
       
-
       
                  Lwt_list.iter_s (fun (asset_id, asset_type) ->

     

       
79
       
79
       
-
       
                    let asset_dir = Fmt.str "%s/%s" assets_dir asset_id in

     

       
80
       
80
       
-
       
                    let asset_file = Fmt.str "%s/asset.bin" asset_dir in

     

       
81
       
81
       
-
       
                    let meta_file = Fmt.str "%s/metadata.json" asset_dir in

     

       
82
       
82
       
-
       
                    

     

       
83
       
83
       
-
       
                    (* Skip if the asset already exists *)

     

       
84
       
84
       
-
       
                    if Sys.file_exists asset_file then

     

       
85
       
85
       
-
       
                      Lwt.return_unit

     

       
86
       
86
       
-
       
                    else begin

     

       
87
       
87
       
-
       
                      (* Create the asset directory *)

     

       
88
       
88
       
-
       
                      (try Unix.mkdir asset_dir 0o755 with Unix.Unix_error (Unix.EEXIST, _, _) -> ());

     

       
89
       
89
       
-
       
                      

     

       
90
       
90
       
-
       
                      (* Download the asset *)

     

       
91
       
91
       
-
       
                      print_endline (Fmt.str "Downloading %s asset %s..." asset_type asset_id);

     

       
92
       
92
       
-
       
                      Karakeep.fetch_asset ~api_key base_url asset_id >>= fun data ->

     

       
93
       
93
       
-
       
                      

     

       
94
       
94
       
-
       
                      (* Guess content type based on first bytes *)

     

       
95
       
95
       
-
       
                      let content_type = 

     

       
96
       
96
       
-
       
                        if String.length data >= 4 && String.sub data 0 4 = "\x89PNG" then

     

       
97
       
97
       
-
       
                          "image/png"

     

       
98
       
98
       
-
       
                        else if String.length data >= 3 && String.sub data 0 3 = "\xFF\xD8\xFF" then

     

       
99
       
99
       
-
       
                          "image/jpeg"

     

       
100
       
100
       
-
       
                        else if String.length data >= 4 && String.sub data 0 4 = "%PDF" then

     

       
101
       
101
       
-
       
                          "application/pdf"

     

       
102
       
102
       
-
       
                        else

     

       
103
       
103
       
-
       
                          "application/octet-stream"

     

       
104
       
104
       
-
       
                      in

     

       
105
       
105
       
-
       
                      

     

       
106
       
106
       
-
       
                      (* Write the asset data *)

     

       
107
       
107
       
-
       
                      Lwt_io.with_file ~mode:Lwt_io.Output asset_file (fun oc ->

     

       
108
       
108
       
-
       
                        Lwt_io.write oc data

     

       
109
       
109
       
-
       
                      ) >>= fun () ->

     

       
110
       
110
       
-
       
                      

     

       
111
       
111
       
-
       
                      (* Write metadata file *)

     

       
112
       
112
       
-
       
                      let metadata = Fmt.str "{\n  \"contentType\": \"%s\",\n  \"assetType\": \"%s\"\n}" 

     

       
113
       
113
       
-
       
                        content_type asset_type in

     

       
114
       
114
       
-
       
                      Lwt_io.with_file ~mode:Lwt_io.Output meta_file (fun oc ->

     

       
115
       
115
       
-
       
                        Lwt_io.write oc metadata

     

       
116
       
116
       
-
       
                      )

     

       
117
       
117
       
-
       
                    end

     

       
118
       
118
       
-
       
                  ) assets

     

       
119
       
119
       
-
       
              ) bookmarks >>= fun () ->

     

       
120
       
120
       
-
       
              

     

       
121
       
121
       
-
       
              print_endline "Asset download completed.";

     

       
122
       
122
       
-
       
              Lwt.return 0

     

       
123
       
123
       
-
       
            end else

     

       
124
       
124
       
-
       
              Lwt.return 0

     

       
125
       
125
       
-
       
          )

     

       
126
       
126
       
-
       
          (fun exn ->

     

       
127
       
127
       
-
       
            prerr_endline (Fmt.str "Error fetching bookmarks: %s" (Printexc.to_string exn));

     

       
128
       
128
       
-
       
            Lwt.return 1

     

       
129
       
129
       
-
       
          )

     

       
130
       
130
       
-
       
      )

     

       
131
       
131
       
-
       


     

       
132
       
132
       
-
       
(* Extract outgoing links from Bushel entries *)

     

       
133
       
133
       
-
       
let update_from_bushel bushel_dir links_file include_domains exclude_domains =

     

       
134
       
134
       
-
       
  (* Parse domain filters if provided *)

     

       
135
       
135
       
-
       
  let include_domains_list = match include_domains with

     

       
136
       
136
       
-
       
    | None -> []

     

       
137
       
137
       
-
       
    | Some s -> String.split_on_char ',' s |> List.map String.trim

     

       
138
       
138
       
-
       
  in

     

       
139
       
139
       
-
       
  

     

       
140
       
140
       
-
       
  let exclude_domains_list = match exclude_domains with

     

       
141
       
141
       
-
       
    | None -> []

     

       
142
       
142
       
-
       
    | Some s -> String.split_on_char ',' s |> List.map String.trim

     

       
143
       
143
       
-
       
  in

     

       
144
       
144
       
-
       
  

     

       
145
       
145
       
-
       
  (* Show filter settings if any *)

     

       
146
       
146
       
-
       
  if include_domains_list <> [] then

     

       
147
       
147
       
-
       
    print_endline (Fmt.str "Including only domains: %s" (String.concat ", " include_domains_list));

     

       
148
       
148
       
-
       
  

     

       
149
       
149
       
-
       
  if exclude_domains_list <> [] then

     

       
150
       
150
       
-
       
    print_endline (Fmt.str "Excluding domains: %s" (String.concat ", " exclude_domains_list));

     

       
151
       
151
       
-
       
  

     

       
152
       
152
       
-
       
  (* Load all entries from the bushel directory *)

     

       
153
       
153
       
-
       
  let notes_dir = Filename.concat bushel_dir "data/notes" in

     

       
154
       
154
       
-
       
  

     

       
155
       
155
       
-
       
  (* Make sure the notes directory exists *)

     

       
156
       
156
       
-
       
  if not (Sys.file_exists notes_dir) then begin

     

       
157
       
157
       
-
       
    prerr_endline (Fmt.str "Error: Notes directory %s does not exist" notes_dir);

     

       
158
       
158
       
-
       
    exit 1

     

       
159
       
159
       
-
       
  end;

     

       
160
       
160
       
-
       
  

     

       
161
       
161
       
-
       
  (* Load all entries with fallback *)

     

       
162
       
162
       
-
       
  print_endline (Fmt.str "Loading entries from %s..." bushel_dir);

     

       
163
       
163
       
-
       
  

     

       
164
       
164
       
-
       
  let entries_data = Bushel.load bushel_dir in

     

       
165
       
165
       
-
       
  let all_entries = Bushel.Entry.all_entries entries_data in

     

       
166
       
166
       
-
       
  print_endline (Fmt.str "Loaded %d entries" (List.length all_entries));

     

       
167
       
167
       
-
       
  

     

       
168
       
168
       
-
       
  (* Extract outgoing links from all entries *)

     

       
169
       
169
       
-
       
  print_endline "Extracting outgoing links...";

     

       
170
       
170
       
-
       
  let extracted_links = ref [] in

     

       
171
       
171
       
-
       
  

     

       
172
       
172
       
-
       
  (* Process each entry *)

     

       
173
       
173
       
-
       
  List.iter (fun entry ->

     

       
174
       
174
       
-
       
    let entry_body = Bushel.Entry.body entry in

     

       
175
       
175
       
-
       
    let entry_slug = Bushel.Entry.slug entry in

     

       
176
       
176
       
-
       
    

     

       
177
       
177
       
-
       
    (* Skip empty bodies *)

     

       
178
       
178
       
-
       
    if entry_body <> "" then begin

     

       
179
       
179
       
-
       
      let links = Bushel.Entry.extract_external_links entry_body in

     

       
180
       
180
       
-
       
      if links <> [] then begin

     

       
181
       
181
       
-
       
        (* Add each link from this entry *)

     

       
182
       
182
       
-
       
        List.iter (fun url ->

     

       
183
       
183
       
-
       
          (* Try to extract domain from URL *)

     

       
184
       
184
       
-
       
          let domain = 

     

       
185
       
185
       
-
       
            try

     

       
186
       
186
       
-
       
              let uri = Uri.of_string url in

     

       
187
       
187
       
-
       
              match Uri.host uri with

     

       
188
       
188
       
-
       
              | Some host -> host

     

       
189
       
189
       
-
       
              | None -> "unknown"

     

       
190
       
190
       
-
       
            with _ -> "unknown"

     

       
191
       
191
       
-
       
          in

     

       
192
       
192
       
-
       
          

     

       
193
       
193
       
-
       
          (* Filter by domain if filters are specified *)

     

       
194
       
194
       
-
       
          let include_by_domain =

     

       
195
       
195
       
-
       
            if include_domains_list <> [] then

     

       
196
       
196
       
-
       
              List.exists (fun filter -> 

     

       
197
       
197
       
-
       
                domain = filter || String.ends_with ~suffix:filter domain

     

       
198
       
198
       
-
       
              ) include_domains_list

     

       
199
       
199
       
-
       
            else true

     

       
200
       
200
       
-
       
          in

     

       
201
       
201
       
-
       
          

     

       
202
       
202
       
-
       
          let exclude_by_domain =

     

       
203
       
203
       
-
       
            List.exists (fun filter -> 

     

       
204
       
204
       
-
       
              domain = filter || String.ends_with ~suffix:filter domain

     

       
205
       
205
       
-
       
            ) exclude_domains_list

     

       
206
       
206
       
-
       
          in

     

       
207
       
207
       
-
       
          

     

       
208
       
208
       
-
       
          if include_by_domain && not exclude_by_domain then begin

     

       
209
       
209
       
-
       
            let date = Bushel.Entry.date entry in

     

       
210
       
210
       
-
       
            

     

       
211
       
211
       
-
       
            (* Extract tags from the entry *)

     

       
212
       
212
       
-
       
            let entry_tags = Bushel.Tags.tags_of_ent entries_data entry in

     

       
213
       
213
       
-
       
            let tag_strings = List.map Bushel.Tags.to_string entry_tags in

     

       
214
       
214
       
-
       
            

     

       
215
       
215
       
-
       
            let link = { 

     

       
216
       
216
       
-
       
              Bushel.Link.url; 

     

       
217
       
217
       
-
       
              date; 

     

       
218
       
218
       
-
       
              description = ""; 

     

       
219
       
219
       
-
       
              karakeep = None;

     

       
220
       
220
       
-
       
              bushel = Some { 

     

       
221
       
221
       
-
       
                Bushel.Link.slugs = [entry_slug]; 

     

       
222
       
222
       
-
       
                tags = tag_strings 

     

       
223
       
223
       
-
       
              };

     

       
224
       
224
       
-
       
            } in

     

       
225
       
225
       
-
       
            extracted_links := link :: !extracted_links

     

       
226
       
226
       
-
       
          end

     

       
227
       
227
       
-
       
        ) links

     

       
228
       
228
       
-
       
      end

     

       
229
       
229
       
-
       
    end

     

       
230
       
230
       
-
       
  ) all_entries;

     

       
231
       
231
       
-
       
  

     

       
232
       
232
       
-
       
  (* Load existing links *)

     

       
233
       
233
       
-
       
  let existing_links = Bushel.Link.load_links_file links_file in

     

       
234
       
234
       
-
       
  

     

       
235
       
235
       
-
       
  (* Merge with existing links - prefer bushel entry dates *)

     

       
236
       
236
       
-
       
  let merged_links = Bushel.Link.merge_links ~prefer_new_date:true existing_links !extracted_links in

     

       
237
       
237
       
-
       
  

     

       
238
       
238
       
-
       
  (* Save the updated links file *)

     

       
239
       
239
       
-
       
  Bushel.Link.save_links_file links_file merged_links;

     

       
240
       
240
       
-
       
  

     

       
241
       
241
       
-
       
  print_endline (Fmt.str "Added %d extracted links from Bushel to %s" 

     

       
242
       
242
       
-
       
    (List.length !extracted_links) links_file);

     

       
243
       
243
       
-
       
  print_endline (Fmt.str "Total links in file: %d" (List.length merged_links));

     

       
244
       
244
       
-
       
  0

     

       
245
       
245
       
-
       


     

       
246
       
246
       
-
       
(* Helper function to filter links that don't have karakeep data for a specific remote *)

     

       
247
       
247
       
-
       
let filter_links_without_karakeep base_url links =

     

       
248
       
248
       
-
       
  List.filter (fun link ->

     

       
249
       
249
       
-
       
    match link.Bushel.Link.karakeep with

     

       
250
       
250
       
-
       
    | Some { remote_url; _ } when remote_url = base_url -> false

     

       
251
       
251
       
-
       
    | _ -> true

     

       
252
       
252
       
-
       
  ) links

     

       
253
       
253
       
-
       


     

       
254
       
254
       
-
       
(* Helper function to apply limit to links if specified *)

     

       
255
       
255
       
-
       
let apply_limit_to_links limit links =

     

       
256
       
256
       
-
       
  match limit with

     

       
257
       
257
       
-
       
  | Some n when n > 0 -> 

     

       
258
       
258
       
-
       
      let rec take_n acc count = function

     

       
259
       
259
       
-
       
        | [] -> List.rev acc

     

       
260
       
260
       
-
       
        | _ when count = 0 -> List.rev acc

     

       
261
       
261
       
-
       
        | x :: xs -> take_n (x :: acc) (count - 1) xs

     

       
262
       
262
       
-
       
      in

     

       
263
       
263
       
-
       
      let limited = take_n [] n links in

     

       
264
       
264
       
-
       
      if List.length links > n then

     

       
265
       
265
       
-
       
        log "Limited to first %d links (out of %d available)\n" n (List.length links);

     

       
266
       
266
       
-
       
      limited

     

       
267
       
267
       
-
       
  | _ -> links

     

       
268
       
268
       
-
       


     

       
269
       
269
       
-
       
(* Helper function to prepare tags for a link *)

     

       
270
       
270
       
-
       
let prepare_tags_for_link tag link =

     

       
271
       
271
       
-
       
  let slug_tags = 

     

       
272
       
272
       
-
       
    match link.Bushel.Link.bushel with

     

       
273
       
273
       
-
       
    | Some { slugs; _ } -> List.map (fun slug -> "bushel:" ^ slug) slugs

     

       
274
       
274
       
-
       
    | None -> []

     

       
275
       
275
       
-
       
  in

     

       
276
       
276
       
-
       
  if tag = "" then slug_tags

     

       
277
       
277
       
-
       
  else tag :: slug_tags

     

       
278
       
278
       
-
       


     

       
279
       
279
       
-
       
(* Helper function to create batches for parallel processing *)

     

       
280
       
280
       
-
       
let create_batches max_concurrent links =

     

       
281
       
281
       
-
       
  let rec create_batches_aux links acc =

     

       
282
       
282
       
-
       
    match links with

     

       
283
       
283
       
-
       
    | [] -> List.rev acc

     

       
284
       
284
       
-
       
    | _ ->

     

       
285
       
285
       
-
       
        let batch, rest = 

     

       
286
       
286
       
-
       
          if List.length links <= max_concurrent then

     

       
287
       
287
       
-
       
            links, []

     

       
288
       
288
       
-
       
          else

     

       
289
       
289
       
-
       
            let rec take n lst batch =

     

       
290
       
290
       
-
       
              if n = 0 || lst = [] then List.rev batch, lst

     

       
291
       
291
       
-
       
              else take (n-1) (List.tl lst) (List.hd lst :: batch)

     

       
292
       
292
       
-
       
            in

     

       
293
       
293
       
-
       
            take max_concurrent links []

     

       
294
       
294
       
-
       
        in

     

       
295
       
295
       
-
       
        create_batches_aux rest (batch :: acc)

     

       
296
       
296
       
-
       
  in

     

       
297
       
297
       
-
       
  create_batches_aux links []

     

       
298
       
298
       
-
       


     

       
299
       
299
       
-
       
(* Helper function to upload a single link to Karakeep *)

     

       
300
       
300
       
-
       
let upload_single_link api_key base_url tag verbose updated_links link =

     

       
301
       
301
       
-
       
  let url = link.Bushel.Link.url in

     

       
302
       
302
       
-
       
  let title = 

     

       
303
       
303
       
-
       
    if link.Bushel.Link.description <> "" then 

     

       
304
       
304
       
-
       
      Some link.Bushel.Link.description 

     

       
305
       
305
       
-
       
    else None 

     

       
306
       
306
       
-
       
  in

     

       
307
       
307
       
-
       
  let tags = prepare_tags_for_link tag link in

     

       
308
       
308
       
-
       
  

     

       
309
       
309
       
-
       
  if verbose then begin

     

       
310
       
310
       
-
       
    log "  Uploading: %s\n" url;

     

       
311
       
311
       
-
       
    if tags <> [] then 

     

       
312
       
312
       
-
       
      log "    Tags: %s\n" (String.concat ", " tags);

     

       
313
       
313
       
-
       
    if title <> None then 

     

       
314
       
314
       
-
       
      log "    Title: %s\n" (Option.get title);

     

       
315
       
315
       
-
       
  end else begin

     

       
316
       
316
       
-
       
    log "Uploading: %s\n" url;

     

       
317
       
317
       
-
       
  end;

     

       
318
       
318
       
-
       
  

     

       
319
       
319
       
-
       
  (* Create the bookmark with tags *)

     

       
320
       
320
       
-
       
  Lwt.catch

     

       
321
       
321
       
-
       
    (fun () ->

     

       
322
       
322
       
-
       
      Karakeep.create_bookmark 

     

       
323
       
323
       
-
       
        ~api_key 

     

       
324
       
324
       
-
       
        ~url 

     

       
325
       
325
       
-
       
        ?title 

     

       
326
       
326
       
-
       
        ~tags 

     

       
327
       
327
       
-
       
        base_url 

     

       
328
       
328
       
-
       
      >>= fun bookmark ->

     

       
329
       
329
       
-
       
      

     

       
330
       
330
       
-
       
      (* Create updated link with karakeep data *)

     

       
331
       
331
       
-
       
      let updated_link = {

     

       
332
       
332
       
-
       
        link with 

     

       
333
       
333
       
-
       
        Bushel.Link.karakeep = 

     

       
334
       
334
       
-
       
          Some { 

     

       
335
       
335
       
-
       
            Bushel.Link.remote_url = base_url; 

     

       
336
       
336
       
-
       
            id = bookmark.id;

     

       
337
       
337
       
-
       
            tags = bookmark.tags;

     

       
338
       
338
       
-
       
            metadata = [];  (* Will be populated on next sync *)

     

       
339
       
339
       
-
       
          }

     

       
340
       
340
       
-
       
      } in

     

       
341
       
341
       
-
       
      updated_links := updated_link :: !updated_links;

     

       
342
       
342
       
-
       
      

     

       
343
       
343
       
-
       
      if verbose then

     

       
344
       
344
       
-
       
        log "    ✓ Added to Karakeep with ID: %s\n" bookmark.id

     

       
345
       
345
       
-
       
      else

     

       
346
       
346
       
-
       
        log "  - Added to Karakeep with ID: %s\n" bookmark.id;

     

       
347
       
347
       
-
       
      Lwt.return 1 (* Success *)

     

       
348
       
348
       
-
       
    )

     

       
349
       
349
       
-
       
    (fun exn ->

     

       
350
       
350
       
-
       
      if verbose then

     

       
351
       
351
       
-
       
        log "    ✗ Error uploading %s: %s\n" url (Printexc.to_string exn)

     

       
352
       
352
       
-
       
      else

     

       
353
       
353
       
-
       
        log "  - Error uploading %s: %s\n" url (Printexc.to_string exn);

     

       
354
       
354
       
-
       
      Lwt.return 0 (* Failure *)

     

       
355
       
355
       
-
       
    )

     

       
356
       
356
       
-
       


     

       
357
       
357
       
-
       
(* Helper function to process a batch of links *)

     

       
358
       
358
       
-
       
let process_batch api_key base_url tag verbose updated_links batch_num total_batches batch =

     

       
359
       
359
       
-
       
  log_verbose verbose "\nProcessing batch %d/%d (%d links)...\n" 

     

       
360
       
360
       
-
       
    (batch_num + 1) total_batches (List.length batch);

     

       
361
       
361
       
-
       
  

     

       
362
       
362
       
-
       
  (* Process links in this batch concurrently *)

     

       
363
       
363
       
-
       
  Lwt_list.map_p (upload_single_link api_key base_url tag verbose updated_links) batch

     

       
364
       
364
       
-
       


     

       
365
       
365
       
-
       
(* Helper function to update links file with new karakeep data *)

     

       
366
       
366
       
-
       
let update_links_file links_file original_links updated_links =

     

       
367
       
367
       
-
       
  if !updated_links <> [] then begin

     

       
368
       
368
       
-
       
    (* Replace the updated links in the original list *)

     

       
369
       
369
       
-
       
    let final_links =

     

       
370
       
370
       
-
       
      List.map (fun link ->

     

       
371
       
371
       
-
       
        let url = link.Bushel.Link.url in

     

       
372
       
372
       
-
       
        let updated = List.find_opt (fun ul -> ul.Bushel.Link.url = url) !updated_links in

     

       
373
       
373
       
-
       
        match updated with

     

       
374
       
374
       
-
       
        | Some ul -> ul

     

       
375
       
375
       
-
       
        | None -> link

     

       
376
       
376
       
-
       
      ) original_links

     

       
377
       
377
       
-
       
    in

     

       
378
       
378
       
-
       
    

     

       
379
       
379
       
-
       
    (* Save the updated links file *)

     

       
380
       
380
       
-
       
    Bushel.Link.save_links_file links_file final_links;

     

       
381
       
381
       
-
       
    

     

       
382
       
382
       
-
       
    log "Updated %s with %d new karakeep_ids\n" 

     

       
383
       
383
       
-
       
      links_file (List.length !updated_links);

     

       
384
       
384
       
-
       
  end

     

       
385
       
385
       
-
       


     

       
386
       
386
       
-
       
(* Upload links to Karakeep that don't already have karakeep data *)

     

       
387
       
387
       
-
       
let upload_to_karakeep base_url api_key_opt links_file tag max_concurrent delay_seconds limit verbose =

     

       
388
       
388
       
-
       
  match api_key_opt with

     

       
389
       
389
       
-
       
  | None ->

     

       
390
       
390
       
-
       
      log "Error: API key is required.\n";

     

       
391
       
391
       
-
       
      log "Please provide one with --api-key or create a ~/.karakeep-api file.\n";

     

       
392
       
392
       
-
       
      1

     

       
393
       
393
       
-
       
  | Some api_key ->

     

       
394
       
394
       
-
       
      (* Load links from file *)

     

       
395
       
395
       
-
       
      log_verbose verbose "Loading links from %s...\n" links_file;

     

       
396
       
396
       
-
       
      let links = Bushel.Link.load_links_file links_file in

     

       
397
       
397
       
-
       
      log_verbose verbose "Loaded %d total links\n" (List.length links);

     

       
398
       
398
       
-
       
      

     

       
399
       
399
       
-
       
      (* Filter links that don't have karakeep data for this remote *)

     

       
400
       
400
       
-
       
      log_verbose verbose "Filtering links that don't have karakeep data for %s...\n" base_url;

     

       
401
       
401
       
-
       
      let filtered_links = filter_links_without_karakeep base_url links in

     

       
402
       
402
       
-
       
      log_verbose verbose "Found %d links without karakeep data\n" (List.length filtered_links);

     

       
403
       
403
       
-
       
      

     

       
404
       
404
       
-
       
      (* Apply limit if specified *)

     

       
405
       
405
       
-
       
      let links_to_upload = apply_limit_to_links limit filtered_links in

     

       
406
       
406
       
-
       
  

     

       
407
       
407
       
-
       
      if links_to_upload = [] then begin

     

       
408
       
408
       
-
       
        log "No links to upload to %s (all links already have karakeep data)\n" base_url;

     

       
409
       
409
       
-
       
        0

     

       
410
       
410
       
-
       
      end else begin

     

       
411
       
411
       
-
       
        log "Found %d links to upload to %s\n" (List.length links_to_upload) base_url;

     

       
412
       
412
       
-
       
        

     

       
413
       
413
       
-
       
        (* Split links into batches for parallel processing *)

     

       
414
       
414
       
-
       
        let batches = create_batches max_concurrent links_to_upload in

     

       
415
       
415
       
-
       
        log_verbose verbose "Processing in %d batches of up to %d links each...\n" 

     

       
416
       
416
       
-
       
          (List.length batches) max_concurrent;

     

       
417
       
417
       
-
       
        log_verbose verbose "Delay between batches: %.1f seconds\n" delay_seconds;

     

       
418
       
418
       
-
       
        

     

       
419
       
419
       
-
       
        (* Process batches and accumulate updated links *)

     

       
420
       
420
       
-
       
        let updated_links = ref [] in

     

       
421
       
421
       
-
       
        

     

       
422
       
422
       
-
       
        let result = Lwt_main.run (

     

       
423
       
423
       
-
       
          Lwt.catch

     

       
424
       
424
       
-
       
            (fun () ->

     

       
425
       
425
       
-
       
              Lwt_list.fold_left_s (fun (total_count, batch_num) batch ->

     

       
426
       
426
       
-
       
                process_batch api_key base_url tag verbose updated_links 

     

       
427
       
427
       
-
       
                  batch_num (List.length batches) batch >>= fun results ->

     

       
428
       
428
       
-
       
                

     

       
429
       
429
       
-
       
                (* Count successes in this batch *)

     

       
430
       
430
       
-
       
                let batch_successes = List.fold_left (+) 0 results in

     

       
431
       
431
       
-
       
                let new_total = total_count + batch_successes in

     

       
432
       
432
       
-
       
                

     

       
433
       
433
       
-
       
                log_verbose verbose "  Batch %d complete: %d/%d successful (Total: %d/%d)\n" 

     

       
434
       
434
       
-
       
                  (batch_num + 1) batch_successes (List.length batch) new_total (new_total + (List.length links_to_upload - new_total));

     

       
435
       
435
       
-
       
                

     

       
436
       
436
       
-
       
                (* Add a delay before processing the next batch *)

     

       
437
       
437
       
-
       
                if batch_num + 1 < List.length batches then begin

     

       
438
       
438
       
-
       
                  log_verbose verbose "  Waiting %.1f seconds before next batch...\n" delay_seconds;

     

       
439
       
439
       
-
       
                  Lwt_unix.sleep delay_seconds >>= fun () ->

     

       
440
       
440
       
-
       
                  Lwt.return (new_total, batch_num + 1)

     

       
441
       
441
       
-
       
                end else

     

       
442
       
442
       
-
       
                  Lwt.return (new_total, batch_num + 1)

     

       
443
       
443
       
-
       
              ) (0, 0) batches >>= fun (final_count, _) ->

     

       
444
       
444
       
-
       
              Lwt.return final_count

     

       
445
       
445
       
-
       
            )

     

       
446
       
446
       
-
       
            (fun exn ->

     

       
447
       
447
       
-
       
              log "Error during upload operation: %s\n" (Printexc.to_string exn);

     

       
448
       
448
       
-
       
              Lwt.return 0

     

       
449
       
449
       
-
       
            )

     

       
450
       
450
       
-
       
        ) in

     

       
451
       
451
       
-
       
        

     

       
452
       
452
       
-
       
        (* Update the links file with the new karakeep_ids *)

     

       
453
       
453
       
-
       
        update_links_file links_file links updated_links;

     

       
454
       
454
       
-
       
        

     

       
455
       
455
       
-
       
        log "Upload complete. %d/%d links uploaded successfully.\n" 

     

       
456
       
456
       
-
       
          result (List.length links_to_upload);

     

       
457
       
457
       
-
       
        

     

       
458
       
458
       
-
       
        0

     

       
459
       
459
       
-
       
      end

     

       
460
       
460
       
-
       


     

       
461
       
461
       
-
       
(* Common arguments *)

     

       
462
       
462
       
-
       
let links_file_arg =

     

       
463
       
463
       
-
       
  let doc = "Links YAML file. Defaults to links.yml." in

     

       
464
       
464
       
-
       
  Arg.(value & opt string "links.yml" & info ["file"; "f"] ~doc ~docv:"FILE")

     

       
465
       
465
       
-
       


     

       
466
       
466
       
-
       
let base_url_arg =

     

       
467
       
467
       
-
       
  let doc = "Base URL of the Karakeep instance" in

     

       
468
       
468
       
-
       
  let default = "https://hoard.recoil.org" in

     

       
469
       
469
       
-
       
  Arg.(value & opt string default & info ["url"] ~doc ~docv:"URL")

     

       
470
       
470
       
-
       


     

       
471
       
471
       
-
       
let api_key_arg =

     

       
472
       
472
       
-
       
  let doc = "API key for Karakeep authentication (ak1_<key_id>_<secret>)" in

     

       
473
       
473
       
-
       
  let get_api_key () =

     

       
474
       
474
       
-
       
    let home = try Sys.getenv "HOME" with Not_found -> "." in

     

       
475
       
475
       
-
       
    let key_path = Filename.concat home ".karakeep-api" in

     

       
476
       
476
       
-
       
    try

     

       
477
       
477
       
-
       
      let ic = open_in key_path in

     

       
478
       
478
       
-
       
      let key = input_line ic in

     

       
479
       
479
       
-
       
      close_in ic;

     

       
480
       
480
       
-
       
      Some (String.trim key)

     

       
481
       
481
       
-
       
    with _ -> None

     

       
482
       
482
       
-
       
  in

     

       
483
       
483
       
-
       
  Arg.(value & opt (some string) (get_api_key ()) & info ["api-key"] ~doc ~docv:"API_KEY")

     

       
484
       
484
       
-
       


     

       
485
       
485
       
-
       
let tag_arg =

     

       
486
       
486
       
-
       
  let doc = "Tag to filter or apply to bookmarks" in

     

       
487
       
487
       
-
       
  Arg.(value & opt string "" & info ["tag"; "t"] ~doc ~docv:"TAG")

     

       
488
       
488
       
-
       


     

       
489
       
489
       
-
       
let download_assets_arg =

     

       
490
       
490
       
-
       
  let doc = "Download assets (screenshots, etc.) from Karakeep" in

     

       
491
       
491
       
-
       
  Arg.(value & flag & info ["download-assets"; "d"] ~doc)

     

       
492
       
492
       
-
       


     

       
493
       
493
       
-
       
let base_dir_arg =

     

       
494
       
494
       
-
       
  let doc = "Base directory of the Bushel project" in

     

       
495
       
495
       
-
       
  Arg.(value & opt string "." & info ["dir"; "d"] ~doc ~docv:"DIR")

     

       
496
       
496
       
-
       


     

       
497
       
497
       
-
       
let include_domains_arg =

     

       
498
       
498
       
-
       
  let doc = "Only include links to these domains (comma-separated list)" in

     

       
499
       
499
       
-
       
  Arg.(value & opt (some string) None & info ["include"] ~doc ~docv:"DOMAINS")

     

       
500
       
500
       
-
       


     

       
501
       
501
       
-
       
let exclude_domains_arg =

     

       
502
       
502
       
-
       
  let doc = "Exclude links to these domains (comma-separated list)" in

     

       
503
       
503
       
-
       
  Arg.(value & opt (some string) None & info ["exclude"] ~doc ~docv:"DOMAINS")

     

       
504
       
504
       
-
       


     

       
505
       
505
       
-
       
let concurrent_arg =

     

       
506
       
506
       
-
       
  let doc = "Maximum number of concurrent uploads (default: 5)" in

     

       
507
       
507
       
-
       
  Arg.(value & opt int 5 & info ["concurrent"; "c"] ~doc ~docv:"NUM")

     

       
508
       
508
       
-
       


     

       
509
       
509
       
-
       
let delay_arg =

     

       
510
       
510
       
-
       
  let doc = "Delay in seconds between batches (default: 1.0)" in

     

       
511
       
511
       
-
       
  Arg.(value & opt float 1.0 & info ["delay"] ~doc ~docv:"SECONDS")

     

       
512
       
512
       
-
       


     

       
513
       
513
       
-
       
let limit_arg =

     

       
514
       
514
       
-
       
  let doc = "Limit number of links to upload (for testing)" in

     

       
515
       
515
       
-
       
  Arg.(value & opt (some int) None & info ["limit"; "l"] ~doc ~docv:"NUM")

     

       
516
       
516
       
-
       


     

       
517
       
517
       
-
       
let verbose_arg =

     

       
518
       
518
       
-
       
  let doc = "Show detailed progress information during upload" in

     

       
519
       
519
       
-
       
  Arg.(value & flag & info ["verbose"; "v"] ~doc)

     

       
520
       
520
       
-
       


     

       
521
       
521
       
-
       
(* Command definitions *)

     

       
522
       
522
       
-
       
let init_cmd =

     

       
523
       
523
       
-
       
  let doc = "Initialize a new links.yml file" in

     

       
524
       
524
       
-
       
  let info = Cmd.info "init" ~doc in

     

       
525
       
525
       
-
       
  Cmd.v info Term.(const init_links_file $ links_file_arg)

     

       
526
       
526
       
-
       


     

       
527
       
527
       
-
       
let karakeep_cmd =

     

       
528
       
528
       
-
       
  let doc = "Update links.yml with links from Karakeep" in

     

       
529
       
529
       
-
       
  let info = Cmd.info "karakeep" ~doc in

     

       
530
       
530
       
-
       
  Cmd.v info Term.(const update_from_karakeep $ base_url_arg $ api_key_arg $ tag_arg $ links_file_arg $ download_assets_arg)

     

       
531
       
531
       
-
       


     

       
532
       
532
       
-
       
let bushel_cmd =

     

       
533
       
533
       
-
       
  let doc = "Update links.yml with outgoing links from Bushel entries" in

     

       
534
       
534
       
-
       
  let info = Cmd.info "bushel" ~doc in

     

       
535
       
535
       
-
       
  Cmd.v info Term.(const update_from_bushel $ base_dir_arg $ links_file_arg $ include_domains_arg $ exclude_domains_arg)

     

       
536
       
536
       
-
       


     

       
537
       
537
       
-
       
let upload_cmd =

     

       
538
       
538
       
-
       
  let doc = "Upload links without karakeep data to Karakeep" in

     

       
539
       
539
       
-
       
  let info = Cmd.info "upload" ~doc in

     

       
540
       
540
       
-
       
  Cmd.v info Term.(const upload_to_karakeep $ base_url_arg $ api_key_arg $ links_file_arg $ tag_arg $ concurrent_arg $ delay_arg $ limit_arg $ verbose_arg)

     

       
541
       
541
       
-
       


     

       
542
       
542
       
-
       
(* Export the term and cmd for use in main bushel.ml *)

     

       
543
       
543
       
-
       
let cmd =

     

       
544
       
544
       
-
       
  let doc = "Manage links between Bushel and Karakeep" in

     

       
545
       
545
       
-
       
  let info = Cmd.info "links" ~doc in

     

       
546
       
546
       
-
       
  Cmd.group info [init_cmd; karakeep_cmd; bushel_cmd; upload_cmd]

     

       
547
       
547
       
-
       


     

       
548
       
548
       
-
       
(* For standalone execution *)

     

       
549
       
549
       
-
       
(* Main entry point removed - accessed through bushel_main.ml *)