My agentic slop goes here. Not intended for anyone else!
anil.recoil.org
1(*
2 * Persistent storage for Atom feed entries using Cacheio and Jsonfeed
3 *)
4
5let src = Logs.Src.create "river.store" ~doc:"River persistent storage"
6module Log = (val Logs.src_log src : Logs.LOG)
7
8(* Types *)
9
10(* Storage metadata that extends Jsonfeed.Item via unknown fields *)
11type storage_meta = {
12 feed_url : string;
13 feed_name : string;
14 feed_title : string;
15 stored_at : Ptime.t;
16}
17
18(* A stored entry is a Jsonfeed.Item.t with storage metadata in unknown fields *)
19type stored_entry = {
20 item : Jsonfeed.Item.t;
21 meta : storage_meta;
22}
23
24(* Stored entry accessors *)
25let entry_item entry = entry.item
26let entry_feed_url entry = entry.meta.feed_url
27let entry_feed_name entry = entry.meta.feed_name
28let entry_feed_title entry = entry.meta.feed_title
29let entry_stored_at entry = entry.meta.stored_at
30
31type feed_info = {
32 url : string;
33 name : string;
34 title : string;
35 last_updated : Ptime.t;
36 entry_count : int;
37}
38
39type t = {
40 cache : Cacheio.t;
41 base_dir : Eio.Fs.dir_ty Eio.Path.t;
42}
43
44(* Helper functions *)
45
46let make_feed_key feed_url =
47 (* Use SHA256 hash of feed URL as directory name for safety *)
48 let hash = Digest.string feed_url |> Digest.to_hex in
49 "feeds/" ^ hash
50
51let make_entry_key feed_url atom_id =
52 (* Store entry under feed directory with atom_id hash *)
53 let feed_key = make_feed_key feed_url in
54 let entry_hash = Digest.string atom_id |> Digest.to_hex in
55 feed_key ^ "/entries/" ^ entry_hash
56
57let make_feed_meta_key feed_url =
58 let feed_key = make_feed_key feed_url in
59 feed_key ^ "/meta.json"
60
61(* JSON serialization using Jsonfeed and Jsont *)
62
63(* Storage metadata codec - stores feed info and storage timestamp *)
64let storage_meta_jsont : storage_meta Jsont.t =
65 Jsont.Object.(
66 map ~kind:"StorageMeta" (fun feed_url feed_name feed_title stored_at : storage_meta ->
67 { feed_url; feed_name; feed_title; stored_at })
68 |> mem "x_river_feed_url" Jsont.string ~enc:(fun m -> m.feed_url)
69 |> mem "x_river_feed_name" Jsont.string ~enc:(fun m -> m.feed_name)
70 |> mem "x_river_feed_title" Jsont.string ~enc:(fun m -> m.feed_title)
71 |> mem "x_river_stored_at" Jsonfeed.Rfc3339.jsont ~enc:(fun m -> m.stored_at)
72 |> finish
73 )
74
75(* Codec for feed_info *)
76let feed_meta_jsont : feed_info Jsont.t =
77 Jsont.Object.(
78 map ~kind:"FeedInfo" (fun url name title last_updated entry_count : feed_info ->
79 { url; name; title; last_updated; entry_count })
80 |> mem "url" Jsont.string ~enc:(fun (m : feed_info) -> m.url)
81 |> mem "name" Jsont.string ~enc:(fun m -> m.name)
82 |> mem "title" Jsont.string ~enc:(fun m -> m.title)
83 |> mem "last_updated" Jsonfeed.Rfc3339.jsont ~enc:(fun m -> m.last_updated)
84 |> mem "entry_count" Jsont.int ~enc:(fun m -> m.entry_count)
85 |> finish
86 )
87
88(* Helper to create item with storage metadata in unknown fields *)
89let merge_storage_meta item meta =
90 let meta_json = Jsont_bytesrw.encode_string' storage_meta_jsont meta
91 |> Result.get_ok in
92 let meta_unknown = Jsont_bytesrw.decode_string' Jsont.json meta_json
93 |> Result.get_ok in
94 Jsonfeed.Item.create
95 ~id:(Jsonfeed.Item.id item)
96 ~content:(Jsonfeed.Item.content item)
97 ?url:(Jsonfeed.Item.url item)
98 ?external_url:(Jsonfeed.Item.external_url item)
99 ?title:(Jsonfeed.Item.title item)
100 ?summary:(Jsonfeed.Item.summary item)
101 ?image:(Jsonfeed.Item.image item)
102 ?banner_image:(Jsonfeed.Item.banner_image item)
103 ?date_published:(Jsonfeed.Item.date_published item)
104 ?date_modified:(Jsonfeed.Item.date_modified item)
105 ?authors:(Jsonfeed.Item.authors item)
106 ?tags:(Jsonfeed.Item.tags item)
107 ?language:(Jsonfeed.Item.language item)
108 ?attachments:(Jsonfeed.Item.attachments item)
109 ?references:(Jsonfeed.Item.references item)
110 ~unknown:meta_unknown
111 ()
112
113(* Helper to extract storage metadata from item's unknown fields *)
114let extract_storage_meta item =
115 let unknown = Jsonfeed.Item.unknown item in
116 let meta_str = Jsont_bytesrw.encode_string' Jsont.json unknown |> Result.get_ok in
117 match Jsont_bytesrw.decode_string' storage_meta_jsont meta_str with
118 | Ok meta -> meta
119 | Error e -> failwith ("Missing storage metadata: " ^ Jsont.Error.to_string e)
120
121(* Stored entry codec - just wraps Jsonfeed.Item.jsont *)
122let stored_entry_jsont : stored_entry Jsont.t =
123 let kind = "StoredEntry" in
124 let of_string s =
125 match Jsont_bytesrw.decode_string' Jsonfeed.Item.jsont s with
126 | Ok item -> Ok { item; meta = extract_storage_meta item }
127 | Error e -> Error (Jsont.Error.to_string e)
128 in
129 let enc entry =
130 let item_with_meta = merge_storage_meta entry.item entry.meta in
131 match Jsont_bytesrw.encode_string' Jsonfeed.Item.jsont item_with_meta with
132 | Ok s -> s
133 | Error e -> failwith ("Failed to encode: " ^ Jsont.Error.to_string e)
134 in
135 Jsont.of_of_string ~kind of_string ~enc
136
137(* Encode/decode functions *)
138let entry_to_string entry =
139 match Jsont_bytesrw.encode_string' stored_entry_jsont entry with
140 | Ok s -> s
141 | Error err -> failwith ("Failed to encode entry: " ^ Jsont.Error.to_string err)
142
143let entry_of_string s =
144 match Jsont_bytesrw.decode_string' stored_entry_jsont s with
145 | Ok entry -> entry
146 | Error err -> failwith ("Failed to parse entry: " ^ Jsont.Error.to_string err)
147
148let feed_meta_to_string meta =
149 match Jsont_bytesrw.encode_string' feed_meta_jsont meta with
150 | Ok s -> s
151 | Error err -> failwith ("Failed to encode feed metadata: " ^ Jsont.Error.to_string err)
152
153let feed_meta_of_string s =
154 match Jsont_bytesrw.decode_string' feed_meta_jsont s with
155 | Ok meta -> meta
156 | Error err -> failwith ("Failed to parse feed metadata: " ^ Jsont.Error.to_string err)
157
158(* Store creation *)
159
160let create ~base_dir =
161 let cache_dir = Eio.Path.(base_dir / "river_store") in
162 (try
163 Eio.Path.mkdir ~perm:0o755 cache_dir
164 with Eio.Io (Eio.Fs.E (Already_exists _), _) -> ());
165 let cache = Cacheio.create ~base_dir:cache_dir in
166 Log.info (fun m -> m "Created River store at %a" Eio.Path.pp cache_dir);
167 { cache; base_dir = cache_dir }
168
169let create_with_xdge xdge =
170 let cache = Cacheio.create_with_xdge xdge in
171 let base_dir = Eio.Path.( / ) (Xdge.cache_dir xdge) "river_store" in
172 Log.info (fun m -> m "Created River store with XDG at %a" Eio.Path.pp base_dir);
173 { cache; base_dir }
174
175(* Convert Post.t to Jsonfeed.Item.t *)
176let item_of_post ~feed_url ~feed_name ~feed_title (post : Post.t) =
177 let content =
178 let html = Soup.to_string post.content in
179 `Html html
180 in
181 let url = Option.map Uri.to_string post.link in
182 let authors =
183 if post.author = "" then None
184 else Some [Jsonfeed.Author.create ~name:post.author ()]
185 in
186 let tags = if post.tags = [] then None else Some post.tags in
187 let item = Jsonfeed.Item.create
188 ~id:post.id
189 ~content
190 ?url
191 ?title:(if post.title = "" then None else Some post.title)
192 ?summary:post.summary
193 ?date_published:post.date
194 ?date_modified:post.date
195 ?authors
196 ?tags
197 ()
198 in
199 let meta = {
200 feed_url;
201 feed_name;
202 feed_title;
203 stored_at = Ptime.of_float_s (Unix.gettimeofday ()) |> Option.get;
204 } in
205 { item; meta }
206
207(* Convert Syndic.Atom.entry to Jsonfeed.Item.t *)
208let item_of_atom ~feed_url ~feed_name ~feed_title (atom_entry : Syndic.Atom.entry) =
209 let atom_id = Uri.to_string atom_entry.id in
210 let date_modified = atom_entry.updated in
211 let date_published = match atom_entry.published with
212 | Some p -> Some p
213 | None -> Some atom_entry.updated
214 in
215 (* Extract content *)
216 let content_html = match atom_entry.content with
217 | Some (Syndic.Atom.Text s) -> Some s
218 | Some (Syndic.Atom.Html (_, s)) -> Some s
219 | Some (Syndic.Atom.Xhtml (_, nodes)) ->
220 let ns_prefix _ = Some "" in
221 Some (String.concat "" (List.map (Syndic.XML.to_string ~ns_prefix) nodes))
222 | Some (Syndic.Atom.Mime _) | Some (Syndic.Atom.Src _) | None -> None
223 in
224 let content_text = match atom_entry.summary with
225 | Some s -> Some (Util.string_of_text_construct s)
226 | None -> None
227 in
228 let content = match content_html, content_text with
229 | Some h, Some t -> `Both (h, t)
230 | Some h, None -> `Html h
231 | None, Some t -> `Text t
232 | None, None -> `Text "" (* Fallback *)
233 in
234 let url = try
235 Some (Uri.to_string (List.find (fun l -> l.Syndic.Atom.rel = Syndic.Atom.Alternate) atom_entry.links).href)
236 with Not_found ->
237 match atom_entry.links with
238 | l :: _ -> Some (Uri.to_string l.href)
239 | [] -> None
240 in
241 let tags =
242 let cat_tags = List.map (fun cat -> cat.Syndic.Atom.term) atom_entry.categories in
243 if cat_tags = [] then None else Some cat_tags
244 in
245 let summary = match atom_entry.summary with
246 | Some s -> Some (Util.string_of_text_construct s)
247 | None -> None
248 in
249 let item = Jsonfeed.Item.create
250 ~id:atom_id
251 ~content
252 ?url
253 ~title:(Util.string_of_text_construct atom_entry.title)
254 ?summary
255 ?date_published
256 ~date_modified
257 ?tags
258 ()
259 in
260 let meta = {
261 feed_url;
262 feed_name;
263 feed_title;
264 stored_at = Ptime.of_float_s (Unix.gettimeofday ()) |> Option.get;
265 } in
266 { item; meta }
267
268(* Feed metadata management *)
269let update_feed_meta store ~feed_url ~feed_name ~feed_title ~sw:_ =
270 let key = make_feed_meta_key feed_url in
271 let meta = {
272 url = feed_url;
273 name = feed_name;
274 title = feed_title;
275 last_updated = Ptime.of_float_s (Unix.gettimeofday ()) |> Option.get;
276 entry_count = 0;
277 } in
278 let json_str = feed_meta_to_string meta in
279 let source = Eio.Flow.string_source json_str in
280 Cacheio.put store.cache ~key ~source ~ttl:None ();
281 Log.debug (fun m -> m "Updated feed metadata for %s" feed_url)
282
283let get_feed_meta store ~feed_url ~sw =
284 let key = make_feed_meta_key feed_url in
285 match Cacheio.get store.cache ~key ~sw with
286 | None -> None
287 | Some source ->
288 try
289 let json_str = Eio.Buf_read.(parse_exn take_all) source ~max_size:Int.max_int in
290 Some (feed_meta_of_string json_str)
291 with e ->
292 Log.err (fun m -> m "Failed to parse feed metadata: %s" (Printexc.to_string e));
293 None
294
295(* Entry storage *)
296
297let store_entry store ~feed_url ~feed_name ~feed_title ~post ~sw =
298 let entry = item_of_post ~feed_url ~feed_name ~feed_title post in
299 let key = make_entry_key feed_url (Jsonfeed.Item.id entry.item) in
300 let json_str = entry_to_string entry in
301 let source = Eio.Flow.string_source json_str in
302 Cacheio.put store.cache ~key ~source ~ttl:None ();
303 Log.debug (fun m -> m "Stored entry %s for feed %s" (Jsonfeed.Item.id entry.item) feed_url);
304 (* Update feed metadata *)
305 update_feed_meta store ~feed_url ~feed_name ~feed_title ~sw
306
307let store_posts store ~feed_url ~feed_name ~feed_title ~posts ~sw =
308 Log.info (fun m -> m "Storing %d posts for feed %s" (List.length posts) feed_url);
309 List.iter (fun post ->
310 store_entry store ~feed_url ~feed_name ~feed_title ~post ~sw
311 ) posts;
312 Log.info (fun m -> m "Stored %d entries for feed %s" (List.length posts) feed_url)
313
314let store_atom_entries store ~feed_url ~feed_name ~feed_title ~entries ~sw =
315 Log.info (fun m -> m "Storing %d Atom entries for feed %s" (List.length entries) feed_url);
316 List.iter (fun atom_entry ->
317 let entry = item_of_atom ~feed_url ~feed_name ~feed_title atom_entry in
318 let key = make_entry_key feed_url (Jsonfeed.Item.id entry.item) in
319 let json_str = entry_to_string entry in
320 let source = Eio.Flow.string_source json_str in
321 Cacheio.put store.cache ~key ~source ~ttl:None ();
322 Log.debug (fun m -> m "Stored Atom entry %s" (Jsonfeed.Item.id entry.item));
323 ) entries;
324 update_feed_meta store ~feed_url ~feed_name ~feed_title ~sw;
325 Log.info (fun m -> m "Stored %d Atom entries for feed %s" (List.length entries) feed_url)
326
327(* Entry retrieval *)
328
329let get_entry store ~feed_url ~atom_id ~sw =
330 let key = make_entry_key feed_url atom_id in
331 match Cacheio.get store.cache ~key ~sw with
332 | None -> None
333 | Some source ->
334 try
335 let json_str = Eio.Buf_read.(parse_exn take_all) source ~max_size:Int.max_int in
336 Some (entry_of_string json_str)
337 with e ->
338 Log.err (fun m -> m "Failed to parse entry: %s" (Printexc.to_string e));
339 None
340
341let list_entries store ~feed_url =
342 let feed_key = make_feed_key feed_url in
343 let prefix = feed_key ^ "/entries/" in
344 let entries = Cacheio.scan store.cache in
345 let feed_entries = List.filter_map (fun (cache_entry : Cacheio.Entry.t) ->
346 let key = Cacheio.Entry.key cache_entry in
347 if String.starts_with ~prefix key then
348 Eio.Switch.run @@ fun sw ->
349 match Cacheio.get store.cache ~key ~sw with
350 | None -> None
351 | Some source ->
352 try
353 let json_str = Eio.Buf_read.(parse_exn take_all) source ~max_size:Int.max_int in
354 Some (entry_of_string json_str)
355 with e ->
356 Log.err (fun m -> m "Failed to parse entry from scan: %s" (Printexc.to_string e));
357 None
358 else None
359 ) entries in
360 (* Sort by date_modified, newest first *)
361 List.sort (fun a b ->
362 let time_a = Jsonfeed.Item.date_modified a.item |> Option.value ~default:a.meta.stored_at in
363 let time_b = Jsonfeed.Item.date_modified b.item |> Option.value ~default:b.meta.stored_at in
364 Ptime.compare time_b time_a
365 ) feed_entries
366
367let list_entries_filtered store ~feed_url ?since ?until ?limit ?(sort=`Updated) () =
368 let entries = list_entries store ~feed_url in
369 (* Filter by time *)
370 let entries = match since with
371 | None -> entries
372 | Some t -> List.filter (fun e ->
373 let time = Jsonfeed.Item.date_modified e.item |> Option.value ~default:e.meta.stored_at in
374 Ptime.is_later time ~than:t || Ptime.equal time t) entries
375 in
376 let entries = match until with
377 | None -> entries
378 | Some t -> List.filter (fun e ->
379 let time = Jsonfeed.Item.date_modified e.item |> Option.value ~default:e.meta.stored_at in
380 Ptime.is_earlier time ~than:t || Ptime.equal time t) entries
381 in
382 (* Sort *)
383 let entries = match sort with
384 | `Published -> List.sort (fun a b ->
385 let pa = Jsonfeed.Item.date_published a.item in
386 let pb = Jsonfeed.Item.date_published b.item in
387 match pa, pb with
388 | Some ta, Some tb -> Ptime.compare tb ta
389 | None, Some _ -> 1
390 | Some _, None -> -1
391 | None, None ->
392 let ta = Jsonfeed.Item.date_modified a.item |> Option.value ~default:a.meta.stored_at in
393 let tb = Jsonfeed.Item.date_modified b.item |> Option.value ~default:b.meta.stored_at in
394 Ptime.compare tb ta
395 ) entries
396 | `Updated -> List.sort (fun a b ->
397 let ta = Jsonfeed.Item.date_modified a.item |> Option.value ~default:a.meta.stored_at in
398 let tb = Jsonfeed.Item.date_modified b.item |> Option.value ~default:b.meta.stored_at in
399 Ptime.compare tb ta
400 ) entries
401 | `Stored -> List.sort (fun a b -> Ptime.compare b.meta.stored_at a.meta.stored_at) entries
402 in
403 (* Limit *)
404 match limit with
405 | None -> entries
406 | Some n -> List.filteri (fun i _ -> i < n) entries
407
408let exists_entry store ~feed_url ~atom_id =
409 let key = make_entry_key feed_url atom_id in
410 Cacheio.exists store.cache ~key
411
412let get_recent_entries store ?(limit=50) () =
413 let entries = Cacheio.scan store.cache in
414 let all_entries = List.filter_map (fun (cache_entry : Cacheio.Entry.t) ->
415 let key = Cacheio.Entry.key cache_entry in
416 if String.contains key '/' &&
417 String.ends_with ~suffix:"entries/" (String.sub key 0 (String.rindex key '/') ^ "/") then
418 Eio.Switch.run @@ fun sw ->
419 match Cacheio.get store.cache ~key ~sw with
420 | None -> None
421 | Some source ->
422 try
423 let json_str = Eio.Buf_read.(parse_exn take_all) source ~max_size:Int.max_int in
424 Some (entry_of_string json_str)
425 with e ->
426 Log.err (fun m -> m "Failed to parse entry: %s" (Printexc.to_string e));
427 None
428 else None
429 ) entries in
430 let sorted = List.sort (fun a b ->
431 let ta = Jsonfeed.Item.date_modified a.item |> Option.value ~default:a.meta.stored_at in
432 let tb = Jsonfeed.Item.date_modified b.item |> Option.value ~default:b.meta.stored_at in
433 Ptime.compare tb ta
434 ) all_entries in
435 List.filteri (fun i _ -> i < limit) sorted
436
437let find_entry_by_id store ~id =
438 Log.debug (fun m -> m "Searching for entry with ID: %s" id);
439 let entries = Cacheio.scan store.cache in
440 let matching_entry = List.find_map (fun (cache_entry : Cacheio.Entry.t) ->
441 let key = Cacheio.Entry.key cache_entry in
442 if String.contains key '/' &&
443 String.ends_with ~suffix:"entries/" (String.sub key 0 (String.rindex key '/') ^ "/") then
444 Eio.Switch.run @@ fun sw ->
445 match Cacheio.get store.cache ~key ~sw with
446 | None -> None
447 | Some source ->
448 (try
449 let json_str = Eio.Buf_read.(parse_exn take_all) source ~max_size:Int.max_int in
450 let entry = entry_of_string json_str in
451 (* Exact ID match only *)
452 if Jsonfeed.Item.id entry.item = id then
453 Some entry
454 else
455 None
456 with e ->
457 Log.err (fun m -> m "Failed to parse entry: %s" (Printexc.to_string e));
458 None)
459 else None
460 ) entries in
461 (match matching_entry with
462 | Some e -> Log.debug (fun m -> m "Found entry: %s"
463 (Jsonfeed.Item.title e.item |> Option.value ~default:"(no title)"))
464 | None -> Log.debug (fun m -> m "No entry found with ID: %s" id));
465 matching_entry
466
467(* Entry management *)
468
469let delete_entry store ~feed_url ~atom_id =
470 let key = make_entry_key feed_url atom_id in
471 Cacheio.delete store.cache ~key;
472 Log.info (fun m -> m "Deleted entry %s from feed %s" atom_id feed_url)
473
474let delete_feed store ~feed_url =
475 let feed_key = make_feed_key feed_url in
476 let prefix = feed_key ^ "/" in
477 let entries = Cacheio.scan store.cache in
478 let count = ref 0 in
479 List.iter (fun (cache_entry : Cacheio.Entry.t) ->
480 let key = Cacheio.Entry.key cache_entry in
481 if String.starts_with ~prefix key then begin
482 Cacheio.delete store.cache ~key;
483 incr count
484 end
485 ) entries;
486 Log.info (fun m -> m "Deleted feed %s (%d entries)" feed_url !count)
487
488let prune_entries store ~feed_url ~keep =
489 let entries = list_entries store ~feed_url in
490 let to_delete = List.filteri (fun i _ -> i >= keep) entries in
491 List.iter (fun entry ->
492 delete_entry store ~feed_url ~atom_id:(Jsonfeed.Item.id entry.item)
493 ) to_delete;
494 let deleted = List.length to_delete in
495 Log.info (fun m -> m "Pruned %d entries from feed %s (kept %d)" deleted feed_url keep);
496 deleted
497
498let prune_old_entries store ~feed_url ~older_than =
499 let entries = list_entries store ~feed_url in
500 let to_delete = List.filter (fun e ->
501 let time = Jsonfeed.Item.date_modified e.item |> Option.value ~default:e.meta.stored_at in
502 Ptime.is_earlier time ~than:older_than
503 ) entries in
504 List.iter (fun entry ->
505 delete_entry store ~feed_url ~atom_id:(Jsonfeed.Item.id entry.item)
506 ) to_delete;
507 let deleted = List.length to_delete in
508 Log.info (fun m -> m "Pruned %d old entries from feed %s" deleted feed_url);
509 deleted
510
511(* Feed information *)
512
513let list_feeds store =
514 let feed_entries = Cacheio.scan store.cache in
515 let feed_metas = List.filter_map (fun (cache_entry : Cacheio.Entry.t) ->
516 let key = Cacheio.Entry.key cache_entry in
517 if String.ends_with ~suffix:"/meta.json" key then
518 Eio.Switch.run @@ fun sw ->
519 match Cacheio.get store.cache ~key ~sw with
520 | None -> None
521 | Some source ->
522 try
523 let json_str = Eio.Buf_read.(parse_exn take_all) source ~max_size:Int.max_int in
524 Some (feed_meta_of_string json_str)
525 with e ->
526 Log.err (fun m -> m "Failed to parse feed metadata: %s" (Printexc.to_string e));
527 None
528 else None
529 ) feed_entries in
530 (* Count entries for each feed *)
531 List.map (fun meta ->
532 let entries = list_entries store ~feed_url:meta.url in
533 { meta with entry_count = List.length entries }
534 ) feed_metas
535
536let get_feed_info store ~feed_url =
537 Eio.Switch.run @@ fun sw ->
538 match get_feed_meta store ~feed_url ~sw with
539 | None -> None
540 | Some meta ->
541 let entries = list_entries store ~feed_url in
542 Some { meta with entry_count = List.length entries }
543
544let stats store =
545 Cacheio.stats store.cache
546
547(* Maintenance *)
548
549let expire store =
550 Cacheio.expire store.cache
551
552let compact _store =
553 (* TODO: Implement compaction logic *)
554 Log.info (fun m -> m "Compaction not yet implemented")
555
556(* Export/Import *)
557
558let export_to_atom store ~feed_url ?title ?limit () =
559 let entries = match limit with
560 | None -> list_entries store ~feed_url
561 | Some n -> list_entries_filtered store ~feed_url ~limit:n ()
562 in
563 let atom_entries = List.map (fun entry ->
564 let item = entry.item in
565 let id = Uri.of_string (Jsonfeed.Item.id item) in
566 let entry_title : Syndic.Atom.text_construct =
567 Syndic.Atom.Text (Jsonfeed.Item.title item |> Option.value ~default:"(no title)") in
568 let links = match Jsonfeed.Item.url item with
569 | Some url_str -> [Syndic.Atom.link ~rel:Syndic.Atom.Alternate (Uri.of_string url_str)]
570 | None -> []
571 in
572 let content_str = match Jsonfeed.Item.content item with
573 | `Html h -> h
574 | `Text t -> t
575 | `Both (h, _) -> h
576 in
577 let entry_content : Syndic.Atom.content = Syndic.Atom.Html (None, content_str) in
578 let author_name = match Jsonfeed.Item.authors item with
579 | Some (a :: _) -> Jsonfeed.Author.name a |> Option.value ~default:entry.meta.feed_name
580 | _ -> entry.meta.feed_name
581 in
582 let author = Syndic.Atom.author author_name in
583 let authors = (author, []) in
584 let updated = Jsonfeed.Item.date_modified item |> Option.value ~default:entry.meta.stored_at in
585 Syndic.Atom.entry ~id ~title:entry_title ~updated
586 ?published:(Jsonfeed.Item.date_published item)
587 ~links ~content:entry_content ~authors ()
588 ) entries in
589 let feed_title : Syndic.Atom.text_construct = match title with
590 | Some t -> Syndic.Atom.Text t
591 | None -> Syndic.Atom.Text ("Archive: " ^ feed_url)
592 in
593 let feed_id = Uri.of_string ("urn:river:archive:" ^ (Digest.string feed_url |> Digest.to_hex)) in
594 let feed_updated = match entries with
595 | [] -> Ptime.of_float_s (Unix.gettimeofday ()) |> Option.get
596 | e :: _ -> Jsonfeed.Item.date_modified e.item |> Option.value ~default:e.meta.stored_at
597 in
598 {
599 Syndic.Atom.id = feed_id;
600 title = feed_title;
601 updated = feed_updated;
602 entries = atom_entries;
603 authors = [];
604 categories = [];
605 contributors = [];
606 generator = Some {
607 Syndic.Atom.version = Some "1.0";
608 uri = None;
609 content = "River Store";
610 };
611 icon = None;
612 links = [];
613 logo = None;
614 rights = None;
615 subtitle = None;
616 }
617
618let import_from_atom store ~feed_url ~feed_name ~feed ~sw =
619 let entries = feed.Syndic.Atom.entries in
620 store_atom_entries store ~feed_url ~feed_name ~feed_title:(Util.string_of_text_construct feed.title) ~entries ~sw;
621 List.length entries
622
623(* Pretty printing *)
624
625let pp_entry fmt entry =
626 let item = entry.item in
627 Format.fprintf fmt "@[<v 2>Entry:@,";
628 Format.fprintf fmt "ID: %s@," (Jsonfeed.Item.id item);
629 Format.fprintf fmt "Title: %s@," (Jsonfeed.Item.title item |> Option.value ~default:"(no title)");
630 Format.fprintf fmt "URL: %s@," (Jsonfeed.Item.url item |> Option.value ~default:"(none)");
631 (match Jsonfeed.Item.date_published item with
632 | Some t -> Format.fprintf fmt "Published: %s@," (Ptime.to_rfc3339 t)
633 | None -> ());
634 (match Jsonfeed.Item.date_modified item with
635 | Some t -> Format.fprintf fmt "Modified: %s@," (Ptime.to_rfc3339 t)
636 | None -> ());
637 Format.fprintf fmt "Feed: %s (%s)@," entry.meta.feed_name entry.meta.feed_url;
638 Format.fprintf fmt "Stored: %s@]" (Ptime.to_rfc3339 entry.meta.stored_at)
639
640let pp_feed_info fmt info =
641 Format.fprintf fmt "@[<v 2>Feed:@,";
642 Format.fprintf fmt "Name: %s@," info.name;
643 Format.fprintf fmt "Title: %s@," info.title;
644 Format.fprintf fmt "URL: %s@," info.url;
645 Format.fprintf fmt "Last updated: %s@," (Ptime.to_rfc3339 info.last_updated);
646 Format.fprintf fmt "Entries: %d@]" info.entry_count
647
648let pp fmt store =
649 let feeds = list_feeds store in
650 Format.fprintf fmt "@[<v 2>River Store:@,";
651 Format.fprintf fmt "Base dir: %a@," Eio.Path.pp store.base_dir;
652 Format.fprintf fmt "Feeds: %d@," (List.length feeds);
653 List.iter (fun feed ->
654 Format.fprintf fmt " - %s: %d entries@," feed.name feed.entry_count
655 ) feeds;
656 Format.fprintf fmt "@]"