zbrox.com
1use super::LexiconSource;
2use crate::lexicon::LexiconDoc;
3use jacquard_common::IntoStatic;
4use miette::{Result, miette};
5use serde::Deserialize;
6use std::collections::HashMap;
7
8#[derive(Debug, Clone)]
9pub struct HttpSource {
10 pub url: String,
11}
12
13#[derive(Deserialize)]
14struct UfosRecord {
15 record: serde_json::Value,
16}
17
18impl HttpSource {
19 fn parse_lexicon_value(
20 value: &serde_json::Value,
21 failed_nsids: &mut std::collections::HashSet<String>,
22 ) -> Option<LexiconDoc<'static>> {
23 match serde_json::to_string(value) {
24 Ok(json) => match serde_json::from_str::<LexiconDoc>(&json) {
25 Ok(doc) => Some(doc.into_static()),
26 Err(_e) => {
27 // Track failed NSID for summary
28 if let serde_json::Value::Object(obj) = value {
29 if let Some(serde_json::Value::String(id)) = obj.get("id") {
30 failed_nsids.insert(id.clone());
31 }
32 }
33 None
34 }
35 },
36 Err(_e) => {
37 // Track failed NSID for summary
38 if let serde_json::Value::Object(obj) = value {
39 if let Some(serde_json::Value::String(id)) = obj.get("id") {
40 failed_nsids.insert(id.clone());
41 }
42 }
43 None
44 }
45 }
46 }
47}
48
49impl LexiconSource for HttpSource {
50 async fn fetch(&self) -> Result<HashMap<String, LexiconDoc<'_>>> {
51 let resp = reqwest::get(&self.url)
52 .await
53 .map_err(|e| miette!("Failed to fetch from {}: {}", self.url, e))?;
54
55 let records: Vec<UfosRecord> = resp
56 .json()
57 .await
58 .map_err(|e| miette!("Failed to parse JSON from {}: {}", self.url, e))?;
59
60 let mut lexicons = HashMap::new();
61 let mut failed_nsids = std::collections::HashSet::new();
62 let total_fetched = records.len();
63
64 for ufos_record in records {
65 if let Some(doc) = Self::parse_lexicon_value(&ufos_record.record, &mut failed_nsids) {
66 let nsid = doc.id.to_string();
67 lexicons.insert(nsid, doc);
68 }
69 }
70
71 if !failed_nsids.is_empty() {
72 eprintln!(
73 "Warning: Failed to parse {} out of {} lexicons from HTTP source",
74 failed_nsids.len(),
75 total_fetched
76 );
77 }
78
79 Ok(lexicons)
80 }
81}