commit f3579c5a2d5407e65680f43e0b0abc96f57e3500 · nekomimi.pet/microcosm-rs

+14

Cargo.lock

···

       3149
       3149
        
       ]

     

       3150
       3150
        
       

     

       3151
       3151
        
       [[package]]

     

       3152
       3152
       +
       name = "serde_qs"

     

       3153
       3153
       +
       version = "1.0.0-rc.3"

     

       3154
       3154
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       3155
       3155
       +
       checksum = "4cb0b9062a400c31442e67d1f2b1e7746bebd691110ebee1b7d0c7293b04fab1"

     

       3156
       3156
       +
       dependencies = [

     

       3157
       3157
       +
        "itoa",

     

       3158
       3158
       +
        "percent-encoding",

     

       3159
       3159
       +
        "ryu",

     

       3160
       3160
       +
        "serde",

     

       3161
       3161
       +
        "thiserror 2.0.12",

     

       3162
       3162
       +
       ]

     

       3163
       3163
       +
       

     

       3164
       3164
       +
       [[package]]

     

       3152
       3165
        
       name = "serde_spanned"

     

       3153
       3166
        
       version = "0.6.8"

     

       3154
       3167
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       3827
       3840
        
        "semver",

     

       3828
       3841
        
        "serde",

     

       3829
       3842
        
        "serde_json",

     

       3843
       3843
       +
        "serde_qs",

     

       3830
       3844
        
        "sha2",

     

       3831
       3845
        
        "tempfile",

     

       3832
       3846
        
        "thiserror 2.0.12",

ufos/Cargo.toml

···

       23
       23
        
       semver = "1.0.26"

     

       24
       24
        
       serde = "1.0.219"

     

       25
       25
        
       serde_json = "1.0.140"

     

       26
       26
       +
       serde_qs = "1.0.0-rc.3"

     

       26
       27
        
       sha2 = "0.10.9"

     

       27
       28
        
       thiserror = "2.0.12"

     

       28
       29
        
       tokio = { version = "1.44.2", features = ["full", "sync", "time"] }

ufos/src/lib.rs

···

       3
       3
        
       pub mod error;

     

       4
       4
        
       pub mod file_consumer;

     

       5
       5
        
       pub mod index_html;

     

       6
       6
       +
       pub mod qs_query;

     

       6
       7
        
       pub mod server;

     

       7
       8
        
       pub mod storage;

     

       8
       9
        
       pub mod storage_fjall;

+73

ufos/src/qs_query.rs

···

       1
       1
       +
       use async_trait::async_trait;

     

       2
       2
       +
       use dropshot::{

     

       3
       3
       +
           ApiEndpointBodyContentType, ExclusiveExtractor, ExtractorMetadata, HttpError, RequestContext,

     

       4
       4
       +
           RequestInfo, ServerContext, SharedExtractor,

     

       5
       5
       +
       };

     

       6
       6
       +
       /// copied from https://github.com/oxidecomputer/dropshot/blob/695e1d8872c988c43066eb0848c87c127eeda361/dropshot/src/extractor/query.rs

     

       7
       7
       +
       /// Apache 2.0: https://github.com/oxidecomputer/dropshot/blob/695e1d8872c988c43066eb0848c87c127eeda361/LICENSE

     

       8
       8
       +
       use schemars::JsonSchema;

     

       9
       9
       +
       use serde::de::DeserializeOwned;

     

       10
       10
       +
       

     

       11
       11
       +
       /// `VecsAllowedQuery<QueryType>` is an extractor used to deserialize an

     

       12
       12
       +
       /// instance of `QueryType` from an HTTP request's query string.  `QueryType`

     

       13
       13
       +
       /// is any structure of yours that implements [serde::Deserialize] and

     

       14
       14
       +
       /// [schemars::JsonSchema].  See the crate documentation for more information.

     

       15
       15
       +
       #[derive(Debug)]

     

       16
       16
       +
       pub struct VecsAllowedQuery<QueryType: DeserializeOwned + JsonSchema + Send + Sync> {

     

       17
       17
       +
           inner: QueryType,

     

       18
       18
       +
       }

     

       19
       19
       +
       impl<QueryType: DeserializeOwned + JsonSchema + Send + Sync> VecsAllowedQuery<QueryType> {

     

       20
       20
       +
           // TODO drop this in favor of Deref?  + Display and Debug for convenience?

     

       21
       21
       +
           pub fn into_inner(self) -> QueryType {

     

       22
       22
       +
               self.inner

     

       23
       23
       +
           }

     

       24
       24
       +
       }

     

       25
       25
       +
       

     

       26
       26
       +
       /// Given an HTTP request, pull out the query string and attempt to deserialize

     

       27
       27
       +
       /// it as an instance of `QueryType`.

     

       28
       28
       +
       fn http_request_load_query<QueryType>(

     

       29
       29
       +
           request: &RequestInfo,

     

       30
       30
       +
       ) -> Result<VecsAllowedQuery<QueryType>, HttpError>

     

       31
       31
       +
       where

     

       32
       32
       +
           QueryType: DeserializeOwned + JsonSchema + Send + Sync,

     

       33
       33
       +
       {

     

       34
       34
       +
           let raw_query_string = request.uri().query().unwrap_or("");

     

       35
       35
       +
           // TODO-correctness: are query strings defined to be urlencoded in this way?

     

       36
       36
       +
           match serde_qs::from_str(raw_query_string) {

     

       37
       37
       +
               Ok(q) => Ok(VecsAllowedQuery { inner: q }),

     

       38
       38
       +
               Err(e) => Err(HttpError::for_bad_request(

     

       39
       39
       +
                   None,

     

       40
       40
       +
                   format!("unable to parse query string: {}", e),

     

       41
       41
       +
               )),

     

       42
       42
       +
           }

     

       43
       43
       +
       }

     

       44
       44
       +
       

     

       45
       45
       +
       // The `SharedExtractor` implementation for Query<QueryType> describes how to

     

       46
       46
       +
       // construct an instance of `Query<QueryType>` from an HTTP request: namely, by

     

       47
       47
       +
       // parsing the query string to an instance of `QueryType`.

     

       48
       48
       +
       // TODO-cleanup We shouldn't have to use the "'static" bound on `QueryType`

     

       49
       49
       +
       // here.  It seems like we ought to be able to use 'async_trait, but that

     

       50
       50
       +
       // doesn't seem to be defined.

     

       51
       51
       +
       #[async_trait]

     

       52
       52
       +
       impl<QueryType> SharedExtractor for VecsAllowedQuery<QueryType>

     

       53
       53
       +
       where

     

       54
       54
       +
           QueryType: JsonSchema + DeserializeOwned + Send + Sync + 'static,

     

       55
       55
       +
       {

     

       56
       56
       +
           async fn from_request<Context: ServerContext>(

     

       57
       57
       +
               rqctx: &RequestContext<Context>,

     

       58
       58
       +
           ) -> Result<VecsAllowedQuery<QueryType>, HttpError> {

     

       59
       59
       +
               http_request_load_query(&rqctx.request)

     

       60
       60
       +
           }

     

       61
       61
       +
       

     

       62
       62
       +
           fn metadata(body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata {

     

       63
       63
       +
               // HACK: would love to use Query here but it "helpfully" panics when it sees a Vec.

     

       64
       64
       +
               // we can't really get at enough of Query's logic to use it directly, sadly, so the

     

       65
       65
       +
               // resulting openapi docs suck (query params are listed as body payload, example

     

       66
       66
       +
               // codes make no sense, etc.)

     

       67
       67
       +
               //

     

       68
       68
       +
               // trying to hack the resulting ExtractorMetadata to look like Query's is a pain:

     

       69
       69
       +
               // things almost work out but then something in dropshot won't be `pub` and it falls

     

       70
       70
       +
               // apart. maybe it's possible, i didn't get it in the time i had.

     

       71
       71
       +
               dropshot::TypedBody::<QueryType>::metadata(body_content_type)

     

       72
       72
       +
           }

     

       73
       73
       +
       }

+43 -11

ufos/src/server.rs

···

       1
       1
        
       use crate::index_html::INDEX_HTML;

     

       2
       2
       +
       use crate::qs_query::VecsAllowedQuery;

     

       2
       3
        
       use crate::storage::StoreReader;

     

       3
       4
        
       use crate::store_types::{HourTruncatedCursor, WeekTruncatedCursor};

     

       4
       5
        
       use crate::{ConsumerInfo, Cursor, JustCount, Nsid, NsidCount, OrderCollectionsBy, UFOsRecord};

     
···

       16
       17
        
       use dropshot::Query;

     

       17
       18
        
       use dropshot::RequestContext;

     

       18
       19
        
       use dropshot::ServerBuilder;

     

       20
       20
       +
       

     

       19
       21
        
       use http::{Response, StatusCode};

     

       20
       22
        
       use schemars::JsonSchema;

     

       21
       23
        
       use serde::{Deserialize, Serialize};

     
···

       83
       85
        
           storage: serde_json::Value,

     

       84
       86
        
           consumer: ConsumerInfo,

     

       85
       87
        
       }

     

       86
       86
       -
       /// Get meta information about UFOs itself

     

       88
       88
       +
       /// UFOs meta-info

     

       87
       89
        
       #[endpoint {

     

       88
       90
        
           method = GET,

     

       89
       91
        
           path = "/meta"

     
···

       145
       147
        
               }

     

       146
       148
        
           }

     

       147
       149
        
       }

     

       148
       148
       -
       /// Get recent records by collection

     

       150
       150
       +
       /// Record samples

     

       151
       151
       +
       ///

     

       152
       152
       +
       /// Get most recent records seen in the firehose, by collection NSID

     

       149
       153
        
       ///

     

       150
       154
        
       /// Multiple collections are supported. They will be delivered in one big array with no

     

       151
       155
        
       /// specified order.

     
···

       195
       199
        
       

     

       196
       200
        
       #[derive(Debug, Deserialize, JsonSchema)]

     

       197
       201
        
       struct TotalSeenCollectionsQuery {

     

       198
       198
       -
           collection: String, // JsonSchema not implemented for Nsid :(

     

       202
       202
       +
           collection: Vec<String>, // JsonSchema not implemented for Nsid :(

     

       203
       203
       +
           /// Limit stats to those seen after this UTC datetime

     

       204
       204
       +
           ///

     

       205
       205
       +
           /// default: 1 week ago

     

       206
       206
       +
           since: Option<DateTime<Utc>>,

     

       207
       207
       +
           /// Limit stats to those seen before this UTC datetime

     

       208
       208
       +
           ///

     

       209
       209
       +
           /// default: now

     

       210
       210
       +
           until: Option<DateTime<Utc>>,

     

       199
       211
        
       }

     

       200
       212
        
       #[derive(Debug, Serialize, JsonSchema)]

     

       201
       213
        
       struct TotalCounts {

     

       202
       214
        
           total_creates: u64,

     

       203
       215
        
           dids_estimate: u64,

     

       204
       216
        
       }

     

       205
       205
       -
       /// Get total records seen by collection

     

       217
       217
       +
       /// Collection stats

     

       218
       218
       +
       ///

     

       219
       219
       +
       /// Get stats for a collection over a specific time period

     

       220
       220
       +
       ///

     

       221
       221
       +
       /// API docs note: the **Body** fields here are actually query parameters!!

     

       222
       222
       +
       ///

     

       223
       223
       +
       /// Due to limitations with dropshot's query parsing (no support for sequences),

     

       224
       224
       +
       /// this is kind of the best i could do for now. sadly.

     

       206
       225
        
       #[endpoint {

     

       207
       226
        
           method = GET,

     

       208
       208
       -
           path = "/records/total-seen"

     

       227
       227
       +
           path = "/collections/stats"

     

       209
       228
        
       }]

     

       210
       229
        
       async fn get_records_total_seen(

     

       211
       230
        
           ctx: RequestContext<Context>,

     

       212
       212
       -
           collection_query: Query<TotalSeenCollectionsQuery>,

     

       231
       231
       +
           query: VecsAllowedQuery<TotalSeenCollectionsQuery>,

     

       213
       232
        
       ) -> OkCorsResponse<HashMap<String, TotalCounts>> {

     

       214
       233
        
           let Context { storage, .. } = ctx.context();

     

       234
       234
       +
           let q = query.into_inner();

     

       215
       235
        
       

     

       216
       216
       -
           let query = collection_query.into_inner();

     

       217
       217
       -
           let collections = to_multiple_nsids(&query.collection)

     

       218
       218
       -
               .map_err(|reason| HttpError::for_bad_request(None, reason))?;

     

       236
       236
       +
           log::warn!("collection: {:?}", q.collection);

     

       237
       237
       +
       

     

       238
       238
       +
           let mut collections = Vec::with_capacity(q.collection.len());

     

       239
       239
       +
           for c in q.collection {

     

       240
       240
       +
               let Ok(nsid) = Nsid::new(c.clone()) else {

     

       241
       241
       +
                   return Err(HttpError::for_bad_request(

     

       242
       242
       +
                       None,

     

       243
       243
       +
                       format!("could not parse collection to nsid: {c}"),

     

       244
       244
       +
                   ));

     

       245
       245
       +
               };

     

       246
       246
       +
               collections.push(nsid);

     

       247
       247
       +
           }

     

       248
       248
       +
       

     

       249
       249
       +
           let since = q.since.map(dt_to_cursor).transpose()?;

     

       250
       250
       +
           let until = q.until.map(dt_to_cursor).transpose()?;

     

       219
       251
        
       

     

       220
       252
        
           let mut seen_by_collection = HashMap::with_capacity(collections.len());

     

       221
       253
        
       

     
···

       283
       315
        
           order: Option<CollectionsQueryOrder>,

     

       284
       316
        
       }

     

       285
       317
        
       

     

       286
       286
       -
       /// Get collection with statistics

     

       318
       318
       +
       /// List collections (with stats)

     

       287
       319
        
       ///

     

       288
       320
        
       /// ## To fetch a full list:

     

       289
       321
        
       ///

     
···

       384
       416
        
           range: Vec<DateTime<Utc>>,

     

       385
       417
        
           series: HashMap<String, Vec<JustCount>>,

     

       386
       418
        
       }

     

       387
       387
       -
       /// Get timeseries data

     

       419
       419
       +
       /// Collection timeseries stats

     

       388
       420
        
       #[endpoint {

     

       389
       421
        
           method = GET,

     

       390
       422
        
           path = "/timeseries"