My agentic slop goes here. Not intended for anyone else!
anil.recoil.org
1(** Requests - A modern HTTP client library for OCaml
2
3 Requests is an HTTP client library for OCaml inspired by Python's requests
4 and urllib3 libraries. It provides a simple, intuitive API for making HTTP
5 requests while handling complexities like TLS configuration, connection
6 pooling, retries, and cookie management.
7
8 {2 High-Level API}
9
10 The Requests library offers two main ways to make HTTP requests:
11
12 {b 1. Main API} (Recommended for most use cases)
13
14 The main API maintains state across requests, handles cookies automatically,
15 spawns requests in concurrent fibers, and provides a simple interface:
16
17 {[
18 open Eio_main
19
20 let () = run @@ fun env ->
21 Switch.run @@ fun sw ->
22
23 (* Create a requests instance *)
24 let req = Requests.create ~sw env in
25
26 (* Configure authentication once *)
27 Requests.set_auth req (Requests.Auth.bearer "your-token");
28
29 (* Make concurrent requests using Fiber.both *)
30 let (user, repos) = Eio.Fiber.both
31 (fun () -> Requests.get req "https://api.github.com/user")
32 (fun () -> Requests.get req "https://api.github.com/user/repos") in
33
34 (* Process responses *)
35 let user_data = Response.body user |> Eio.Flow.read_all in
36 let repos_data = Response.body repos |> Eio.Flow.read_all in
37
38 (* No cleanup needed - responses auto-close with the switch *)
39 ]}
40
41 {b 2. One-shot requests} (For stateless operations)
42
43 The One module provides lower-level control for stateless,
44 one-off requests without session state:
45
46 {[
47 (* Create a one-shot client *)
48 let client = Requests.One.create ~clock:env#clock ~net:env#net () in
49
50 (* Make a simple GET request *)
51 let response = Requests.One.get ~sw ~client "https://api.github.com" in
52 Printf.printf "Status: %d\n" (Requests.Response.status_code response);
53
54 (* POST with custom headers and body *)
55 let response = Requests.One.post ~sw ~client
56 ~headers:(Requests.Headers.empty
57 |> Requests.Headers.content_type Requests.Mime.json
58 |> Requests.Headers.set "X-API-Key" "secret")
59 ~body:(Requests.Body.json {|{"name": "Alice"}|})
60 "https://api.example.com/users"
61
62 (* No cleanup needed - responses auto-close with the switch *)
63 ]}
64
65 {2 Features}
66
67 - {b Simple API}: Intuitive functions for GET, POST, PUT, DELETE, etc.
68 - {b Authentication}: Built-in support for Basic, Bearer, Digest, and OAuth
69 - {b Streaming}: Upload and download large files efficiently
70 - {b Retries}: Automatic retry with exponential backoff
71 - {b Timeouts}: Configurable connection and read timeouts
72 - {b Cookie Management}: Automatic cookie handling with persistence
73 - {b TLS/SSL}: Secure connections with certificate verification
74 - {b Error Handling}: Comprehensive error types and recovery
75
76 {2 Common Use Cases}
77
78 {b Working with JSON APIs:}
79 {[
80 let response = Requests.post req "https://api.example.com/data"
81 ~body:(Requests.Body.json {|{"key": "value"}|}) in
82 let body_text =
83 Requests.Response.body response
84 |> Eio.Flow.read_all in
85 print_endline body_text
86 (* Response auto-closes with switch *)
87 ]}
88
89 {b File uploads:}
90 {[
91 let body = Requests.Body.multipart [
92 { name = "file"; filename = Some "document.pdf";
93 content_type = Requests.Mime.pdf;
94 content = `File (Eio.Path.(fs / "document.pdf")) };
95 { name = "description"; filename = None;
96 content_type = Requests.Mime.text_plain;
97 content = `String "Important document" }
98 ] in
99 let response = Requests.post req "https://example.com/upload"
100 ~body
101 (* Response auto-closes with switch *)
102 ]}
103
104 {b Streaming downloads:}
105 {[
106 Requests.One.download ~sw ~client
107 "https://example.com/large-file.zip"
108 ~sink:(Eio.Path.(fs / "download.zip" |> sink))
109 ]}
110
111 {2 Choosing Between Main API and One}
112
113 Use the {b main API (Requests.t)} when you need:
114 - Cookie persistence across requests
115 - Automatic retry handling
116 - Shared authentication across requests
117 - Request/response history tracking
118 - Configuration persistence to disk
119
120 Use {b One} when you need:
121 - One-off stateless requests
122 - Fine-grained control over connections
123 - Minimal overhead
124 - Custom connection pooling
125 - Direct streaming without cookies
126*)
127
128(** {1 Main API}
129
130 The main Requests API provides stateful HTTP clients with automatic cookie
131 management and persistent configuration. Requests execute synchronously by default.
132 Use Eio.Fiber.both or Eio.Fiber.all for concurrent execution.
133*)
134
135type t
136(** A stateful HTTP client that maintains cookies, auth, configuration, and
137 connection pools across requests. The internal clock and network types are
138 hidden from external users. *)
139
140(** {2 Creation and Configuration} *)
141
142val create :
143 sw:Eio.Switch.t ->
144 ?http_pool:(([> float Eio.Time.clock_ty] as 'clock) Eio.Resource.t,
145 ([> [> `Generic] Eio.Net.ty] as 'net) Eio.Resource.t) Conpool.t ->
146 ?https_pool:('clock Eio.Resource.t, 'net Eio.Resource.t) Conpool.t ->
147 ?cookie_jar:Cookeio.jar ->
148 ?default_headers:Headers.t ->
149 ?auth:Auth.t ->
150 ?timeout:Timeout.t ->
151 ?follow_redirects:bool ->
152 ?max_redirects:int ->
153 ?verify_tls:bool ->
154 ?tls_config:Tls.Config.client ->
155 ?max_connections_per_host:int ->
156 ?connection_idle_timeout:float ->
157 ?connection_lifetime:float ->
158 ?retry:Retry.config ->
159 ?persist_cookies:bool ->
160 ?xdg:Xdge.t ->
161 < clock: 'clock Eio.Resource.t; net: 'net Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; .. > ->
162 t
163(** Create a new requests instance with persistent state and connection pooling.
164 All resources are bound to the provided switch and will be cleaned up automatically.
165
166 @param sw Switch for resource management
167 @param http_pool Optional pre-configured HTTP connection pool (creates new if not provided)
168 @param https_pool Optional pre-configured HTTPS connection pool (creates new if not provided)
169 @param cookie_jar Cookie storage (default: empty in-memory jar)
170 @param default_headers Headers included in every request
171 @param auth Default authentication
172 @param timeout Default timeout configuration
173 @param follow_redirects Whether to follow HTTP redirects (default: true)
174 @param max_redirects Maximum redirects to follow (default: 10)
175 @param verify_tls Whether to verify TLS certificates (default: true)
176 @param tls_config Custom TLS configuration for HTTPS pool (default: system CA certs)
177 @param max_connections_per_host Maximum pooled connections per host:port (default: 10)
178 @param connection_idle_timeout Max idle time before closing pooled connection (default: 60s)
179 @param connection_lifetime Max lifetime of any pooled connection (default: 300s)
180 @param retry Retry configuration for failed requests
181 @param persist_cookies Whether to persist cookies to disk (default: false)
182 @param xdg XDG directory context for cookies (required if persist_cookies=true)
183
184 {b Note:} HTTP caching has been disabled for simplicity. See CACHEIO.md for integration notes
185 if you need to restore caching functionality in the future.
186*)
187
188(** {2 Configuration Management} *)
189
190val set_default_header : t -> string -> string -> t
191(** Add or update a default header. Returns a new session with the updated header.
192 The original session's connection pools are shared. *)
193
194val remove_default_header : t -> string -> t
195(** Remove a default header. Returns a new session without the header. *)
196
197val set_auth : t -> Auth.t -> t
198(** Set default authentication. Returns a new session with auth configured. *)
199
200val clear_auth : t -> t
201(** Clear authentication. Returns a new session without auth. *)
202
203val set_timeout : t -> Timeout.t -> t
204(** Set default timeout. Returns a new session with the timeout configured. *)
205
206val set_retry : t -> Retry.config -> t
207(** Set retry configuration. Returns a new session with retry configured. *)
208
209(** {2 Request Methods}
210
211 All request methods execute synchronously. To make concurrent requests,
212 you must explicitly use Eio.Fiber.both or Eio.Fiber.all.
213 The response will auto-close when the parent switch closes.
214
215 Example of concurrent requests using Fiber.both:
216 {[
217 let req = Requests.create ~sw env in
218
219 (* Use Fiber.both for two concurrent requests *)
220 let (r1, r2) = Eio.Fiber.both
221 (fun () -> Requests.get req "https://api1.example.com")
222 (fun () -> Requests.post req "https://api2.example.com" ~body)
223 in
224
225 (* Process responses *)
226 let body1 = Response.body r1 |> Eio.Flow.read_all in
227 let body2 = Response.body r2 |> Eio.Flow.read_all in
228 ]}
229
230 Example using Fiber.all for multiple requests:
231 {[
232 let req = Requests.create ~sw env in
233
234 (* Use Fiber.all for multiple concurrent requests *)
235 let urls = [
236 "https://api1.example.com";
237 "https://api2.example.com";
238 "https://api3.example.com";
239 ] in
240
241 let responses = ref [] in
242 Eio.Fiber.all [
243 (fun () -> responses := Requests.get req (List.nth urls 0) :: !responses);
244 (fun () -> responses := Requests.get req (List.nth urls 1) :: !responses);
245 (fun () -> responses := Requests.get req (List.nth urls 2) :: !responses);
246 ];
247
248 (* Process all responses *)
249 List.iter (fun r ->
250 let body = Response.body r |> Eio.Flow.read_all in
251 print_endline body
252 ) !responses
253 ]}
254
255 Example using Promise for concurrent requests with individual control:
256 {[
257 let req = Requests.create ~sw env in
258
259 (* Start requests in parallel using promises *)
260 let p1, r1 = Eio.Promise.create () in
261 let p2, r2 = Eio.Promise.create () in
262 let p3, r3 = Eio.Promise.create () in
263
264 Eio.Fiber.fork ~sw (fun () ->
265 Eio.Promise.resolve r1 (Requests.get req "https://api1.example.com")
266 );
267 Eio.Fiber.fork ~sw (fun () ->
268 Eio.Promise.resolve r2 (Requests.post req "https://api2.example.com" ~body)
269 );
270 Eio.Fiber.fork ~sw (fun () ->
271 Eio.Promise.resolve r3 (Requests.get req "https://api3.example.com")
272 );
273
274 (* Wait for all promises and process *)
275 let resp1 = Eio.Promise.await p1 in
276 let resp2 = Eio.Promise.await p2 in
277 let resp3 = Eio.Promise.await p3 in
278
279 (* Process responses *)
280 let body1 = Response.body resp1 |> Eio.Flow.read_all in
281 let body2 = Response.body resp2 |> Eio.Flow.read_all in
282 let body3 = Response.body resp3 |> Eio.Flow.read_all in
283 ]}
284*)
285
286val request :
287 t ->
288 ?headers:Headers.t ->
289 ?body:Body.t ->
290 ?auth:Auth.t ->
291 ?timeout:Timeout.t ->
292 ?follow_redirects:bool ->
293 ?max_redirects:int ->
294 method_:Method.t ->
295 string ->
296 Response.t
297(** Make a concurrent HTTP request *)
298
299val get :
300 t ->
301 ?headers:Headers.t ->
302 ?auth:Auth.t ->
303 ?timeout:Timeout.t ->
304 ?params:(string * string) list ->
305 string ->
306 Response.t
307(** Concurrent GET request *)
308
309val post :
310 t ->
311 ?headers:Headers.t ->
312 ?body:Body.t ->
313 ?auth:Auth.t ->
314 ?timeout:Timeout.t ->
315 string ->
316 Response.t
317(** Concurrent POST request *)
318
319val put :
320 t ->
321 ?headers:Headers.t ->
322 ?body:Body.t ->
323 ?auth:Auth.t ->
324 ?timeout:Timeout.t ->
325 string ->
326 Response.t
327(** Concurrent PUT request *)
328
329val patch :
330 t ->
331 ?headers:Headers.t ->
332 ?body:Body.t ->
333 ?auth:Auth.t ->
334 ?timeout:Timeout.t ->
335 string ->
336 Response.t
337(** Concurrent PATCH request *)
338
339val delete :
340 t ->
341 ?headers:Headers.t ->
342 ?auth:Auth.t ->
343 ?timeout:Timeout.t ->
344 string ->
345 Response.t
346(** Concurrent DELETE request *)
347
348val head :
349 t ->
350 ?headers:Headers.t ->
351 ?auth:Auth.t ->
352 ?timeout:Timeout.t ->
353 string ->
354 Response.t
355(** Concurrent HEAD request *)
356
357val options :
358 t ->
359 ?headers:Headers.t ->
360 ?auth:Auth.t ->
361 ?timeout:Timeout.t ->
362 string ->
363 Response.t
364(** Concurrent OPTIONS request *)
365
366(** {2 Cookie Management} *)
367
368val cookies : t -> Cookeio.jar
369(** Get the cookie jar for direct manipulation *)
370
371val clear_cookies : t -> unit
372(** Clear all cookies *)
373
374(** {1 Cmdliner Integration} *)
375
376module Cmd : sig
377 (** Cmdliner integration for Requests configuration.
378
379 This module provides command-line argument handling for configuring
380 HTTP requests, including XDG directory paths, timeouts, retries,
381 and other parameters. *)
382
383 (** Configuration from command line and environment *)
384 type config = {
385 xdg : Xdge.t * Xdge.Cmd.t; (** XDG paths and their sources *)
386 persist_cookies : bool; (** Whether to persist cookies *)
387 verify_tls : bool; (** Whether to verify TLS certificates *)
388 timeout : float option; (** Request timeout in seconds *)
389 max_retries : int; (** Maximum number of retries *)
390 retry_backoff : float; (** Retry backoff factor *)
391 follow_redirects : bool; (** Whether to follow redirects *)
392 max_redirects : int; (** Maximum number of redirects *)
393 user_agent : string option; (** User-Agent header *)
394 }
395
396 val create : config ->
397 < clock: [> float Eio.Time.clock_ty] Eio.Resource.t;
398 net: [> [> `Generic] Eio.Net.ty] Eio.Resource.t;
399 fs: Eio.Fs.dir_ty Eio.Path.t; .. > ->
400 Eio.Switch.t -> t
401 (** [create config env sw] creates a requests instance from command-line configuration *)
402
403 (** {2 Individual Terms} *)
404
405 val persist_cookies_term : string -> bool Cmdliner.Term.t
406 (** Term for [--persist-cookies] flag with app-specific env var *)
407
408 val verify_tls_term : string -> bool Cmdliner.Term.t
409 (** Term for [--no-verify-tls] flag with app-specific env var *)
410
411 val timeout_term : string -> float option Cmdliner.Term.t
412 (** Term for [--timeout SECONDS] option with app-specific env var *)
413
414 val retries_term : string -> int Cmdliner.Term.t
415 (** Term for [--max-retries N] option with app-specific env var *)
416
417 val retry_backoff_term : string -> float Cmdliner.Term.t
418 (** Term for [--retry-backoff FACTOR] option with app-specific env var *)
419
420 val follow_redirects_term : string -> bool Cmdliner.Term.t
421 (** Term for [--no-follow-redirects] flag with app-specific env var *)
422
423 val max_redirects_term : string -> int Cmdliner.Term.t
424 (** Term for [--max-redirects N] option with app-specific env var *)
425
426 val user_agent_term : string -> string option Cmdliner.Term.t
427 (** Term for [--user-agent STRING] option with app-specific env var *)
428
429 (** {2 Combined Terms} *)
430
431 val config_term : string -> Eio.Fs.dir_ty Eio.Path.t -> config Cmdliner.Term.t
432 (** [config_term app_name fs] creates a complete configuration term.
433
434 This combines all individual terms plus XDG configuration into
435 a single term that can be used to configure requests.
436
437 {b Generated Flags:}
438 - [--config-dir DIR]: Configuration directory
439 - [--data-dir DIR]: Data directory
440 - [--cache-dir DIR]: Cache directory
441 - [--persist-cookies]: Enable cookie persistence
442 - [--no-verify-tls]: Disable TLS verification
443 - [--timeout SECONDS]: Request timeout
444 - [--max-retries N]: Maximum retries
445 - [--retry-backoff FACTOR]: Retry backoff multiplier
446 - [--no-follow-redirects]: Disable redirect following
447 - [--max-redirects N]: Maximum redirects to follow
448 - [--user-agent STRING]: User-Agent header
449
450 {b Example:}
451 {[
452 let open Cmdliner in
453 let config_t = Requests.Cmd.config_term "myapp" env#fs in
454 let main config =
455 Eio.Switch.run @@ fun sw ->
456 let req = Requests.Cmd.create config env sw in
457 (* Use requests *)
458 in
459 let cmd = Cmd.v info Term.(const main $ config_t) in
460 Cmd.eval cmd
461 ]} *)
462
463 val requests_term : string ->
464 < clock: [> float Eio.Time.clock_ty] Eio.Resource.t;
465 net: [> [> `Generic] Eio.Net.ty] Eio.Resource.t;
466 fs: Eio.Fs.dir_ty Eio.Path.t; .. > ->
467 Eio.Switch.t -> t Cmdliner.Term.t
468 (** [requests_term app_name env sw] creates a term that directly produces a requests instance.
469
470 This is a convenience function that combines configuration parsing
471 with requests creation.
472
473 {b Example:}
474 {[
475 let open Cmdliner in
476 let main req =
477 (* Use requests directly *)
478 let resp = Requests.get req "https://example.com" in
479 (* ... *)
480 in
481 Eio.Switch.run @@ fun sw ->
482 let req_t = Requests.Cmd.requests_term "myapp" env sw in
483 let cmd = Cmd.v info Term.(const main $ req_t) in
484 Cmd.eval cmd
485 ]} *)
486
487 val minimal_term : string -> Eio.Fs.dir_ty Eio.Path.t -> (Xdge.t * bool) Cmdliner.Term.t
488 (** [minimal_term app_name fs] creates a minimal configuration term.
489
490 This only provides:
491 - [--cache-dir DIR]: Cache directory for responses
492 - [--persist-cookies]: Cookie persistence flag
493
494 Returns the XDG context and persist_cookies boolean.
495
496 {b Example:}
497 {[
498 let open Cmdliner in
499 let minimal_t = Requests.Cmd.minimal_term "myapp" env#fs in
500 let main (xdg, persist) =
501 Eio.Switch.run @@ fun sw ->
502 let req = Requests.create ~sw ~xdg ~persist_cookies:persist env in
503 (* Use requests *)
504 in
505 let cmd = Cmd.v info Term.(const main $ minimal_t) in
506 Cmd.eval cmd
507 ]} *)
508
509 (** {2 Documentation} *)
510
511 val env_docs : string -> string
512 (** [env_docs app_name] generates environment variable documentation.
513
514 Returns formatted documentation for all environment variables that
515 affect requests configuration, including XDG variables.
516
517 {b Included Variables:}
518 - [${APP_NAME}_CONFIG_DIR]: Configuration directory
519 - [${APP_NAME}_DATA_DIR]: Data directory
520 - [${APP_NAME}_CACHE_DIR]: Cache directory
521 - [${APP_NAME}_STATE_DIR]: State directory
522 - [XDG_CONFIG_HOME], [XDG_DATA_HOME], [XDG_CACHE_HOME], [XDG_STATE_HOME]
523 - [HTTP_PROXY], [HTTPS_PROXY], [NO_PROXY] (when proxy support is added)
524
525 {b Example:}
526 {[
527 let env_info = Cmdliner.Cmd.Env.info
528 ~docs:Cmdliner.Manpage.s_environment
529 ~doc:(Requests.Cmd.env_docs "myapp")
530 ()
531 ]} *)
532
533 val pp_config : Format.formatter -> config -> unit
534 (** Pretty print configuration for debugging *)
535end
536
537(** Retry policies and backoff strategies *)
538module Retry = Retry
539
540(** {1 One-Shot API}
541
542 The One module provides direct control over HTTP requests without
543 session state. Use this for stateless operations or when you need
544 fine-grained control.
545*)
546
547(** One-shot HTTP client for stateless requests *)
548module One = One
549
550(** {1 Core Types}
551
552 These modules define the fundamental types used throughout the library.
553*)
554
555(** HTTP response handling *)
556module Response = Response
557
558(** Request body construction and encoding *)
559module Body = Body
560
561(** HTTP headers manipulation *)
562module Headers = Headers
563
564(** Authentication schemes (Basic, Bearer, OAuth, etc.) *)
565module Auth = Auth
566
567(** Error types and exception handling *)
568module Error = Error
569
570(** {1 Supporting Types} *)
571
572(** HTTP status codes and reason phrases *)
573module Status = Status
574
575(** HTTP request methods (GET, POST, etc.) *)
576module Method = Method
577
578(** MIME types for content negotiation *)
579module Mime = Mime
580
581(** Timeout configuration for requests *)
582module Timeout = Timeout
583
584(** {2 Logging} *)
585
586(** Log source for the requests library.
587 Use [Logs.Src.set_level src] to control logging verbosity.
588 Example: [Logs.Src.set_level Requests.src (Some Logs.Debug)] *)
589val src : Logs.Src.t