My agentic slop goes here. Not intended for anyone else!
anil.recoil.org
1(** Requests - A modern HTTP client library for OCaml
2
3 Requests is an HTTP client library for OCaml inspired by Python's requests
4 and urllib3 libraries. It provides a simple, intuitive API for making HTTP
5 requests while handling complexities like TLS configuration, connection
6 pooling, retries, and cookie management.
7
8 {2 High-Level API}
9
10 The Requests library offers two main ways to make HTTP requests:
11
12 {b 1. Main API} (Recommended for most use cases)
13
14 The main API maintains state across requests, handles cookies automatically,
15 spawns requests in concurrent fibers, and provides a simple interface:
16
17 {[
18 open Eio_main
19
20 let () = run @@ fun env ->
21 Switch.run @@ fun sw ->
22
23 (* Create a requests instance *)
24 let req = Requests.create ~sw env in
25
26 (* Configure authentication once *)
27 Requests.set_auth req (Requests.Auth.bearer "your-token");
28
29 (* Make concurrent requests using Fiber.both *)
30 let (user, repos) = Eio.Fiber.both
31 (fun () -> Requests.get req "https://api.github.com/user")
32 (fun () -> Requests.get req "https://api.github.com/user/repos") in
33
34 (* Process responses *)
35 let user_data = Response.body user |> Eio.Flow.read_all in
36 let repos_data = Response.body repos |> Eio.Flow.read_all in
37
38 (* No cleanup needed - responses auto-close with the switch *)
39 ]}
40
41 {b 2. One-shot requests} (For stateless operations)
42
43 The One module provides lower-level control for stateless,
44 one-off requests without session state:
45
46 {[
47 (* Create a one-shot client *)
48 let client = Requests.One.create ~clock:env#clock ~net:env#net () in
49
50 (* Make a simple GET request *)
51 let response = Requests.One.get ~sw ~client "https://api.github.com" in
52 Printf.printf "Status: %d\n" (Requests.Response.status_code response);
53
54 (* POST with custom headers and body *)
55 let response = Requests.One.post ~sw ~client
56 ~headers:(Requests.Headers.empty
57 |> Requests.Headers.content_type Requests.Mime.json
58 |> Requests.Headers.set "X-API-Key" "secret")
59 ~body:(Requests.Body.json {|{"name": "Alice"}|})
60 "https://api.example.com/users"
61
62 (* No cleanup needed - responses auto-close with the switch *)
63 ]}
64
65 {2 Features}
66
67 - {b Simple API}: Intuitive functions for GET, POST, PUT, DELETE, etc.
68 - {b Authentication}: Built-in support for Basic, Bearer, Digest, and OAuth
69 - {b Streaming}: Upload and download large files efficiently
70 - {b Retries}: Automatic retry with exponential backoff
71 - {b Timeouts}: Configurable connection and read timeouts
72 - {b Cookie Management}: Automatic cookie handling with persistence
73 - {b TLS/SSL}: Secure connections with certificate verification
74 - {b Error Handling}: Comprehensive error types and recovery
75
76 {2 Common Use Cases}
77
78 {b Working with JSON APIs:}
79 {[
80 let response = Requests.post req "https://api.example.com/data"
81 ~body:(Requests.Body.json {|{"key": "value"}|}) in
82 let body_text =
83 Requests.Response.body response
84 |> Eio.Flow.read_all in
85 print_endline body_text
86 (* Response auto-closes with switch *)
87 ]}
88
89 {b File uploads:}
90 {[
91 let body = Requests.Body.multipart [
92 { name = "file"; filename = Some "document.pdf";
93 content_type = Requests.Mime.pdf;
94 content = `File (Eio.Path.(fs / "document.pdf")) };
95 { name = "description"; filename = None;
96 content_type = Requests.Mime.text_plain;
97 content = `String "Important document" }
98 ] in
99 let response = Requests.post req "https://example.com/upload"
100 ~body
101 (* Response auto-closes with switch *)
102 ]}
103
104 {b Streaming downloads:}
105 {[
106 Requests.One.download ~sw ~client
107 "https://example.com/large-file.zip"
108 ~sink:(Eio.Path.(fs / "download.zip" |> sink))
109 ]}
110
111 {2 Choosing Between Main API and One}
112
113 Use the {b main API (Requests.t)} when you need:
114 - Cookie persistence across requests
115 - Automatic retry handling
116 - Shared authentication across requests
117 - Request/response history tracking
118 - Configuration persistence to disk
119
120 Use {b One} when you need:
121 - One-off stateless requests
122 - Fine-grained control over connections
123 - Minimal overhead
124 - Custom connection pooling
125 - Direct streaming without cookies
126*)
127
128(** {1 Main API}
129
130 The main Requests API provides stateful HTTP clients with automatic cookie
131 management and persistent configuration. Requests execute synchronously by default.
132 Use Eio.Fiber.both or Eio.Fiber.all for concurrent execution.
133*)
134
135type ('clock, 'net) t
136(** A stateful HTTP client that maintains cookies, auth, configuration, and
137 connection pools across requests. *)
138
139(** {2 Creation and Configuration} *)
140
141val create :
142 sw:Eio.Switch.t ->
143 ?http_pool:('clock Eio.Time.clock, 'net Eio.Net.t) Conpool.t ->
144 ?https_pool:('clock Eio.Time.clock, 'net Eio.Net.t) Conpool.t ->
145 ?cookie_jar:Cookeio.jar ->
146 ?default_headers:Headers.t ->
147 ?auth:Auth.t ->
148 ?timeout:Timeout.t ->
149 ?follow_redirects:bool ->
150 ?max_redirects:int ->
151 ?verify_tls:bool ->
152 ?tls_config:Tls.Config.client ->
153 ?max_connections_per_host:int ->
154 ?connection_idle_timeout:float ->
155 ?connection_lifetime:float ->
156 ?retry:Retry.config ->
157 ?persist_cookies:bool ->
158 ?xdg:Xdge.t ->
159 < clock: 'clock Eio.Resource.t; net: 'net Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; .. > ->
160 ('clock Eio.Resource.t, 'net Eio.Resource.t) t
161(** Create a new requests instance with persistent state and connection pooling.
162 All resources are bound to the provided switch and will be cleaned up automatically.
163
164 @param sw Switch for resource management
165 @param http_pool Optional pre-configured HTTP connection pool (creates new if not provided)
166 @param https_pool Optional pre-configured HTTPS connection pool (creates new if not provided)
167 @param cookie_jar Cookie storage (default: empty in-memory jar)
168 @param default_headers Headers included in every request
169 @param auth Default authentication
170 @param timeout Default timeout configuration
171 @param follow_redirects Whether to follow HTTP redirects (default: true)
172 @param max_redirects Maximum redirects to follow (default: 10)
173 @param verify_tls Whether to verify TLS certificates (default: true)
174 @param tls_config Custom TLS configuration for HTTPS pool (default: system CA certs)
175 @param max_connections_per_host Maximum pooled connections per host:port (default: 10)
176 @param connection_idle_timeout Max idle time before closing pooled connection (default: 60s)
177 @param connection_lifetime Max lifetime of any pooled connection (default: 300s)
178 @param retry Retry configuration for failed requests
179 @param persist_cookies Whether to persist cookies to disk (default: false)
180 @param xdg XDG directory context for cookies (required if persist_cookies=true)
181
182 {b Note:} HTTP caching has been disabled for simplicity. See CACHEIO.md for integration notes
183 if you need to restore caching functionality in the future.
184*)
185
186(** {2 Configuration Management} *)
187
188val set_default_header : ('clock, 'net) t -> string -> string -> ('clock, 'net) t
189(** Add or update a default header. Returns a new session with the updated header.
190 The original session's connection pools are shared. *)
191
192val remove_default_header : ('clock, 'net) t -> string -> ('clock, 'net) t
193(** Remove a default header. Returns a new session without the header. *)
194
195val set_auth : ('clock, 'net) t -> Auth.t -> ('clock, 'net) t
196(** Set default authentication. Returns a new session with auth configured. *)
197
198val clear_auth : ('clock, 'net) t -> ('clock, 'net) t
199(** Clear authentication. Returns a new session without auth. *)
200
201val set_timeout : ('clock, 'net) t -> Timeout.t -> ('clock, 'net) t
202(** Set default timeout. Returns a new session with the timeout configured. *)
203
204val set_retry : ('clock, 'net) t -> Retry.config -> ('clock, 'net) t
205(** Set retry configuration. Returns a new session with retry configured. *)
206
207(** {2 Request Methods}
208
209 All request methods execute synchronously. To make concurrent requests,
210 you must explicitly use Eio.Fiber.both or Eio.Fiber.all.
211 The response will auto-close when the parent switch closes.
212
213 Example of concurrent requests using Fiber.both:
214 {[
215 let req = Requests.create ~sw env in
216
217 (* Use Fiber.both for two concurrent requests *)
218 let (r1, r2) = Eio.Fiber.both
219 (fun () -> Requests.get req "https://api1.example.com")
220 (fun () -> Requests.post req "https://api2.example.com" ~body)
221 in
222
223 (* Process responses *)
224 let body1 = Response.body r1 |> Eio.Flow.read_all in
225 let body2 = Response.body r2 |> Eio.Flow.read_all in
226 ]}
227
228 Example using Fiber.all for multiple requests:
229 {[
230 let req = Requests.create ~sw env in
231
232 (* Use Fiber.all for multiple concurrent requests *)
233 let urls = [
234 "https://api1.example.com";
235 "https://api2.example.com";
236 "https://api3.example.com";
237 ] in
238
239 let responses = ref [] in
240 Eio.Fiber.all [
241 (fun () -> responses := Requests.get req (List.nth urls 0) :: !responses);
242 (fun () -> responses := Requests.get req (List.nth urls 1) :: !responses);
243 (fun () -> responses := Requests.get req (List.nth urls 2) :: !responses);
244 ];
245
246 (* Process all responses *)
247 List.iter (fun r ->
248 let body = Response.body r |> Eio.Flow.read_all in
249 print_endline body
250 ) !responses
251 ]}
252
253 Example using Promise for concurrent requests with individual control:
254 {[
255 let req = Requests.create ~sw env in
256
257 (* Start requests in parallel using promises *)
258 let p1, r1 = Eio.Promise.create () in
259 let p2, r2 = Eio.Promise.create () in
260 let p3, r3 = Eio.Promise.create () in
261
262 Eio.Fiber.fork ~sw (fun () ->
263 Eio.Promise.resolve r1 (Requests.get req "https://api1.example.com")
264 );
265 Eio.Fiber.fork ~sw (fun () ->
266 Eio.Promise.resolve r2 (Requests.post req "https://api2.example.com" ~body)
267 );
268 Eio.Fiber.fork ~sw (fun () ->
269 Eio.Promise.resolve r3 (Requests.get req "https://api3.example.com")
270 );
271
272 (* Wait for all promises and process *)
273 let resp1 = Eio.Promise.await p1 in
274 let resp2 = Eio.Promise.await p2 in
275 let resp3 = Eio.Promise.await p3 in
276
277 (* Process responses *)
278 let body1 = Response.body resp1 |> Eio.Flow.read_all in
279 let body2 = Response.body resp2 |> Eio.Flow.read_all in
280 let body3 = Response.body resp3 |> Eio.Flow.read_all in
281 ]}
282*)
283
284val request :
285 (_ Eio.Time.clock, _ Eio.Net.t) t ->
286 ?headers:Headers.t ->
287 ?body:Body.t ->
288 ?auth:Auth.t ->
289 ?timeout:Timeout.t ->
290 ?follow_redirects:bool ->
291 ?max_redirects:int ->
292 method_:Method.t ->
293 string ->
294 Response.t
295(** Make a concurrent HTTP request *)
296
297val get :
298 (_ Eio.Time.clock, _ Eio.Net.t) t ->
299 ?headers:Headers.t ->
300 ?auth:Auth.t ->
301 ?timeout:Timeout.t ->
302 ?params:(string * string) list ->
303 string ->
304 Response.t
305(** Concurrent GET request *)
306
307val post :
308 (_ Eio.Time.clock, _ Eio.Net.t) t ->
309 ?headers:Headers.t ->
310 ?body:Body.t ->
311 ?auth:Auth.t ->
312 ?timeout:Timeout.t ->
313 string ->
314 Response.t
315(** Concurrent POST request *)
316
317val put :
318 (_ Eio.Time.clock, _ Eio.Net.t) t ->
319 ?headers:Headers.t ->
320 ?body:Body.t ->
321 ?auth:Auth.t ->
322 ?timeout:Timeout.t ->
323 string ->
324 Response.t
325(** Concurrent PUT request *)
326
327val patch :
328 (_ Eio.Time.clock, _ Eio.Net.t) t ->
329 ?headers:Headers.t ->
330 ?body:Body.t ->
331 ?auth:Auth.t ->
332 ?timeout:Timeout.t ->
333 string ->
334 Response.t
335(** Concurrent PATCH request *)
336
337val delete :
338 (_ Eio.Time.clock, _ Eio.Net.t) t ->
339 ?headers:Headers.t ->
340 ?auth:Auth.t ->
341 ?timeout:Timeout.t ->
342 string ->
343 Response.t
344(** Concurrent DELETE request *)
345
346val head :
347 (_ Eio.Time.clock, _ Eio.Net.t) t ->
348 ?headers:Headers.t ->
349 ?auth:Auth.t ->
350 ?timeout:Timeout.t ->
351 string ->
352 Response.t
353(** Concurrent HEAD request *)
354
355val options :
356 (_ Eio.Time.clock, _ Eio.Net.t) t ->
357 ?headers:Headers.t ->
358 ?auth:Auth.t ->
359 ?timeout:Timeout.t ->
360 string ->
361 Response.t
362(** Concurrent OPTIONS request *)
363
364(** {2 Cookie Management} *)
365
366val cookies : ('clock, 'net) t -> Cookeio.jar
367(** Get the cookie jar for direct manipulation *)
368
369val clear_cookies : ('clock, 'net) t -> unit
370(** Clear all cookies *)
371
372(** {1 Cmdliner Integration} *)
373
374module Cmd : sig
375 (** Cmdliner integration for Requests configuration.
376
377 This module provides command-line argument handling for configuring
378 HTTP requests, including XDG directory paths, timeouts, retries,
379 and other parameters. *)
380
381 (** Configuration from command line and environment *)
382 type config = {
383 xdg : Xdge.t * Xdge.Cmd.t; (** XDG paths and their sources *)
384 persist_cookies : bool; (** Whether to persist cookies *)
385 verify_tls : bool; (** Whether to verify TLS certificates *)
386 timeout : float option; (** Request timeout in seconds *)
387 max_retries : int; (** Maximum number of retries *)
388 retry_backoff : float; (** Retry backoff factor *)
389 follow_redirects : bool; (** Whether to follow redirects *)
390 max_redirects : int; (** Maximum number of redirects *)
391 user_agent : string option; (** User-Agent header *)
392 }
393
394 val create : config -> < clock: ([> float Eio.Time.clock_ty ] as 'clock) Eio.Resource.t; net: ([> [>`Generic] Eio.Net.ty ] as 'net) Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; .. > -> Eio.Switch.t -> ('clock Eio.Resource.t, 'net Eio.Resource.t) t
395 (** [create config env sw] creates a requests instance from command-line configuration *)
396
397 (** {2 Individual Terms} *)
398
399 val persist_cookies_term : string -> bool Cmdliner.Term.t
400 (** Term for [--persist-cookies] flag with app-specific env var *)
401
402 val verify_tls_term : string -> bool Cmdliner.Term.t
403 (** Term for [--no-verify-tls] flag with app-specific env var *)
404
405 val timeout_term : string -> float option Cmdliner.Term.t
406 (** Term for [--timeout SECONDS] option with app-specific env var *)
407
408 val retries_term : string -> int Cmdliner.Term.t
409 (** Term for [--max-retries N] option with app-specific env var *)
410
411 val retry_backoff_term : string -> float Cmdliner.Term.t
412 (** Term for [--retry-backoff FACTOR] option with app-specific env var *)
413
414 val follow_redirects_term : string -> bool Cmdliner.Term.t
415 (** Term for [--no-follow-redirects] flag with app-specific env var *)
416
417 val max_redirects_term : string -> int Cmdliner.Term.t
418 (** Term for [--max-redirects N] option with app-specific env var *)
419
420 val user_agent_term : string -> string option Cmdliner.Term.t
421 (** Term for [--user-agent STRING] option with app-specific env var *)
422
423 (** {2 Combined Terms} *)
424
425 val config_term : string -> Eio.Fs.dir_ty Eio.Path.t -> config Cmdliner.Term.t
426 (** [config_term app_name fs] creates a complete configuration term.
427
428 This combines all individual terms plus XDG configuration into
429 a single term that can be used to configure requests.
430
431 {b Generated Flags:}
432 - [--config-dir DIR]: Configuration directory
433 - [--data-dir DIR]: Data directory
434 - [--cache-dir DIR]: Cache directory
435 - [--persist-cookies]: Enable cookie persistence
436 - [--no-verify-tls]: Disable TLS verification
437 - [--timeout SECONDS]: Request timeout
438 - [--max-retries N]: Maximum retries
439 - [--retry-backoff FACTOR]: Retry backoff multiplier
440 - [--no-follow-redirects]: Disable redirect following
441 - [--max-redirects N]: Maximum redirects to follow
442 - [--user-agent STRING]: User-Agent header
443
444 {b Example:}
445 {[
446 let open Cmdliner in
447 let config_t = Requests.Cmd.config_term "myapp" env#fs in
448 let main config =
449 Eio.Switch.run @@ fun sw ->
450 let req = Requests.Cmd.create config env sw in
451 (* Use requests *)
452 in
453 let cmd = Cmd.v info Term.(const main $ config_t) in
454 Cmd.eval cmd
455 ]} *)
456
457 val requests_term : string -> < clock: ([> float Eio.Time.clock_ty ] as 'clock) Eio.Resource.t; net: ([> [>`Generic] Eio.Net.ty ] as 'net) Eio.Resource.t; fs: Eio.Fs.dir_ty Eio.Path.t; .. > -> Eio.Switch.t -> ('clock Eio.Resource.t, 'net Eio.Resource.t) t Cmdliner.Term.t
458 (** [requests_term app_name env sw] creates a term that directly produces a requests instance.
459
460 This is a convenience function that combines configuration parsing
461 with requests creation.
462
463 {b Example:}
464 {[
465 let open Cmdliner in
466 let main req =
467 (* Use requests directly *)
468 let resp = Requests.get req "https://example.com" in
469 (* ... *)
470 in
471 Eio.Switch.run @@ fun sw ->
472 let req_t = Requests.Cmd.requests_term "myapp" env sw in
473 let cmd = Cmd.v info Term.(const main $ req_t) in
474 Cmd.eval cmd
475 ]} *)
476
477 val minimal_term : string -> Eio.Fs.dir_ty Eio.Path.t -> (Xdge.t * bool) Cmdliner.Term.t
478 (** [minimal_term app_name fs] creates a minimal configuration term.
479
480 This only provides:
481 - [--cache-dir DIR]: Cache directory for responses
482 - [--persist-cookies]: Cookie persistence flag
483
484 Returns the XDG context and persist_cookies boolean.
485
486 {b Example:}
487 {[
488 let open Cmdliner in
489 let minimal_t = Requests.Cmd.minimal_term "myapp" env#fs in
490 let main (xdg, persist) =
491 Eio.Switch.run @@ fun sw ->
492 let req = Requests.create ~sw ~xdg ~persist_cookies:persist env in
493 (* Use requests *)
494 in
495 let cmd = Cmd.v info Term.(const main $ minimal_t) in
496 Cmd.eval cmd
497 ]} *)
498
499 (** {2 Documentation} *)
500
501 val env_docs : string -> string
502 (** [env_docs app_name] generates environment variable documentation.
503
504 Returns formatted documentation for all environment variables that
505 affect requests configuration, including XDG variables.
506
507 {b Included Variables:}
508 - [${APP_NAME}_CONFIG_DIR]: Configuration directory
509 - [${APP_NAME}_DATA_DIR]: Data directory
510 - [${APP_NAME}_CACHE_DIR]: Cache directory
511 - [${APP_NAME}_STATE_DIR]: State directory
512 - [XDG_CONFIG_HOME], [XDG_DATA_HOME], [XDG_CACHE_HOME], [XDG_STATE_HOME]
513 - [HTTP_PROXY], [HTTPS_PROXY], [NO_PROXY] (when proxy support is added)
514
515 {b Example:}
516 {[
517 let env_info = Cmdliner.Cmd.Env.info
518 ~docs:Cmdliner.Manpage.s_environment
519 ~doc:(Requests.Cmd.env_docs "myapp")
520 ()
521 ]} *)
522
523 val pp_config : Format.formatter -> config -> unit
524 (** Pretty print configuration for debugging *)
525end
526
527(** Retry policies and backoff strategies *)
528module Retry = Retry
529
530(** {1 One-Shot API}
531
532 The One module provides direct control over HTTP requests without
533 session state. Use this for stateless operations or when you need
534 fine-grained control.
535*)
536
537(** One-shot HTTP client for stateless requests *)
538module One = One
539
540(** {1 Core Types}
541
542 These modules define the fundamental types used throughout the library.
543*)
544
545(** HTTP response handling *)
546module Response = Response
547
548(** Request body construction and encoding *)
549module Body = Body
550
551(** HTTP headers manipulation *)
552module Headers = Headers
553
554(** Authentication schemes (Basic, Bearer, OAuth, etc.) *)
555module Auth = Auth
556
557(** Error types and exception handling *)
558module Error = Error
559
560(** {1 Supporting Types} *)
561
562(** HTTP status codes and reason phrases *)
563module Status = Status
564
565(** HTTP request methods (GET, POST, etc.) *)
566module Method = Method
567
568(** MIME types for content negotiation *)
569module Mime = Mime
570
571(** Timeout configuration for requests *)
572module Timeout = Timeout
573
574(** {2 Logging} *)
575
576(** Log source for the requests library.
577 Use [Logs.Src.set_level src] to control logging verbosity.
578 Example: [Logs.Src.set_level Requests.src (Some Logs.Debug)] *)
579val src : Logs.Src.t