commit 68dea75a5670960f8f19e06af9e0635a6c161163 · anil.recoil.org/slop

+1 -80

stack/requests/CLAUDE.md

···

       5
       5
        
       successfully abstracts TLS configuration and CA certificate complexity while

     

       6
       6
        
       providing a clean Eio-style API.

     

       7
       7
        
       

     

       8
       8
       -
       ### Key Implementation Details

     

       8
       8
       +
       As a matter of good hygeine, dont import Requests at the top of the test files but use it when needed or do a local open

     

       9
       9
        
       

     

       10
       10
       -
       1. **Module Organization**:

     

       11
       11
       -
          - Auth module must be defined before Config in both .mli and .ml files (Config references Auth.t)

     

       12
       12
       -
          - ConnectionPool uses a GADT to hide the network type parameter

     

       13
       13
       -
          - Session and PoolManager use internal type aliases to avoid naming conflicts

     

       14
       14
       -
       

     

       15
       15
       -
       2. **Buffer Size Limitations**:

     

       16
       16
       -
          - Cannot use `Int.max_int` for Eio buffer sizes (causes Bigarray creation errors)

     

       17
       17
       -
          - Use reasonable sizes like `16 * 1024 * 1024` (16MB) for buffer creation

     

       18
       18
       -
          - When converting Cohttp_eio body to Buf_read, always specify max_size

     

       19
       19
       -
       

     

       20
       20
       -
       3. **Type Constraints**:

     

       21
       21
       -
          - Network types use polymorphic variants: `[> \`Generic] Net.t`

     

       22
       22
       -
          - Cannot use wildcard `_` in type declarations

     

       23
       23
       -
          - Use GADTs or existential types when hiding type parameters in module signatures

     

       24
       24
       -
       

     

       25
       25
       -
       4. **Build Commands**:

     

       26
       26
       -
          - Development build: `opam exec -- dune build @check`

     

       27
       27
       -
          - Release build (ignores warnings): `opam exec -- dune build @check --profile=release`

     

       28
       28
       -
          - Documentation: `opam exec -- dune build @doc`

     

       29
       29
       -
       

     

       30
       30
       -
       ### Implemented Features

     

       31
       31
       -
       

     

       32
       32
       -
       #### Core (urllib3-inspired):

     

       33
       33
       -
       - ✅ Per-host connection pooling with configurable limits

     

       34
       34
       -
       - ✅ Automatic retries with exponential backoff (default 10 retries)

     

       35
       35
       -
       - ✅ Advanced timeout handling (connect, read, total)

     

       36
       36
       -
       - ✅ HTTP caching with Cache-Control support

     

       37
       37
       -
       - ✅ Progress tracking via OCaml Logs

     

       38
       38
       -
       - ✅ Cookie handling in sessions

     

       39
       39
       -
       

     

       40
       40
       -
       #### Authentication (requests-inspired):

     

       41
       41
       -
       - ✅ Basic authentication

     

       42
       42
       -
       - ✅ Digest authentication (RFC 2617)

     

       43
       43
       -
       - ✅ Bearer token (OAuth 2.0)

     

       44
       44
       -
       - ✅ OAuth 1.0a support

     

       45
       45
       -
       - ✅ OAuth 2.0 support

     

       46
       46
       -
       - ✅ Custom authentication handlers

     

       47
       47
       -
       

     

       48
       48
       -
       #### Streaming:

     

       49
       49
       -
       - ✅ Upload streaming with Eio Flow sources

     

       50
       50
       -
       - ✅ Download streaming with Eio Flow sinks

     

       51
       51
       -
       - ✅ Response iteration (chunked/line-based)

     

       52
       52
       -
       

     

       53
       53
       -
       ### TODO - Future Work

     

       54
       54
       -
       

     

       55
       55
       -
       1. **High Priority** (Core functionality):

     

       56
       56
       -
          - [ ] Fix remaining build issues in lib/ruminant.ml (Github.Api.Https module)

     

       57
       57
       -
          - [ ] Fix bin/main.ml (data.issues field issue)

     

       58
       58
       -
          - [ ] Complete OAuth implementation (currently stubbed)

     

       59
       59
       -
          - [ ] Implement actual HTTP caching storage (memory/file backends partially done)

     

       60
       60
       -
          - [ ] Add proper connection pool lifecycle management

     

       61
       61
       -
       

     

       62
       62
       -
       2. **Medium Priority** (Enhanced features from DESIGN.md):

     

       63
       63
       -
          - [ ] Implement hooks system for request/response middleware

     

       64
       64
       -
          - [ ] Add SOCKS proxy support (design in DESIGN.md)

     

       65
       65
       -
          - [ ] Implement character encoding detection and conversion

     

       66
       66
       -
          - [ ] Add compression support (gzip, deflate, brotli when available)

     

       67
       67
       -
          - [ ] Implement proper multipart form data encoding

     

       68
       68
       -
       

     

       69
       69
       -
       3. **Low Priority** (Advanced features):

     

       70
       70
       -
          - [ ] WebSocket support

     

       71
       71
       -
          - [ ] HTTP/2 support (when cohttp-eio adds it)

     

       72
       72
       -
          - [ ] HTTP/3 support (future)

     

       73
       73
       -
          - [ ] Connection persistence and keep-alive optimization

     

       74
       74
       -
          - [ ] Advanced certificate pinning

     

       75
       75
       -
       

     

       76
       76
       -
       4. **Testing & Documentation**:

     

       77
       77
       -
          - [ ] Add comprehensive test suite for requests library

     

       78
       78
       -
          - [ ] Create examples directory with common use cases

     

       79
       79
       -
          - [ ] Write API documentation and usage guide

     

       80
       80
       -
          - [ ] Add benchmarks comparing with Python requests

     

       81
       81
       -
       

     

       82
       82
       -
       ### Known Issues

     

       83
       83
       -
       

     

       84
       84
       -
       2. **Type Shadowing**: Module-level type `t` can cause shadowing issues in nested modules. Use unique type names or GADTs to avoid conflicts.

     

       85
       85
       -
       

     

       86
       86
       -
       3. **Module Dependencies**: Auth module must come before Config in the interface due to Config's create function taking an Auth.t parameter.

     

       87
       87
       -
       

     

       88
       88
       -
       4. **Warnings**: Many unused variables in stub implementations need cleanup once features are fully implemented.

+1 -1

stack/requests/lib/dune

···

       1
       1
        
       (library

     

       2
       2
        
        (public_name requests)

     

       3
       3
        
        (name requests)

     

       4
       4
       -
        (modules :standard \ digest_auth requests_cache requests_types)

     

       4
       4
       +
        (modules :standard \ digest_auth)

     

       5
       5
        
        (libraries

     

       6
       6
        
         eio

     

       7
       7
        
         eio.unix

stack/requests/lib/requests.ml

···

       14
       14
        
       module Status = Status

     

       15
       15
        
       module Error = Error

     

       16
       16
        
       module Retry = Retry

     

       17
       17
       +
       module Cache = Requests_cache

     

       17
       18
        
       

     

       18
       19
        
       (* Main API - Session functionality with concurrent fiber spawning *)

     

       19
       20

stack/requests/lib/requests.mli

···

       542
       542
        
       (** Timeout configuration for requests *)

     

       543
       543
        
       module Timeout = Timeout

     

       544
       544
        
       

     

       545
       545
       +
       (** HTTP caching with cache control and range request support *)

     

       546
       546
       +
       module Cache = Requests_cache

     

       547
       547
       +
       

     

       545
       548
        
       (** {2 Logging} *)

     

       546
       549
        
       

     

       547
       550
        
       (** Log source for the requests library.

+3 -3

stack/requests/lib/requests_cache.ml

···

       282
       282
        
               (* Store metadata *)

     

       283
       283
        
               let metadata_key = key ^ ".meta" in

     

       284
       284
        
               let metadata = serialize_metadata ~status ~headers in

     

       285
       285
       -
               let metadata_source = Flow.string_source metadata in

     

       285
       285
       +
               let metadata_source = Eio.Flow.string_source metadata in

     

       286
       286
        
               Cacheio.put cache ~key:metadata_key ~source:metadata_source ~ttl ();

     

       287
       287
        
       

     

       288
       288
        
               (* Store body directly from source *)

     
···

       354
       354
        
                  let rec read_chunks () =

     

       355
       355
        
                    let chunk = Cstruct.create 8192 in

     

       356
       356
        
                    try

     

       357
       357
       -
                      let n = Flow.single_read source chunk in

     

       357
       357
       +
                      let n = Eio.Flow.single_read source chunk in

     

       358
       358
        
                      if n > 0 then begin

     

       359
       359
        
                        on_chunk (Cstruct.to_string ~off:0 ~len:n chunk);

     

       360
       360
        
                        read_chunks ()

     
···

       369
       369
        
                    let rec read_chunk () =

     

       370
       370
        
                      let chunk = Cstruct.create 8192 in

     

       371
       371
        
                      try

     

       372
       372
       -
                        let n = Flow.single_read source chunk in

     

       372
       372
       +
                        let n = Eio.Flow.single_read source chunk in

     

       373
       373
        
                        if n > 0 then begin

     

       374
       374
        
                          on_chunk (Cstruct.to_string ~off:0 ~len:n chunk);

     

       375
       375
        
                          read_chunk ()