commit 5a4ba6b0c2593bdaaa5f33003faa0218ed22f858 · anil.recoil.org/slop

+118

dancer/CLAUDE.md

···

       371
       371
        
       

     

       372
       372
        
       **Phase 4: Oversight** (Week 4)

     

       373
       373
        
       10. `dancer-ui` - Dashboard and monitoring

     

       374
       374
       +
       

     

       375
       375
       +
       ## dancer-logs Implementation Details

     

       376
       376
       +
       

     

       377
       377
       +
       ### Design Decisions

     

       378
       378
       +
       

     

       379
       379
       +
       1. **Comprehensive Schema**: The SQLite schema captures everything that might be useful for Claude's analysis:

     

       380
       380
       +
          - Core fields (timestamp, level, source, message)

     

       381
       381
       +
          - Error classification (type, code, hash, stack trace)

     

       382
       382
       +
          - Execution context (PIDs, threads, fibers, domains)

     

       383
       383
       +
          - Request/session tracking (IDs for correlation)

     

       384
       384
       +
          - Source location (file, line, function, module)

     

       385
       385
       +
          - Performance metrics (duration, memory, CPU)

     

       386
       386
       +
          - Network context (IPs, ports, methods, status codes)

     

       387
       387
       +
          - User context (user ID, tenant ID)

     

       388
       388
       +
          - System context (OS, versions, environment, containers)

     

       389
       389
       +
          - Flexible metadata (JSON fields for tags, labels, custom data)

     

       390
       390
       +
       

     

       391
       391
       +
       2. **Full-Text Search**: Using SQLite's FTS5 with porter stemming and unicode support for comprehensive search across all text fields.

     

       392
       392
       +
       

     

       393
       393
       +
       3. **Pattern Detection**: 

     

       394
       394
       +
          - Normalizes messages by replacing numbers with "N", hex with "0xHEX", UUIDs with "UUID"

     

       395
       395
       +
          - Tracks patterns with occurrence counts, severity scores, and consultation history

     

       396
       396
       +
          - Time-series bucketing for trend analysis

     

       397
       397
       +
       

     

       398
       398
       +
       4. **Logs Reporter Integration**: 

     

       399
       399
       +
          - Implements OCaml Logs reporter interface

     

       400
       400
       +
          - Can chain with existing reporters

     

       401
       401
       +
          - Thread-safe with Eio mutexes

     

       402
       402
       +
          - Captures Eio fiber IDs and domain IDs

     

       403
       403
       +
       

     

       404
       404
       +
       5. **CLI Tool**: Colorful terminal interface using Fmt with:

     

       405
       405
       +
          - `list` - Browse logs with filters

     

       406
       406
       +
          - `search` - Full-text search

     

       407
       407
       +
          - `patterns` - View detected patterns

     

       408
       408
       +
          - `errors` - Recent error summary

     

       409
       409
       +
          - `stats` - Database statistics

     

       410
       410
       +
          - `export` - Export for analysis (JSON, CSV, Claude format)

     

       411
       411
       +
       

     

       412
       412
       +
       ### Key Learnings

     

       413
       413
       +
       

     

       414
       414
       +
       1. **No Truncation**: Keep all data intact for Claude to analyze. Storage is cheap, context is valuable.

     

       415
       415
       +
       

     

       416
       416
       +
       2. **Structured Everything**: The more structure we capture, the better Claude can understand patterns and correlations.

     

       417
       417
       +
       

     

       418
       418
       +
       3. **Synchronous is Fine**: For now, synchronous SQLite operations are acceptable. Performance optimization can come later.

     

       419
       419
       +
       

     

       420
       420
       +
       4. **Reporter Chaining**: Following Logs library conventions by supporting reporter chaining allows integration with existing logging infrastructure.

     

       421
       421
       +
       

     

       422
       422
       +
       5. **Export for Claude**: Special export format that prepares context within token limits, focusing on recent errors and patterns.

     

       423
       423
       +
       

     

       424
       424
       +
       ### Usage Example

     

       425
       425
       +
       

     

       426
       426
       +
       ```ocaml

     

       427
       427
       +
       (* Initialize the database *)

     

       428
       428
       +
       let db = Dancer_logs.init ~path:"app_logs.db" () in

     

       429
       429
       +
       

     

       430
       430
       +
       (* Create and set the reporter *)

     

       431
       431
       +
       let reporter = Dancer_logs.reporter db in

     

       432
       432
       +
       let reporter = Dancer_logs.chain_reporter db (Logs_fmt.reporter ()) in

     

       433
       433
       +
       Logs.set_reporter reporter;

     

       434
       434
       +
       

     

       435
       435
       +
       (* Use with context *)

     

       436
       436
       +
       let ctx = Dancer_logs.Context.(

     

       437
       437
       +
         empty_context

     

       438
       438
       +
         |> with_session "session-123"

     

       439
       439
       +
         |> with_request "req-456"

     

       440
       440
       +
         |> with_user "user-789"

     

       441
       441
       +
         |> add_tag "payment"

     

       442
       442
       +
         |> add_label "environment" "production"

     

       443
       443
       +
       ) in

     

       444
       444
       +
       

     

       445
       445
       +
       (* Log with full context *)

     

       446
       446
       +
       Dancer_logs.log db

     

       447
       447
       +
         ~level:Logs.Error

     

       448
       448
       +
         ~source:"Payment.Gateway"

     

       449
       449
       +
         ~message:"Payment failed: timeout"

     

       450
       450
       +
         ~context:ctx

     

       451
       451
       +
         ~error:{

     

       452
       452
       +
           error_type = Some "TimeoutException";

     

       453
       453
       +
           error_code = Some "GATEWAY_TIMEOUT";

     

       454
       454
       +
           error_hash = Some (hash_of_error);

     

       455
       455
       +
           stack_trace = Some (Printexc.get_backtrace ());

     

       456
       456
       +
         }

     

       457
       457
       +
         ~performance:{

     

       458
       458
       +
           duration_ms = Some 30000.0;

     

       459
       459
       +
           memory_before = Some 1000000;

     

       460
       460
       +
           memory_after = Some 1200000;

     

       461
       461
       +
           cpu_time_ms = Some 50.0;

     

       462
       462
       +
         }

     

       463
       463
       +
         ();

     

       464
       464
       +
       ```

     

       465
       465
       +
       

     

       466
       466
       +
       ### CLI Usage

     

       467
       467
       +
       

     

       468
       468
       +
       ```bash

     

       469
       469
       +
       # View recent errors

     

       470
       470
       +
       dancer-logs errors

     

       471
       471
       +
       

     

       472
       472
       +
       # Search for specific patterns

     

       473
       473
       +
       dancer-logs search "connection refused"

     

       474
       474
       +
       

     

       475
       475
       +
       # View error patterns

     

       476
       476
       +
       dancer-logs patterns --min-count 10

     

       477
       477
       +
       

     

       478
       478
       +
       # Export context for Claude

     

       479
       479
       +
       dancer-logs export -f claude -o context.txt --since 24

     

       480
       480
       +
       

     

       481
       481
       +
       # Follow logs in real-time (to be implemented)

     

       482
       482
       +
       dancer-logs list -f

     

       483
       483
       +
       ```

     

       484
       484
       +
       

     

       485
       485
       +
       ### Next Steps

     

       486
       486
       +
       

     

       487
       487
       +
       1. Implement pattern detection algorithm that runs periodically

     

       488
       488
       +
       2. Add metrics collection from /proc and system calls

     

       489
       489
       +
       3. Implement log following mode for real-time monitoring

     

       490
       490
       +
       4. Add session correlation views

     

       491
       491
       +
       5. Create integration tests with sample applications

+22

dancer/Makefile

···

       1
       1
       +
       .PHONY: deps build clean test doc

     

       2
       2
       +
       

     

       3
       3
       +
       deps:

     

       4
       4
       +
       	opam install -y sqlite3 eio eio_main logs fmt ptime digestif yojson cmdliner ppx_deriving alcotest str

     

       5
       5
       +
       

     

       6
       6
       +
       build:

     

       7
       7
       +
       	opam exec -- dune build @check

     

       8
       8
       +
       

     

       9
       9
       +
       clean:

     

       10
       10
       +
       	opam exec -- dune clean

     

       11
       11
       +
       

     

       12
       12
       +
       test:

     

       13
       13
       +
       	opam exec -- dune runtest

     

       14
       14
       +
       

     

       15
       15
       +
       doc:

     

       16
       16
       +
       	opam exec -- dune build @doc

     

       17
       17
       +
       

     

       18
       18
       +
       run-cli:

     

       19
       19
       +
       	opam exec -- dune exec dancer-logs-cli -- --help

     

       20
       20
       +
       

     

       21
       21
       +
       install:

     

       22
       22
       +
       	opam exec -- dune install

+72

dancer/README.md

···

       1
       1
       +
       # Dancer - Self-Modifying OCaml Services

     

       2
       2
       +
       

     

       3
       3
       +
       Dancer is an ambitious OCaml library that enables long-running services to automatically improve themselves through AI-assisted code generation, with comprehensive logging, pattern detection, and safe deployment mechanisms.

     

       4
       4
       +
       

     

       5
       5
       +
       ## Components

     

       6
       6
       +
       

     

       7
       7
       +
       ### dancer-logs

     

       8
       8
       +
       

     

       9
       9
       +
       Comprehensive structured logging with SQLite backend. Features:

     

       10
       10
       +
       - Full structured logging with no truncation

     

       11
       11
       +
       - SQLite storage with FTS5 full-text search

     

       12
       12
       +
       - Pattern detection and normalization

     

       13
       13
       +
       - Eio fiber and domain tracking

     

       14
       14
       +
       - Rich context capture (sessions, requests, traces, performance metrics)

     

       15
       15
       +
       - OCaml Logs reporter integration

     

       16
       16
       +
       - Colorful CLI for log analysis

     

       17
       17
       +
       

     

       18
       18
       +
       ## Installation

     

       19
       19
       +
       

     

       20
       20
       +
       ```bash

     

       21
       21
       +
       # Install dependencies

     

       22
       22
       +
       make deps

     

       23
       23
       +
       

     

       24
       24
       +
       # Build the project

     

       25
       25
       +
       make build

     

       26
       26
       +
       

     

       27
       27
       +
       # Run the CLI

     

       28
       28
       +
       make run-cli

     

       29
       29
       +
       ```

     

       30
       30
       +
       

     

       31
       31
       +
       ## Usage

     

       32
       32
       +
       

     

       33
       33
       +
       ```ocaml

     

       34
       34
       +
       (* Initialize logging *)

     

       35
       35
       +
       let db = Dancer_logs.init ~path:"app.db" () in

     

       36
       36
       +
       let reporter = Dancer_logs.reporter db in

     

       37
       37
       +
       Logs.set_reporter reporter;

     

       38
       38
       +
       

     

       39
       39
       +
       (* Log with context *)

     

       40
       40
       +
       let ctx = Dancer_logs.Context.(

     

       41
       41
       +
         empty_context

     

       42
       42
       +
         |> with_session "session-123"

     

       43
       43
       +
         |> with_request "req-456"

     

       44
       44
       +
       ) in

     

       45
       45
       +
       

     

       46
       46
       +
       Dancer_logs.log db ~level:Logs.Error ~source:"MyApp" 

     

       47
       47
       +
         ~message:"Connection failed" ~context:ctx ()

     

       48
       48
       +
       ```

     

       49
       49
       +
       

     

       50
       50
       +
       ## CLI Usage

     

       51
       51
       +
       

     

       52
       52
       +
       ```bash

     

       53
       53
       +
       # View recent errors

     

       54
       54
       +
       dancer-logs errors

     

       55
       55
       +
       

     

       56
       56
       +
       # Search logs

     

       57
       57
       +
       dancer-logs search "connection refused"

     

       58
       58
       +
       

     

       59
       59
       +
       # View patterns

     

       60
       60
       +
       dancer-logs patterns --min-count 10

     

       61
       61
       +
       

     

       62
       62
       +
       # Export for Claude

     

       63
       63
       +
       dancer-logs export -f claude -o context.txt

     

       64
       64
       +
       ```

     

       65
       65
       +
       

     

       66
       66
       +
       ## Architecture

     

       67
       67
       +
       

     

       68
       68
       +
       See [CLAUDE.md](CLAUDE.md) for detailed architecture and design decisions.

     

       69
       69
       +
       

     

       70
       70
       +
       ## License

     

       71
       71
       +
       

     

       72
       72
       +
       MIT

+352

dancer/bin/dancer_logs_cli.ml

···

       1
       1
       +
       open Dancer_logs

     

       2
       2
       +
       open Cmdliner

     

       3
       3
       +
       

     

       4
       4
       +
       let style_ok = Fmt.styled `Green

     

       5
       5
       +
       let style_error = Fmt.styled `Red  

     

       6
       6
       +
       let style_warning = Fmt.styled `Yellow

     

       7
       7
       +
       let style_info = Fmt.styled `Blue

     

       8
       8
       +
       let style_debug = Fmt.styled `Cyan

     

       9
       9
       +
       let style_dim = Fmt.styled `Faint

     

       10
       10
       +
       let style_bold = Fmt.styled `Bold

     

       11
       11
       +
       

     

       12
       12
       +
       let pp_level ppf = function

     

       13
       13
       +
         | Logs.App -> Fmt.pf ppf "%a" style_bold "APP"

     

       14
       14
       +
         | Logs.Error -> Fmt.pf ppf "%a" style_error "ERR"

     

       15
       15
       +
         | Logs.Warning -> Fmt.pf ppf "%a" style_warning "WRN"

     

       16
       16
       +
         | Logs.Info -> Fmt.pf ppf "%a" style_info "INF"

     

       17
       17
       +
         | Logs.Debug -> Fmt.pf ppf "%a" style_debug "DBG"

     

       18
       18
       +
       

     

       19
       19
       +
       let pp_timestamp ppf timestamp =

     

       20
       20
       +
         let ptime = Ptime.of_float_s timestamp |> Option.get in

     

       21
       21
       +
         Fmt.pf ppf "%a" (Ptime.pp_human ~frac_s:3 ()) ptime

     

       22
       22
       +
       

     

       23
       23
       +
       let pp_source ppf source =

     

       24
       24
       +
         Fmt.pf ppf "%a" style_bold source

     

       25
       25
       +
       

     

       26
       26
       +
       let pp_duration ppf ms =

     

       27
       27
       +
         if ms > 1000.0 then

     

       28
       28
       +
           Fmt.pf ppf "%a" style_warning (Printf.sprintf "%.1fs" (ms /. 1000.0))

     

       29
       29
       +
         else if ms > 100.0 then

     

       30
       30
       +
           Fmt.pf ppf "%a" style_warning (Printf.sprintf "%.0fms" ms)

     

       31
       31
       +
         else

     

       32
       32
       +
           Fmt.pf ppf "%a" style_ok (Printf.sprintf "%.1fms" ms)

     

       33
       33
       +
       

     

       34
       34
       +
       let pp_log_row ppf row =

     

       35
       35
       +
         match row with

     

       36
       36
       +
         | timestamp :: level :: source :: message :: rest ->

     

       37
       37
       +
             let timestamp = match timestamp with

     

       38
       38
       +
               | Sqlite3.Data.FLOAT f -> f

     

       39
       39
       +
               | _ -> 0.0

     

       40
       40
       +
             in

     

       41
       41
       +
             let level = match level with

     

       42
       42
       +
               | Sqlite3.Data.INT i -> 

     

       43
       43
       +
                   (match Int64.to_int i with

     

       44
       44
       +
                   | 0 -> Logs.App

     

       45
       45
       +
                   | 1 -> Logs.Error

     

       46
       46
       +
                   | 2 -> Logs.Warning

     

       47
       47
       +
                   | 3 -> Logs.Info

     

       48
       48
       +
                   | _ -> Logs.Debug)

     

       49
       49
       +
               | _ -> Logs.Info

     

       50
       50
       +
             in

     

       51
       51
       +
             let source = match source with

     

       52
       52
       +
               | Sqlite3.Data.TEXT s -> s

     

       53
       53
       +
               | _ -> "unknown"

     

       54
       54
       +
             in

     

       55
       55
       +
             let message = match message with

     

       56
       56
       +
               | Sqlite3.Data.TEXT s -> s

     

       57
       57
       +
               | _ -> ""

     

       58
       58
       +
             in

     

       59
       59
       +
             

     

       60
       60
       +
             Fmt.pf ppf "@[<h>%a %a %a@]@.  %s@."

     

       61
       61
       +
               pp_timestamp timestamp

     

       62
       62
       +
               pp_level level

     

       63
       63
       +
               pp_source source

     

       64
       64
       +
               message;

     

       65
       65
       +
             

     

       66
       66
       +
             List.iteri (fun i data ->

     

       67
       67
       +
               match data with

     

       68
       68
       +
               | Sqlite3.Data.TEXT s when String.length s > 0 ->

     

       69
       69
       +
                   Fmt.pf ppf "  %a: %s@." style_dim 

     

       70
       70
       +
                     (match i with

     

       71
       71
       +
                     | 0 -> "error"

     

       72
       72
       +
                     | 1 -> "code"

     

       73
       73
       +
                     | 2 -> "trace"

     

       74
       74
       +
                     | _ -> string_of_int i) s

     

       75
       75
       +
               | Sqlite3.Data.INT i -> 

     

       76
       76
       +
                   Fmt.pf ppf "  %a: %Ld@." style_dim (string_of_int i) i

     

       77
       77
       +
               | Sqlite3.Data.FLOAT f ->

     

       78
       78
       +
                   Fmt.pf ppf "  %a: %f@." style_dim (string_of_int i) f

     

       79
       79
       +
               | _ -> ()

     

       80
       80
       +
             ) rest

     

       81
       81
       +
         | _ -> ()

     

       82
       82
       +
       

     

       83
       83
       +
       let pp_pattern ppf pattern =

     

       84
       84
       +
         match pattern with

     

       85
       85
       +
         | [id; hash; pattern_text; first_seen; last_seen; count; severity] ->

     

       86
       86
       +
             let count = match count with

     

       87
       87
       +
               | Sqlite3.Data.INT i -> Int64.to_int i

     

       88
       88
       +
               | _ -> 0

     

       89
       89
       +
             in

     

       90
       90
       +
             let severity_color = 

     

       91
       91
       +
               match severity with

     

       92
       92
       +
               | Sqlite3.Data.FLOAT f when f > 7.0 -> style_error

     

       93
       93
       +
               | Sqlite3.Data.FLOAT f when f > 4.0 -> style_warning

     

       94
       94
       +
               | _ -> style_info

     

       95
       95
       +
             in

     

       96
       96
       +
             

     

       97
       97
       +
             Fmt.pf ppf "@[<v>%a Pattern (×%d) %a@.  %s@.  First: %a | Last: %a@]@."

     

       98
       98
       +
               severity_color "●"

     

       99
       99
       +
               count

     

       100
       100
       +
               style_dim 

     

       101
       101
       +
               (match hash with Sqlite3.Data.TEXT s -> String.sub s 0 8 | _ -> "")

     

       102
       102
       +
               (match pattern_text with Sqlite3.Data.TEXT s -> s | _ -> "")

     

       103
       103
       +
               pp_timestamp (match first_seen with Sqlite3.Data.FLOAT f -> f | _ -> 0.0)

     

       104
       104
       +
               pp_timestamp (match last_seen with Sqlite3.Data.FLOAT f -> f | _ -> 0.0)

     

       105
       105
       +
         | _ -> ()

     

       106
       106
       +
       

     

       107
       107
       +
       let pp_error_summary ppf row =

     

       108
       108
       +
         match row with

     

       109
       109
       +
         | [hash; error_type; error_code; count; sessions; users; last; first; sources; avg_dur; sample] ->

     

       110
       110
       +
             let count = match count with Sqlite3.Data.INT i -> Int64.to_int i | _ -> 0 in

     

       111
       111
       +
             let sessions = match sessions with Sqlite3.Data.INT i -> Int64.to_int i | _ -> 0 in

     

       112
       112
       +
             let error_type = match error_type with Sqlite3.Data.TEXT s -> s | _ -> "unknown" in

     

       113
       113
       +
             

     

       114
       114
       +
             Fmt.pf ppf "@[<v>%a %s (×%d in %d sessions)@.  %s@.  Sources: %s@]@."

     

       115
       115
       +
               style_error "ERROR"

     

       116
       116
       +
               error_type count sessions

     

       117
       117
       +
               (match sample with Sqlite3.Data.TEXT s -> 

     

       118
       118
       +
                 if String.length s > 80 then String.sub s 0 77 ^ "..." else s 

     

       119
       119
       +
               | _ -> "")

     

       120
       120
       +
               (match sources with Sqlite3.Data.TEXT s -> s | _ -> "")

     

       121
       121
       +
         | _ -> ()

     

       122
       122
       +
       

     

       123
       123
       +
       let list_cmd =

     

       124
       124
       +
         let list db_path level source since limit follow =

     

       125
       125
       +
           Eio_main.run @@ fun _env ->

     

       126
       126
       +
           let db = Dancer_logs.init ~path:db_path () in

     

       127
       127
       +
           

     

       128
       128
       +
           let filter = {

     

       129
       129
       +
             Query.default_filter with

     

       130
       130
       +
             level;

     

       131
       131
       +
             source;

     

       132
       132
       +
             since = Option.map (fun h -> Unix.gettimeofday () -. (h *. 3600.0)) since;

     

       133
       133
       +
             limit;

     

       134
       134
       +
           } in

     

       135
       135
       +
           

     

       136
       136
       +
           let rows = Query.list db filter in

     

       137
       137
       +
           

     

       138
       138
       +
           Fmt.pr "@[<v>%a@]@." 

     

       139
       139
       +
             (Fmt.list ~sep:Fmt.cut pp_log_row) rows;

     

       140
       140
       +
           

     

       141
       141
       +
           Dancer_logs.close db

     

       142
       142
       +
         in

     

       143
       143
       +
         

     

       144
       144
       +
         let db_path = Arg.(value & opt string "dancer_logs.db" & 

     

       145
       145
       +
           info ["d"; "db"] ~doc:"Database path") in

     

       146
       146
       +
         

     

       147
       147
       +
         let level = Arg.(value & opt (some (enum [

     

       148
       148
       +
           "error", Logs.Error;

     

       149
       149
       +
           "warning", Logs.Warning;

     

       150
       150
       +
           "info", Logs.Info;

     

       151
       151
       +
           "debug", Logs.Debug;

     

       152
       152
       +
         ])) None & info ["l"; "level"] ~doc:"Filter by log level") in

     

       153
       153
       +
         

     

       154
       154
       +
         let source = Arg.(value & opt (some string) None & 

     

       155
       155
       +
           info ["s"; "source"] ~doc:"Filter by source module") in

     

       156
       156
       +
         

     

       157
       157
       +
         let since = Arg.(value & opt (some float) None & 

     

       158
       158
       +
           info ["since"] ~doc:"Show logs from last N hours") in

     

       159
       159
       +
         

     

       160
       160
       +
         let limit = Arg.(value & opt int 100 & 

     

       161
       161
       +
           info ["n"; "limit"] ~doc:"Maximum number of results") in

     

       162
       162
       +
         

     

       163
       163
       +
         let follow = Arg.(value & flag & 

     

       164
       164
       +
           info ["f"; "follow"] ~doc:"Follow mode (like tail -f)") in

     

       165
       165
       +
         

     

       166
       166
       +
         let doc = "List log entries" in

     

       167
       167
       +
         let info = Cmd.info "list" ~doc in

     

       168
       168
       +
         Cmd.v info Term.(const list $ db_path $ level $ source $ since $ limit $ follow)

     

       169
       169
       +
       

     

       170
       170
       +
       let search_cmd =

     

       171
       171
       +
         let search db_path query limit =

     

       172
       172
       +
           Eio_main.run @@ fun _env ->

     

       173
       173
       +
           let db = Dancer_logs.init ~path:db_path () in

     

       174
       174
       +
           

     

       175
       175
       +
           let rows = Query.search db query ~limit () in

     

       176
       176
       +
           

     

       177
       177
       +
           Fmt.pr "@[<v>%a Found %d matches@.@.%a@]@."

     

       178
       178
       +
             style_bold "🔍" (List.length rows)

     

       179
       179
       +
             (Fmt.list ~sep:(Fmt.any "@.---@.") pp_log_row) rows;

     

       180
       180
       +
           

     

       181
       181
       +
           Dancer_logs.close db

     

       182
       182
       +
         in

     

       183
       183
       +
         

     

       184
       184
       +
         let db_path = Arg.(value & opt string "dancer_logs.db" & 

     

       185
       185
       +
           info ["d"; "db"] ~doc:"Database path") in

     

       186
       186
       +
         

     

       187
       187
       +
         let query = Arg.(required & pos 0 (some string) None & 

     

       188
       188
       +
           info [] ~docv:"QUERY" ~doc:"Search query") in

     

       189
       189
       +
         

     

       190
       190
       +
         let limit = Arg.(value & opt int 100 & 

     

       191
       191
       +
           info ["n"; "limit"] ~doc:"Maximum number of results") in

     

       192
       192
       +
         

     

       193
       193
       +
         let doc = "Full-text search in logs" in

     

       194
       194
       +
         let info = Cmd.info "search" ~doc in

     

       195
       195
       +
         Cmd.v info Term.(const search $ db_path $ query $ limit)

     

       196
       196
       +
       

     

       197
       197
       +
       let patterns_cmd =

     

       198
       198
       +
         let patterns db_path min_count since =

     

       199
       199
       +
           Eio_main.run @@ fun _env ->

     

       200
       200
       +
           let db = Dancer_logs.init ~path:db_path () in

     

       201
       201
       +
           

     

       202
       202
       +
           let patterns = Pattern.list db ~min_count ?since () in

     

       203
       203
       +
           

     

       204
       204
       +
           Fmt.pr "@[<v>%a Detected Patterns@.@.%a@]@."

     

       205
       205
       +
             style_bold "📊"

     

       206
       206
       +
             (Fmt.list ~sep:Fmt.cut (fun ppf p ->

     

       207
       207
       +
               pp_pattern ppf [

     

       208
       208
       +
                 Sqlite3.Data.INT (Int64.of_int p.Pattern.id);

     

       209
       209
       +
                 Sqlite3.Data.TEXT p.pattern_hash;

     

       210
       210
       +
                 Sqlite3.Data.TEXT p.pattern;

     

       211
       211
       +
                 Sqlite3.Data.FLOAT p.first_seen;

     

       212
       212
       +
                 Sqlite3.Data.FLOAT p.last_seen;

     

       213
       213
       +
                 Sqlite3.Data.INT (Int64.of_int p.occurrence_count);

     

       214
       214
       +
                 match p.severity_score with

     

       215
       215
       +
                 | Some s -> Sqlite3.Data.FLOAT s

     

       216
       216
       +
                 | None -> Sqlite3.Data.NULL

     

       217
       217
       +
               ]

     

       218
       218
       +
             )) patterns;

     

       219
       219
       +
           

     

       220
       220
       +
           Dancer_logs.close db

     

       221
       221
       +
         in

     

       222
       222
       +
         

     

       223
       223
       +
         let db_path = Arg.(value & opt string "dancer_logs.db" & 

     

       224
       224
       +
           info ["d"; "db"] ~doc:"Database path") in

     

       225
       225
       +
         

     

       226
       226
       +
         let min_count = Arg.(value & opt int 5 & 

     

       227
       227
       +
           info ["c"; "min-count"] ~doc:"Minimum occurrence count") in

     

       228
       228
       +
         

     

       229
       229
       +
         let since = Arg.(value & opt (some float) None & 

     

       230
       230
       +
           info ["since"] ~doc:"Patterns from last N hours") in

     

       231
       231
       +
         

     

       232
       232
       +
         let doc = "Show detected error patterns" in

     

       233
       233
       +
         let info = Cmd.info "patterns" ~doc in

     

       234
       234
       +
         Cmd.v info Term.(const patterns $ db_path $ min_count $ since)

     

       235
       235
       +
       

     

       236
       236
       +
       let errors_cmd =

     

       237
       237
       +
         let errors db_path limit =

     

       238
       238
       +
           Eio_main.run @@ fun _env ->

     

       239
       239
       +
           let db = Dancer_logs.init ~path:db_path () in

     

       240
       240
       +
           

     

       241
       241
       +
           let errors = Query.recent_errors db ~limit () in

     

       242
       242
       +
           

     

       243
       243
       +
           Fmt.pr "@[<v>%a Recent Errors (last 24h)@.@.%a@]@."

     

       244
       244
       +
             style_error "⚠️"

     

       245
       245
       +
             (Fmt.list ~sep:(Fmt.any "@.") pp_error_summary) errors;

     

       246
       246
       +
           

     

       247
       247
       +
           Dancer_logs.close db

     

       248
       248
       +
         in

     

       249
       249
       +
         

     

       250
       250
       +
         let db_path = Arg.(value & opt string "dancer_logs.db" & 

     

       251
       251
       +
           info ["d"; "db"] ~doc:"Database path") in

     

       252
       252
       +
         

     

       253
       253
       +
         let limit = Arg.(value & opt int 50 & 

     

       254
       254
       +
           info ["n"; "limit"] ~doc:"Maximum number of results") in

     

       255
       255
       +
         

     

       256
       256
       +
         let doc = "Show recent errors with aggregation" in

     

       257
       257
       +
         let info = Cmd.info "errors" ~doc in

     

       258
       258
       +
         Cmd.v info Term.(const errors $ db_path $ limit)

     

       259
       259
       +
       

     

       260
       260
       +
       let stats_cmd =

     

       261
       261
       +
         let stats db_path =

     

       262
       262
       +
           Eio_main.run @@ fun _env ->

     

       263
       263
       +
           let db = Dancer_logs.init ~path:db_path () in

     

       264
       264
       +
           

     

       265
       265
       +
           let count_by_level = Query.count_by_level db ~since:(Unix.gettimeofday () -. 86400.0) () in

     

       266
       266
       +
           

     

       267
       267
       +
           Fmt.pr "@[<v>%a Database Statistics@.@." style_bold "📈";

     

       268
       268
       +
           

     

       269
       269
       +
           List.iter (fun (level, count) ->

     

       270
       270
       +
             Fmt.pr "  %a: %d@." pp_level level count

     

       271
       271
       +
           ) count_by_level;

     

       272
       272
       +
           

     

       273
       273
       +
           let db_stats = Dancer_logs.stats db in

     

       274
       274
       +
           List.iter (fun (k, v) ->

     

       275
       275
       +
             Fmt.pr "  %a: %s@." style_dim k v

     

       276
       276
       +
           ) db_stats;

     

       277
       277
       +
           

     

       278
       278
       +
           Fmt.pr "@]@.";

     

       279
       279
       +
           

     

       280
       280
       +
           Dancer_logs.close db

     

       281
       281
       +
         in

     

       282
       282
       +
         

     

       283
       283
       +
         let db_path = Arg.(value & opt string "dancer_logs.db" & 

     

       284
       284
       +
           info ["d"; "db"] ~doc:"Database path") in

     

       285
       285
       +
         

     

       286
       286
       +
         let doc = "Show database statistics" in

     

       287
       287
       +
         let info = Cmd.info "stats" ~doc in

     

       288
       288
       +
         Cmd.v info Term.(const stats $ db_path)

     

       289
       289
       +
       

     

       290
       290
       +
       let export_cmd =

     

       291
       291
       +
         let export db_path format output since =

     

       292
       292
       +
           Eio_main.run @@ fun _env ->

     

       293
       293
       +
           let db = Dancer_logs.init ~path:db_path () in

     

       294
       294
       +
           

     

       295
       295
       +
           let filter = {

     

       296
       296
       +
             Query.default_filter with

     

       297
       297
       +
             since = Option.map (fun h -> Unix.gettimeofday () -. (h *. 3600.0)) since;

     

       298
       298
       +
             limit = 10000;

     

       299
       299
       +
           } in

     

       300
       300
       +
           

     

       301
       301
       +
           let rows = Query.list db filter in

     

       302
       302
       +
           

     

       303
       303
       +
           let content = match format with

     

       304
       304
       +
             | "json" -> Yojson.Safe.to_string (Export.to_json rows)

     

       305
       305
       +
             | "csv" -> Export.to_csv rows

     

       306
       306
       +
             | "claude" -> Export.for_claude db ()

     

       307
       307
       +
             | _ -> failwith "Unknown format"

     

       308
       308
       +
           in

     

       309
       309
       +
           

     

       310
       310
       +
           let oc = open_out output in

     

       311
       311
       +
           output_string oc content;

     

       312
       312
       +
           close_out oc;

     

       313
       313
       +
           

     

       314
       314
       +
           Fmt.pr "%a Exported %d logs to %s@."

     

       315
       315
       +
             style_ok "✓" (List.length rows) output;

     

       316
       316
       +
           

     

       317
       317
       +
           Dancer_logs.close db

     

       318
       318
       +
         in

     

       319
       319
       +
         

     

       320
       320
       +
         let db_path = Arg.(value & opt string "dancer_logs.db" & 

     

       321
       321
       +
           info ["d"; "db"] ~doc:"Database path") in

     

       322
       322
       +
         

     

       323
       323
       +
         let format = Arg.(value & opt (enum [

     

       324
       324
       +
           "json", "json";

     

       325
       325
       +
           "csv", "csv";

     

       326
       326
       +
           "claude", "claude";

     

       327
       327
       +
         ]) "json" & info ["f"; "format"] ~doc:"Export format") in

     

       328
       328
       +
         

     

       329
       329
       +
         let output = Arg.(required & opt (some string) None & 

     

       330
       330
       +
           info ["o"; "output"] ~doc:"Output file") in

     

       331
       331
       +
         

     

       332
       332
       +
         let since = Arg.(value & opt (some float) None & 

     

       333
       333
       +
           info ["since"] ~doc:"Export logs from last N hours") in

     

       334
       334
       +
         

     

       335
       335
       +
         let doc = "Export logs to file" in

     

       336
       336
       +
         let info = Cmd.info "export" ~doc in

     

       337
       337
       +
         Cmd.v info Term.(const export $ db_path $ format $ output $ since)

     

       338
       338
       +
       

     

       339
       339
       +
       let main_cmd =

     

       340
       340
       +
         let doc = "Dancer Logs CLI - Query and analyze structured logs" in

     

       341
       341
       +
         let info = Cmd.info "dancer-logs" ~version:"0.1.0" ~doc in

     

       342
       342
       +
         let default = Term.(ret (const (`Help (`Pager, None)))) in

     

       343
       343
       +
         Cmd.group info ~default [

     

       344
       344
       +
           list_cmd;

     

       345
       345
       +
           search_cmd;

     

       346
       346
       +
           patterns_cmd;

     

       347
       347
       +
           errors_cmd;

     

       348
       348
       +
           stats_cmd;

     

       349
       349
       +
           export_cmd;

     

       350
       350
       +
         ]

     

       351
       351
       +
       

     

       352
       352
       +
       let () = exit (Cmd.eval main_cmd)

dancer/bin/dune

···

       1
       1
       +
       (executable

     

       2
       2
       +
        (public_name dancer-logs-cli)

     

       3
       3
       +
        (package dancer-logs-cli)

     

       4
       4
       +
        (name dancer_logs_cli)

     

       5
       5
       +
        (libraries dancer_logs cmdliner fmt fmt.tty ptime ptime.clock.os eio_main))

+34

dancer/dancer-logs-cli.opam

···

       1
       1
       +
       # This file is generated by dune, edit dune-project instead

     

       2
       2
       +
       opam-version: "2.0"

     

       3
       3
       +
       version: "0.1.0"

     

       4
       4
       +
       synopsis: "CLI tool for querying and analyzing dancer-logs"

     

       5
       5
       +
       description: """

     

       6
       6
       +
       Command-line interface for browsing, searching, and exporting 

     

       7
       7
       +
         logs captured by dancer-logs"""

     

       8
       8
       +
       maintainer: ["anil@recoil.org"]

     

       9
       9
       +
       authors: ["Anil Madhavapeddy"]

     

       10
       10
       +
       license: "MIT"

     

       11
       11
       +
       homepage: "https://github.com/avsm/dancer"

     

       12
       12
       +
       bug-reports: "https://github.com/avsm/dancer/issues"

     

       13
       13
       +
       depends: [

     

       14
       14
       +
         "dune" {>= "3.0"}

     

       15
       15
       +
         "dancer-logs"

     

       16
       16
       +
         "cmdliner"

     

       17
       17
       +
         "fmt"

     

       18
       18
       +
         "odoc" {with-doc}

     

       19
       19
       +
       ]

     

       20
       20
       +
       build: [

     

       21
       21
       +
         ["dune" "subst"] {dev}

     

       22
       22
       +
         [

     

       23
       23
       +
           "dune"

     

       24
       24
       +
           "build"

     

       25
       25
       +
           "-p"

     

       26
       26
       +
           name

     

       27
       27
       +
           "-j"

     

       28
       28
       +
           jobs

     

       29
       29
       +
           "@install"

     

       30
       30
       +
           "@runtest" {with-test}

     

       31
       31
       +
           "@doc" {with-doc}

     

       32
       32
       +
         ]

     

       33
       33
       +
       ]

     

       34
       34
       +
       dev-repo: "git+https://github.com/avsm/dancer.git"

+44

dancer/dancer-logs.opam

···

       1
       1
       +
       # This file is generated by dune, edit dune-project instead

     

       2
       2
       +
       opam-version: "2.0"

     

       3
       3
       +
       version: "0.1.0"

     

       4
       4
       +
       synopsis: "Comprehensive structured logging with SQLite backend for Dancer"

     

       5
       5
       +
       description: """

     

       6
       6
       +
       dancer-logs provides structured logging with SQLite storage, 

     

       7
       7
       +
         pattern detection, and comprehensive context capture for AI-assisted debugging 

     

       8
       8
       +
         and self-modification"""

     

       9
       9
       +
       maintainer: ["anil@recoil.org"]

     

       10
       10
       +
       authors: ["Anil Madhavapeddy"]

     

       11
       11
       +
       license: "MIT"

     

       12
       12
       +
       homepage: "https://github.com/avsm/dancer"

     

       13
       13
       +
       bug-reports: "https://github.com/avsm/dancer/issues"

     

       14
       14
       +
       depends: [

     

       15
       15
       +
         "ocaml"

     

       16
       16
       +
         "dune" {>= "3.0"}

     

       17
       17
       +
         "eio"

     

       18
       18
       +
         "eio_main"

     

       19
       19
       +
         "logs"

     

       20
       20
       +
         "sqlite3"

     

       21
       21
       +
         "fmt"

     

       22
       22
       +
         "ptime"

     

       23
       23
       +
         "digestif"

     

       24
       24
       +
         "yojson"

     

       25
       25
       +
         "ppx_deriving"

     

       26
       26
       +
         "cmdliner"

     

       27
       27
       +
         "alcotest" {with-test}

     

       28
       28
       +
         "odoc" {with-doc}

     

       29
       29
       +
       ]

     

       30
       30
       +
       build: [

     

       31
       31
       +
         ["dune" "subst"] {dev}

     

       32
       32
       +
         [

     

       33
       33
       +
           "dune"

     

       34
       34
       +
           "build"

     

       35
       35
       +
           "-p"

     

       36
       36
       +
           name

     

       37
       37
       +
           "-j"

     

       38
       38
       +
           jobs

     

       39
       39
       +
           "@install"

     

       40
       40
       +
           "@runtest" {with-test}

     

       41
       41
       +
           "@doc" {with-doc}

     

       42
       42
       +
         ]

     

       43
       43
       +
       ]

     

       44
       44
       +
       dev-repo: "git+https://github.com/avsm/dancer.git"

+45

dancer/dune-project

···

       1
       1
       +
       (lang dune 3.0)

     

       2
       2
       +
       (name dancer)

     

       3
       3
       +
       (version 0.1.0)

     

       4
       4
       +
       

     

       5
       5
       +
       (generate_opam_files true)

     

       6
       6
       +
       

     

       7
       7
       +
       (source

     

       8
       8
       +
        (github avsm/dancer))

     

       9
       9
       +
       

     

       10
       10
       +
       (license MIT)

     

       11
       11
       +
       

     

       12
       12
       +
       (authors "Anil Madhavapeddy")

     

       13
       13
       +
       

     

       14
       14
       +
       (maintainers "anil@recoil.org")

     

       15
       15
       +
       

     

       16
       16
       +
       (package

     

       17
       17
       +
        (name dancer-logs)

     

       18
       18
       +
        (synopsis "Comprehensive structured logging with SQLite backend for Dancer")

     

       19
       19
       +
        (description "dancer-logs provides structured logging with SQLite storage, 

     

       20
       20
       +
         pattern detection, and comprehensive context capture for AI-assisted debugging 

     

       21
       21
       +
         and self-modification")

     

       22
       22
       +
        (depends

     

       23
       23
       +
         ocaml

     

       24
       24
       +
         dune

     

       25
       25
       +
         eio

     

       26
       26
       +
         eio_main

     

       27
       27
       +
         logs

     

       28
       28
       +
         sqlite3

     

       29
       29
       +
         fmt

     

       30
       30
       +
         ptime

     

       31
       31
       +
         digestif

     

       32
       32
       +
         yojson

     

       33
       33
       +
         ppx_deriving

     

       34
       34
       +
         cmdliner

     

       35
       35
       +
         (alcotest :with-test)))

     

       36
       36
       +
       

     

       37
       37
       +
       (package

     

       38
       38
       +
        (name dancer-logs-cli)

     

       39
       39
       +
        (synopsis "CLI tool for querying and analyzing dancer-logs")

     

       40
       40
       +
        (description "Command-line interface for browsing, searching, and exporting 

     

       41
       41
       +
         logs captured by dancer-logs")

     

       42
       42
       +
        (depends

     

       43
       43
       +
         dancer-logs

     

       44
       44
       +
         cmdliner

     

       45
       45
       +
         fmt))

+545

dancer/lib/dancer_logs/dancer_logs.ml

···

       1
       1
       +
       open Eio

     

       2
       2
       +
       

     

       3
       3
       +
       type log_context = {

     

       4
       4
       +
         session_id : string option;

     

       5
       5
       +
         request_id : string option;

     

       6
       6
       +
         trace_id : string option;

     

       7
       7
       +
         span_id : string option;

     

       8
       8
       +
         parent_span_id : string option;

     

       9
       9
       +
         user_id : string option;

     

       10
       10
       +
         tenant_id : string option;

     

       11
       11
       +
         fiber_id : string option;

     

       12
       12
       +
         domain_id : int option;

     

       13
       13
       +
         tags : string list;

     

       14
       14
       +
         labels : (string * string) list;

     

       15
       15
       +
         custom : (string * Yojson.Safe.t) list;

     

       16
       16
       +
       }

     

       17
       17
       +
       

     

       18
       18
       +
       let empty_context = {

     

       19
       19
       +
         session_id = None;

     

       20
       20
       +
         request_id = None;

     

       21
       21
       +
         trace_id = None;

     

       22
       22
       +
         span_id = None;

     

       23
       23
       +
         parent_span_id = None;

     

       24
       24
       +
         user_id = None;

     

       25
       25
       +
         tenant_id = None;

     

       26
       26
       +
         fiber_id = None;

     

       27
       27
       +
         domain_id = None;

     

       28
       28
       +
         tags = [];

     

       29
       29
       +
         labels = [];

     

       30
       30
       +
         custom = [];

     

       31
       31
       +
       }

     

       32
       32
       +
       

     

       33
       33
       +
       type source_location = {

     

       34
       34
       +
         file_path : string option;

     

       35
       35
       +
         line_number : int option;

     

       36
       36
       +
         column_number : int option;

     

       37
       37
       +
         function_name : string option;

     

       38
       38
       +
         module_path : string option;

     

       39
       39
       +
       }

     

       40
       40
       +
       

     

       41
       41
       +
       type performance_metrics = {

     

       42
       42
       +
         duration_ms : float option;

     

       43
       43
       +
         memory_before : int option;

     

       44
       44
       +
         memory_after : int option;

     

       45
       45
       +
         cpu_time_ms : float option;

     

       46
       46
       +
       }

     

       47
       47
       +
       

     

       48
       48
       +
       type network_context = {

     

       49
       49
       +
         client_ip : string option;

     

       50
       50
       +
         server_ip : string option;

     

       51
       51
       +
         host_name : string option;

     

       52
       52
       +
         port : int option;

     

       53
       53
       +
         protocol : string option;

     

       54
       54
       +
         method_ : string option;

     

       55
       55
       +
         path : string option;

     

       56
       56
       +
         status_code : int option;

     

       57
       57
       +
         bytes_sent : int option;

     

       58
       58
       +
         bytes_received : int option;

     

       59
       59
       +
       }

     

       60
       60
       +
       

     

       61
       61
       +
       type error_info = {

     

       62
       62
       +
         error_type : string option;

     

       63
       63
       +
         error_code : string option;

     

       64
       64
       +
         error_hash : string option;

     

       65
       65
       +
         stack_trace : string option;

     

       66
       66
       +
       }

     

       67
       67
       +
       

     

       68
       68
       +
       type system_info = {

     

       69
       69
       +
         os_name : string;

     

       70
       70
       +
         os_version : string;

     

       71
       71
       +
         ocaml_version : string;

     

       72
       72
       +
         app_version : string option;

     

       73
       73
       +
         environment : string option;

     

       74
       74
       +
         region : string option;

     

       75
       75
       +
         container_id : string option;

     

       76
       76
       +
       }

     

       77
       77
       +
       

     

       78
       78
       +
       type t = {

     

       79
       79
       +
         db : Sqlite3.db;

     

       80
       80
       +
         mutex : Eio.Mutex.t;

     

       81
       81
       +
       }

     

       82
       82
       +
       

     

       83
       83
       +
       let get_system_info () =

     

       84
       84
       +
         {

     

       85
       85
       +
           os_name = Sys.os_type;

     

       86
       86
       +
           os_version = 

     

       87
       87
       +
             (try 

     

       88
       88
       +
               let ic = Unix.open_process_in "uname -r" in

     

       89
       89
       +
               let version = input_line ic in

     

       90
       90
       +
               close_in ic;

     

       91
       91
       +
               version

     

       92
       92
       +
             with _ -> "unknown");

     

       93
       93
       +
           ocaml_version = Sys.ocaml_version;

     

       94
       94
       +
           app_version = None;

     

       95
       95
       +
           environment = Sys.getenv_opt "ENVIRONMENT";

     

       96
       96
       +
           region = Sys.getenv_opt "REGION";

     

       97
       97
       +
           container_id = Sys.getenv_opt "HOSTNAME";

     

       98
       98
       +
         }

     

       99
       99
       +
       

     

       100
       100
       +
       let init ?(path="dancer_logs.db") () =

     

       101
       101
       +
         let db = Sqlite3.db_open path in

     

       102
       102
       +
         

     

       103
       103
       +
         let schema = 

     

       104
       104
       +
           try

     

       105
       105
       +
             let ic = open_in "lib/dancer_logs/schema.sql" in

     

       106
       106
       +
             let content = really_input_string ic (in_channel_length ic) in

     

       107
       107
       +
             close_in ic;

     

       108
       108
       +
             content

     

       109
       109
       +
           with _ ->

     

       110
       110
       +
             try

     

       111
       111
       +
               let ic = open_in "schema.sql" in

     

       112
       112
       +
               let content = really_input_string ic (in_channel_length ic) in

     

       113
       113
       +
               close_in ic;

     

       114
       114
       +
               content

     

       115
       115
       +
             with _ ->

     

       116
       116
       +
               failwith "Could not find schema.sql"

     

       117
       117
       +
         in

     

       118
       118
       +
         

     

       119
       119
       +
         let statements = String.split_on_char ';' schema in

     

       120
       120
       +
         List.iter (fun stmt ->

     

       121
       121
       +
           let stmt = String.trim stmt in

     

       122
       122
       +
           if stmt <> "" then

     

       123
       123
       +
             match Sqlite3.exec db stmt with

     

       124
       124
       +
             | Sqlite3.Rc.OK -> ()

     

       125
       125
       +
             | rc -> Printf.eprintf "SQL error: %s\n%s\n" (Sqlite3.Rc.to_string rc) stmt

     

       126
       126
       +
         ) statements;

     

       127
       127
       +
         

     

       128
       128
       +
         { db; mutex = Eio.Mutex.create () }

     

       129
       129
       +
       

     

       130
       130
       +
       let close t =

     

       131
       131
       +
         let _ = Sqlite3.db_close t.db in

     

       132
       132
       +
         ()

     

       133
       133
       +
       

     

       134
       134
       +
       let bind_text stmt idx = function

     

       135
       135
       +
         | None -> Sqlite3.bind stmt idx Sqlite3.Data.NULL

     

       136
       136
       +
         | Some s -> Sqlite3.bind stmt idx (Sqlite3.Data.TEXT s)

     

       137
       137
       +
       

     

       138
       138
       +
       let bind_int stmt idx = function

     

       139
       139
       +
         | None -> Sqlite3.bind stmt idx Sqlite3.Data.NULL

     

       140
       140
       +
         | Some i -> Sqlite3.bind stmt idx (Sqlite3.Data.INT (Int64.of_int i))

     

       141
       141
       +
       

     

       142
       142
       +
       let bind_float stmt idx = function

     

       143
       143
       +
         | None -> Sqlite3.bind stmt idx Sqlite3.Data.NULL

     

       144
       144
       +
         | Some f -> Sqlite3.bind stmt idx (Sqlite3.Data.FLOAT f)

     

       145
       145
       +
       

     

       146
       146
       +
       let log t ~level ~source ~message

     

       147
       147
       +
           ?(context=empty_context)

     

       148
       148
       +
           ?(location={file_path=None; line_number=None; column_number=None; 

     

       149
       149
       +
                       function_name=None; module_path=None})

     

       150
       150
       +
           ?(performance={duration_ms=None; memory_before=None; memory_after=None; 

     

       151
       151
       +
                          cpu_time_ms=None})

     

       152
       152
       +
           ?(network={client_ip=None; server_ip=None; host_name=None; port=None;

     

       153
       153
       +
                      protocol=None; method_=None; path=None; status_code=None;

     

       154
       154
       +
                      bytes_sent=None; bytes_received=None})

     

       155
       155
       +
           ?(error={error_type=None; error_code=None; error_hash=None; stack_trace=None})

     

       156
       156
       +
           () =

     

       157
       157
       +
         

     

       158
       158
       +
         Eio.Mutex.use_rw ~protect:false t.mutex @@ fun () ->

     

       159
       159
       +
         

     

       160
       160
       +
         let level_int = match level with

     

       161
       161
       +
           | Logs.App -> 0

     

       162
       162
       +
           | Logs.Error -> 1

     

       163
       163
       +
           | Logs.Warning -> 2

     

       164
       164
       +
           | Logs.Info -> 3

     

       165
       165
       +
           | Logs.Debug -> 4

     

       166
       166
       +
         in

     

       167
       167
       +
         

     

       168
       168
       +
         let sql = "

     

       169
       169
       +
           INSERT INTO logs (

     

       170
       170
       +
             timestamp, level, source, message,

     

       171
       171
       +
             error_type, error_code, error_hash, stack_trace,

     

       172
       172
       +
             pid, ppid, thread_id, fiber_id, domain_id,

     

       173
       173
       +
             session_id, request_id, trace_id, span_id, parent_span_id,

     

       174
       174
       +
             file_path, file_name, line_number, column_number, function_name, module_path,

     

       175
       175
       +
             duration_ms, memory_before, memory_after, memory_delta, cpu_time_ms,

     

       176
       176
       +
             client_ip, server_ip, host_name, port, protocol, method, path, 

     

       177
       177
       +
             status_code, bytes_sent, bytes_received,

     

       178
       178
       +
             user_id, tenant_id,

     

       179
       179
       +
             tags, labels, context, custom_fields

     

       180
       180
       +
           ) VALUES (

     

       181
       181
       +
             ?1, ?2, ?3, ?4,

     

       182
       182
       +
             ?5, ?6, ?7, ?8,

     

       183
       183
       +
             ?9, ?10, ?11, ?12, ?13,

     

       184
       184
       +
             ?14, ?15, ?16, ?17, ?18,

     

       185
       185
       +
             ?19, ?20, ?21, ?22, ?23, ?24,

     

       186
       186
       +
             ?25, ?26, ?27, ?28, ?29,

     

       187
       187
       +
             ?30, ?31, ?32, ?33, ?34, ?35, ?36,

     

       188
       188
       +
             ?37, ?38, ?39,

     

       189
       189
       +
             ?40, ?41,

     

       190
       190
       +
             ?42, ?43, ?44, ?45

     

       191
       191
       +
           )

     

       192
       192
       +
         " in

     

       193
       193
       +
         

     

       194
       194
       +
         let stmt = Sqlite3.prepare t.db sql in

     

       195
       195
       +
         

     

       196
       196
       +
         let timestamp = Unix.gettimeofday () in

     

       197
       197
       +
         let pid = Unix.getpid () in

     

       198
       198
       +
         let ppid = Unix.getppid () in

     

       199
       199
       +
         

     

       200
       200
       +
         let memory_delta = match performance.memory_before, performance.memory_after with

     

       201
       201
       +
           | Some b, Some a -> Some (a - b)

     

       202
       202
       +
           | _ -> None

     

       203
       203
       +
         in

     

       204
       204
       +
         

     

       205
       205
       +
         let file_name = match location.file_path with

     

       206
       206
       +
           | Some path -> Some (Filename.basename path)

     

       207
       207
       +
           | None -> None

     

       208
       208
       +
         in

     

       209
       209
       +
         

     

       210
       210
       +
         let tags_json = match context.tags with

     

       211
       211
       +
           | [] -> None

     

       212
       212
       +
           | tags -> Some (Yojson.Safe.to_string (`List (List.map (fun t -> `String t) tags)))

     

       213
       213
       +
         in

     

       214
       214
       +
         

     

       215
       215
       +
         let labels_json = match context.labels with

     

       216
       216
       +
           | [] -> None

     

       217
       217
       +
           | labels -> 

     

       218
       218
       +
               let obj = `Assoc (List.map (fun (k, v) -> (k, `String v)) labels) in

     

       219
       219
       +
               Some (Yojson.Safe.to_string obj)

     

       220
       220
       +
         in

     

       221
       221
       +
         

     

       222
       222
       +
         let custom_json = match context.custom with

     

       223
       223
       +
           | [] -> None

     

       224
       224
       +
           | custom -> Some (Yojson.Safe.to_string (`Assoc custom))

     

       225
       225
       +
         in

     

       226
       226
       +
         

     

       227
       227
       +
         let _ = Sqlite3.bind stmt 1 (Sqlite3.Data.FLOAT timestamp) in

     

       228
       228
       +
         let _ = Sqlite3.bind stmt 2 (Sqlite3.Data.INT (Int64.of_int level_int)) in

     

       229
       229
       +
         let _ = Sqlite3.bind stmt 3 (Sqlite3.Data.TEXT source) in

     

       230
       230
       +
         let _ = Sqlite3.bind stmt 4 (Sqlite3.Data.TEXT message) in

     

       231
       231
       +
         

     

       232
       232
       +
         let _ = bind_text stmt 5 error.error_type in

     

       233
       233
       +
         let _ = bind_text stmt 6 error.error_code in

     

       234
       234
       +
         let _ = bind_text stmt 7 error.error_hash in

     

       235
       235
       +
         let _ = bind_text stmt 8 error.stack_trace in

     

       236
       236
       +
         

     

       237
       237
       +
         let _ = Sqlite3.bind stmt 9 (Sqlite3.Data.INT (Int64.of_int pid)) in

     

       238
       238
       +
         let _ = Sqlite3.bind stmt 10 (Sqlite3.Data.INT (Int64.of_int ppid)) in

     

       239
       239
       +
         let _ = bind_text stmt 11 None in

     

       240
       240
       +
         let _ = bind_text stmt 12 context.fiber_id in

     

       241
       241
       +
         let _ = bind_int stmt 13 context.domain_id in

     

       242
       242
       +
         

     

       243
       243
       +
         let _ = bind_text stmt 14 context.session_id in

     

       244
       244
       +
         let _ = bind_text stmt 15 context.request_id in

     

       245
       245
       +
         let _ = bind_text stmt 16 context.trace_id in

     

       246
       246
       +
         let _ = bind_text stmt 17 context.span_id in

     

       247
       247
       +
         let _ = bind_text stmt 18 context.parent_span_id in

     

       248
       248
       +
         

     

       249
       249
       +
         let _ = bind_text stmt 19 location.file_path in

     

       250
       250
       +
         let _ = bind_text stmt 20 file_name in

     

       251
       251
       +
         let _ = bind_int stmt 21 location.line_number in

     

       252
       252
       +
         let _ = bind_int stmt 22 location.column_number in

     

       253
       253
       +
         let _ = bind_text stmt 23 location.function_name in

     

       254
       254
       +
         let _ = bind_text stmt 24 location.module_path in

     

       255
       255
       +
         

     

       256
       256
       +
         let _ = bind_float stmt 25 performance.duration_ms in

     

       257
       257
       +
         let _ = bind_int stmt 26 performance.memory_before in

     

       258
       258
       +
         let _ = bind_int stmt 27 performance.memory_after in

     

       259
       259
       +
         let _ = bind_int stmt 28 memory_delta in

     

       260
       260
       +
         let _ = bind_float stmt 29 performance.cpu_time_ms in

     

       261
       261
       +
         

     

       262
       262
       +
         let _ = bind_text stmt 30 network.client_ip in

     

       263
       263
       +
         let _ = bind_text stmt 31 network.server_ip in

     

       264
       264
       +
         let _ = bind_text stmt 32 network.host_name in

     

       265
       265
       +
         let _ = bind_int stmt 33 network.port in

     

       266
       266
       +
         let _ = bind_text stmt 34 network.protocol in

     

       267
       267
       +
         let _ = bind_text stmt 35 network.method_ in

     

       268
       268
       +
         let _ = bind_text stmt 36 network.path in

     

       269
       269
       +
         let _ = bind_int stmt 37 network.status_code in

     

       270
       270
       +
         let _ = bind_int stmt 38 network.bytes_sent in

     

       271
       271
       +
         let _ = bind_int stmt 39 network.bytes_received in

     

       272
       272
       +
         

     

       273
       273
       +
         let _ = bind_text stmt 40 context.user_id in

     

       274
       274
       +
         let _ = bind_text stmt 41 context.tenant_id in

     

       275
       275
       +
         

     

       276
       276
       +
         let _ = bind_text stmt 42 tags_json in

     

       277
       277
       +
         let _ = bind_text stmt 43 labels_json in

     

       278
       278
       +
         let _ = bind_text stmt 44 None in

     

       279
       279
       +
         let _ = bind_text stmt 45 custom_json in

     

       280
       280
       +
         

     

       281
       281
       +
         let _ = Sqlite3.step stmt in

     

       282
       282
       +
         let _ = Sqlite3.finalize stmt in

     

       283
       283
       +
         ()

     

       284
       284
       +
       

     

       285
       285
       +
       let reporter t =

     

       286
       286
       +
         let report src level ~over k msgf =

     

       287
       287
       +
           let source = Logs.Src.name src in

     

       288
       288
       +
           msgf @@ fun ?header ?tags fmt ->

     

       289
       289
       +
             Format.kasprintf (fun message ->

     

       290
       290
       +
               log t ~level ~source ~message ();

     

       291
       291
       +
               over ();

     

       292
       292
       +
               k ()

     

       293
       293
       +
             ) fmt

     

       294
       294
       +
         in

     

       295
       295
       +
         { Logs.report }

     

       296
       296
       +
       

     

       297
       297
       +
       let chain_reporter t next =

     

       298
       298
       +
         let report src level ~over k msgf =

     

       299
       299
       +
           let source = Logs.Src.name src in

     

       300
       300
       +
           msgf @@ fun ?header ?tags fmt ->

     

       301
       301
       +
             Format.kasprintf (fun message ->

     

       302
       302
       +
               log t ~level ~source ~message ();

     

       303
       303
       +
               next.Logs.report src level ~over k (fun f -> f "%s" message)

     

       304
       304
       +
             ) fmt

     

       305
       305
       +
         in

     

       306
       306
       +
         { Logs.report }

     

       307
       307
       +
       

     

       308
       308
       +
       module Context = struct

     

       309
       309
       +
         let with_session id ctx = { ctx with session_id = Some id }

     

       310
       310
       +
         let with_request id ctx = { ctx with request_id = Some id }

     

       311
       311
       +
         let with_trace id ctx = { ctx with trace_id = Some id }

     

       312
       312
       +
         let with_span id ?parent ctx = 

     

       313
       313
       +
           { ctx with span_id = Some id; parent_span_id = parent }

     

       314
       314
       +
         let with_user id ctx = { ctx with user_id = Some id }

     

       315
       315
       +
         let with_tenant id ctx = { ctx with tenant_id = Some id }

     

       316
       316
       +
         let with_fiber id ctx = { ctx with fiber_id = Some id }

     

       317
       317
       +
         let with_domain id ctx = { ctx with domain_id = Some id }

     

       318
       318
       +
         let add_tag tag ctx = { ctx with tags = tag :: ctx.tags }

     

       319
       319
       +
         let add_label key value ctx = { ctx with labels = (key, value) :: ctx.labels }

     

       320
       320
       +
         let add_custom key json ctx = { ctx with custom = (key, json) :: ctx.custom }

     

       321
       321
       +
         

     

       322
       322
       +
         let of_eio fiber =

     

       323
       323
       +
           let fiber_id = Eio.Fiber.id fiber in

     

       324
       324
       +
           { empty_context with 

     

       325
       325
       +
             fiber_id = Some (string_of_int fiber_id);

     

       326
       326
       +
             domain_id = Some (Domain.self () |> Domain.get_id :> int)

     

       327
       327
       +
           }

     

       328
       328
       +
         

     

       329
       329
       +
         let current_fiber_id () =

     

       330
       330
       +
           try

     

       331
       331
       +
             Some (string_of_int (Eio.Fiber.id (Eio.Fiber.self ())))

     

       332
       332
       +
           with _ -> None

     

       333
       333
       +
       end

     

       334
       334
       +
       

     

       335
       335
       +
       module Pattern = struct

     

       336
       336
       +
         type t = {

     

       337
       337
       +
           id : int;

     

       338
       338
       +
           pattern_hash : string;

     

       339
       339
       +
           pattern : string;

     

       340
       340
       +
           first_seen : float;

     

       341
       341
       +
           last_seen : float;

     

       342
       342
       +
           occurrence_count : int;

     

       343
       343
       +
           severity_score : float option;

     

       344
       344
       +
         }

     

       345
       345
       +
         

     

       346
       346
       +
         let normalize_message msg =

     

       347
       347
       +
           let msg = Str.global_replace (Str.regexp "[0-9]+") "N" msg in

     

       348
       348
       +
           let msg = Str.global_replace (Str.regexp "0x[0-9a-fA-F]+") "0xHEX" msg in

     

       349
       349
       +
           let msg = Str.global_replace (Str.regexp "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}") "UUID" msg in

     

       350
       350
       +
           msg

     

       351
       351
       +
         

     

       352
       352
       +
         let hash_pattern pattern =

     

       353
       353
       +
           Digestif.SHA256.digest_string pattern |> Digestif.SHA256.to_hex

     

       354
       354
       +
         

     

       355
       355
       +
         let detect t =

     

       356
       356
       +
           ()

     

       357
       357
       +
         

     

       358
       358
       +
         let list t ?(min_count=1) ?(since=0.0) () =

     

       359
       359
       +
           []

     

       360
       360
       +
         

     

       361
       361
       +
         let mark_consulted t ~pattern_hash =

     

       362
       362
       +
           let sql = "UPDATE patterns SET last_consultation = ?1, consultation_count = consultation_count + 1 WHERE pattern_hash = ?2" in

     

       363
       363
       +
           let stmt = Sqlite3.prepare t.db sql in

     

       364
       364
       +
           let _ = Sqlite3.bind stmt 1 (Sqlite3.Data.FLOAT (Unix.gettimeofday ())) in

     

       365
       365
       +
           let _ = Sqlite3.bind stmt 2 (Sqlite3.Data.TEXT pattern_hash) in

     

       366
       366
       +
           let _ = Sqlite3.step stmt in

     

       367
       367
       +
           let _ = Sqlite3.finalize stmt in

     

       368
       368
       +
           ()

     

       369
       369
       +
       end

     

       370
       370
       +
       

     

       371
       371
       +
       module Query = struct

     

       372
       372
       +
         type filter = {

     

       373
       373
       +
           level : Logs.level option;

     

       374
       374
       +
           source : string option;

     

       375
       375
       +
           since : float option;

     

       376
       376
       +
           until : float option;

     

       377
       377
       +
           session_id : string option;

     

       378
       378
       +
           request_id : string option;

     

       379
       379
       +
           trace_id : string option;

     

       380
       380
       +
           user_id : string option;

     

       381
       381
       +
           error_type : string option;

     

       382
       382
       +
           min_duration_ms : float option;

     

       383
       383
       +
           max_duration_ms : float option;

     

       384
       384
       +
           limit : int;

     

       385
       385
       +
         }

     

       386
       386
       +
         

     

       387
       387
       +
         let default_filter = {

     

       388
       388
       +
           level = None;

     

       389
       389
       +
           source = None;

     

       390
       390
       +
           since = None;

     

       391
       391
       +
           until = None;

     

       392
       392
       +
           session_id = None;

     

       393
       393
       +
           request_id = None;

     

       394
       394
       +
           trace_id = None;

     

       395
       395
       +
           user_id = None;

     

       396
       396
       +
           error_type = None;

     

       397
       397
       +
           min_duration_ms = None;

     

       398
       398
       +
           max_duration_ms = None;

     

       399
       399
       +
           limit = 100;

     

       400
       400
       +
         }

     

       401
       401
       +
         

     

       402
       402
       +
         let list t filter =

     

       403
       403
       +
           []

     

       404
       404
       +
         

     

       405
       405
       +
         let search t query ?(limit=100) () =

     

       406
       406
       +
           let sql = Printf.sprintf "

     

       407
       407
       +
             SELECT logs.* FROM logs 

     

       408
       408
       +
             JOIN logs_fts ON logs.id = logs_fts.rowid

     

       409
       409
       +
             WHERE logs_fts MATCH ?1

     

       410
       410
       +
             ORDER BY timestamp DESC

     

       411
       411
       +
             LIMIT %d

     

       412
       412
       +
           " limit in

     

       413
       413
       +
           

     

       414
       414
       +
           let stmt = Sqlite3.prepare t.db sql in

     

       415
       415
       +
           let _ = Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT query) in

     

       416
       416
       +
           

     

       417
       417
       +
           let rec collect acc =

     

       418
       418
       +
             match Sqlite3.step stmt with

     

       419
       419
       +
             | Sqlite3.Rc.ROW -> collect (Sqlite3.row_data stmt :: acc)

     

       420
       420
       +
             | _ -> List.rev acc

     

       421
       421
       +
           in

     

       422
       422
       +
           

     

       423
       423
       +
           let results = collect [] in

     

       424
       424
       +
           let _ = Sqlite3.finalize stmt in

     

       425
       425
       +
           results

     

       426
       426
       +
         

     

       427
       427
       +
         let count_by_level t ?(since=0.0) () =

     

       428
       428
       +
           []

     

       429
       429
       +
         

     

       430
       430
       +
         let recent_errors t ?(limit=100) () =

     

       431
       431
       +
           let sql = Printf.sprintf "SELECT * FROM recent_errors LIMIT %d" limit in

     

       432
       432
       +
           let stmt = Sqlite3.prepare t.db sql in

     

       433
       433
       +
           

     

       434
       434
       +
           let rec collect acc =

     

       435
       435
       +
             match Sqlite3.step stmt with

     

       436
       436
       +
             | Sqlite3.Rc.ROW -> collect (Sqlite3.row_data stmt :: acc)

     

       437
       437
       +
             | _ -> List.rev acc

     

       438
       438
       +
           in

     

       439
       439
       +
           

     

       440
       440
       +
           let results = collect [] in

     

       441
       441
       +
           let _ = Sqlite3.finalize stmt in

     

       442
       442
       +
           results

     

       443
       443
       +
         

     

       444
       444
       +
         let slow_operations t ?(threshold_ms=100.0) () =

     

       445
       445
       +
           []

     

       446
       446
       +
         

     

       447
       447
       +
         let error_trends t ?(days=7) () =

     

       448
       448
       +
           []

     

       449
       449
       +
         

     

       450
       450
       +
         let session_summary t ~session_id =

     

       451
       451
       +
           []

     

       452
       452
       +
       end

     

       453
       453
       +
       

     

       454
       454
       +
       module Metrics = struct

     

       455
       455
       +
         type system_snapshot = {

     

       456
       456
       +
           timestamp : float;

     

       457
       457
       +
           cpu_usage_percent : float option;

     

       458
       458
       +
           memory_used_bytes : int option;

     

       459
       459
       +
           memory_available_bytes : int option;

     

       460
       460
       +
           disk_usage_percent : float option;

     

       461
       461
       +
           network_rx_bytes_per_sec : float option;

     

       462
       462
       +
           network_tx_bytes_per_sec : float option;

     

       463
       463
       +
           gc_minor_collections : int option;

     

       464
       464
       +
           gc_major_collections : int option;

     

       465
       465
       +
           gc_heap_words : int option;

     

       466
       466
       +
         }

     

       467
       467
       +
         

     

       468
       468
       +
         let snapshot () =

     

       469
       469
       +
           let gc_stat = Gc.stat () in

     

       470
       470
       +
           {

     

       471
       471
       +
             timestamp = Unix.gettimeofday ();

     

       472
       472
       +
             cpu_usage_percent = None;

     

       473
       473
       +
             memory_used_bytes = Some (gc_stat.Gc.heap_words * (Sys.word_size / 8));

     

       474
       474
       +
             memory_available_bytes = None;

     

       475
       475
       +
             disk_usage_percent = None;

     

       476
       476
       +
             network_rx_bytes_per_sec = None;

     

       477
       477
       +
             network_tx_bytes_per_sec = None;

     

       478
       478
       +
             gc_minor_collections = Some gc_stat.Gc.minor_collections;

     

       479
       479
       +
             gc_major_collections = Some gc_stat.Gc.major_collections;

     

       480
       480
       +
             gc_heap_words = Some gc_stat.Gc.heap_words;

     

       481
       481
       +
           }

     

       482
       482
       +
         

     

       483
       483
       +
         let record t snapshot =

     

       484
       484
       +
           ()

     

       485
       485
       +
         

     

       486
       486
       +
         let get_range t ~since ~until =

     

       487
       487
       +
           []

     

       488
       488
       +
       end

     

       489
       489
       +
       

     

       490
       490
       +
       module Export = struct

     

       491
       491
       +
         let to_json rows =

     

       492
       492
       +
           `List (List.map (fun row ->

     

       493
       493
       +
             `List (List.map (function

     

       494
       494
       +
               | Sqlite3.Data.NULL -> `Null

     

       495
       495
       +
               | Sqlite3.Data.INT i -> `Int (Int64.to_int i)

     

       496
       496
       +
               | Sqlite3.Data.FLOAT f -> `Float f

     

       497
       497
       +
               | Sqlite3.Data.TEXT s -> `String s

     

       498
       498
       +
               | Sqlite3.Data.BLOB b -> `String (Bytes.to_string b)

     

       499
       499
       +
             ) row)

     

       500
       500
       +
           ) rows)

     

       501
       501
       +
         

     

       502
       502
       +
         let to_csv rows =

     

       503
       503
       +
           ""

     

       504
       504
       +
         

     

       505
       505
       +
         let for_claude t ?(limit=1000) ?(context_size=50000) () =

     

       506
       506
       +
           let sql = Printf.sprintf "

     

       507
       507
       +
             SELECT timestamp, level, source, message, error_type, stack_trace

     

       508
       508
       +
             FROM logs

     

       509
       509
       +
             WHERE level <= 2

     

       510
       510
       +
             ORDER BY timestamp DESC

     

       511
       511
       +
             LIMIT %d

     

       512
       512
       +
           " limit in

     

       513
       513
       +
           

     

       514
       514
       +
           let stmt = Sqlite3.prepare t.db sql in

     

       515
       515
       +
           

     

       516
       516
       +
           let rec collect acc size =

     

       517
       517
       +
             if size > context_size then acc

     

       518
       518
       +
             else match Sqlite3.step stmt with

     

       519
       519
       +
             | Sqlite3.Rc.ROW ->

     

       520
       520
       +
                 let row = Sqlite3.row_data stmt in

     

       521
       521
       +
                 let text = String.concat " | " (List.map (function

     

       522
       522
       +
                   | Sqlite3.Data.NULL -> "NULL"

     

       523
       523
       +
                   | Sqlite3.Data.INT i -> Int64.to_string i

     

       524
       524
       +
                   | Sqlite3.Data.FLOAT f -> string_of_float f

     

       525
       525
       +
                   | Sqlite3.Data.TEXT s -> s

     

       526
       526
       +
                   | Sqlite3.Data.BLOB b -> Bytes.to_string b

     

       527
       527
       +
                 ) row) in

     

       528
       528
       +
                 collect (text :: acc) (size + String.length text)

     

       529
       529
       +
             | _ -> acc

     

       530
       530
       +
           in

     

       531
       531
       +
           

     

       532
       532
       +
           let lines = collect [] 0 in

     

       533
       533
       +
           let _ = Sqlite3.finalize stmt in

     

       534
       534
       +
           String.concat "\n" (List.rev lines)

     

       535
       535
       +
       end

     

       536
       536
       +
       

     

       537
       537
       +
       let vacuum t =

     

       538
       538
       +
         let _ = Sqlite3.exec t.db "VACUUM" in

     

       539
       539
       +
         ()

     

       540
       540
       +
       

     

       541
       541
       +
       let stats t =

     

       542
       542
       +
         []

     

       543
       543
       +
       

     

       544
       544
       +
       let prune t ~older_than ?(dry_run=false) () =

     

       545
       545
       +
         0

+235

dancer/lib/dancer_logs/dancer_logs.mli

···

       1
       1
       +
       (** Dancer Logs - Comprehensive structured logging with SQLite backend *)

     

       2
       2
       +
       

     

       3
       3
       +
       (** {1 Types} *)

     

       4
       4
       +
       

     

       5
       5
       +
       type log_context = {

     

       6
       6
       +
         session_id : string option;

     

       7
       7
       +
         request_id : string option;

     

       8
       8
       +
         trace_id : string option;

     

       9
       9
       +
         span_id : string option;

     

       10
       10
       +
         parent_span_id : string option;

     

       11
       11
       +
         user_id : string option;

     

       12
       12
       +
         tenant_id : string option;

     

       13
       13
       +
         fiber_id : string option;

     

       14
       14
       +
         domain_id : int option;

     

       15
       15
       +
         tags : string list;

     

       16
       16
       +
         labels : (string * string) list;

     

       17
       17
       +
         custom : (string * Yojson.Safe.t) list;

     

       18
       18
       +
       }

     

       19
       19
       +
       

     

       20
       20
       +
       val empty_context : log_context

     

       21
       21
       +
       

     

       22
       22
       +
       type source_location = {

     

       23
       23
       +
         file_path : string option;

     

       24
       24
       +
         line_number : int option;

     

       25
       25
       +
         column_number : int option;

     

       26
       26
       +
         function_name : string option;

     

       27
       27
       +
         module_path : string option;

     

       28
       28
       +
       }

     

       29
       29
       +
       

     

       30
       30
       +
       type performance_metrics = {

     

       31
       31
       +
         duration_ms : float option;

     

       32
       32
       +
         memory_before : int option;

     

       33
       33
       +
         memory_after : int option;

     

       34
       34
       +
         cpu_time_ms : float option;

     

       35
       35
       +
       }

     

       36
       36
       +
       

     

       37
       37
       +
       type network_context = {

     

       38
       38
       +
         client_ip : string option;

     

       39
       39
       +
         server_ip : string option;

     

       40
       40
       +
         host_name : string option;

     

       41
       41
       +
         port : int option;

     

       42
       42
       +
         protocol : string option;

     

       43
       43
       +
         method_ : string option;

     

       44
       44
       +
         path : string option;

     

       45
       45
       +
         status_code : int option;

     

       46
       46
       +
         bytes_sent : int option;

     

       47
       47
       +
         bytes_received : int option;

     

       48
       48
       +
       }

     

       49
       49
       +
       

     

       50
       50
       +
       type error_info = {

     

       51
       51
       +
         error_type : string option;

     

       52
       52
       +
         error_code : string option;

     

       53
       53
       +
         error_hash : string option;

     

       54
       54
       +
         stack_trace : string option;

     

       55
       55
       +
       }

     

       56
       56
       +
       

     

       57
       57
       +
       type system_info = {

     

       58
       58
       +
         os_name : string;

     

       59
       59
       +
         os_version : string;

     

       60
       60
       +
         ocaml_version : string;

     

       61
       61
       +
         app_version : string option;

     

       62
       62
       +
         environment : string option;

     

       63
       63
       +
         region : string option;

     

       64
       64
       +
         container_id : string option;

     

       65
       65
       +
       }

     

       66
       66
       +
       

     

       67
       67
       +
       (** {1 Database} *)

     

       68
       68
       +
       

     

       69
       69
       +
       type t

     

       70
       70
       +
       (** The database connection *)

     

       71
       71
       +
       

     

       72
       72
       +
       val init : ?path:string -> unit -> t

     

       73
       73
       +
       (** Initialize the database. Default path is "dancer_logs.db" *)

     

       74
       74
       +
       

     

       75
       75
       +
       val close : t -> unit

     

       76
       76
       +
       (** Close the database connection *)

     

       77
       77
       +
       

     

       78
       78
       +
       (** {1 Logging} *)

     

       79
       79
       +
       

     

       80
       80
       +
       val log :

     

       81
       81
       +
         t ->

     

       82
       82
       +
         level:Logs.level ->

     

       83
       83
       +
         source:string ->

     

       84
       84
       +
         message:string ->

     

       85
       85
       +
         ?context:log_context ->

     

       86
       86
       +
         ?location:source_location ->

     

       87
       87
       +
         ?performance:performance_metrics ->

     

       88
       88
       +
         ?network:network_context ->

     

       89
       89
       +
         ?error:error_info ->

     

       90
       90
       +
         unit ->

     

       91
       91
       +
         unit

     

       92
       92
       +
       (** Log a message with full context *)

     

       93
       93
       +
       

     

       94
       94
       +
       (** {1 Reporter} *)

     

       95
       95
       +
       

     

       96
       96
       +
       val reporter : t -> Logs.reporter

     

       97
       97
       +
       (** Create a Logs reporter that writes to the database *)

     

       98
       98
       +
       

     

       99
       99
       +
       val chain_reporter : t -> Logs.reporter -> Logs.reporter

     

       100
       100
       +
       (** Chain this reporter with an existing one *)

     

       101
       101
       +
       

     

       102
       102
       +
       (** {1 Context Management} *)

     

       103
       103
       +
       

     

       104
       104
       +
       module Context : sig

     

       105
       105
       +
         val with_session : string -> log_context -> log_context

     

       106
       106
       +
         val with_request : string -> log_context -> log_context

     

       107
       107
       +
         val with_trace : string -> log_context -> log_context

     

       108
       108
       +
         val with_span : string -> ?parent:string -> log_context -> log_context

     

       109
       109
       +
         val with_user : string -> log_context -> log_context

     

       110
       110
       +
         val with_tenant : string -> log_context -> log_context

     

       111
       111
       +
         val with_fiber : string -> log_context -> log_context

     

       112
       112
       +
         val with_domain : int -> log_context -> log_context

     

       113
       113
       +
         val add_tag : string -> log_context -> log_context

     

       114
       114
       +
         val add_label : string -> string -> log_context -> log_context

     

       115
       115
       +
         val add_custom : string -> Yojson.Safe.t -> log_context -> log_context

     

       116
       116
       +
         

     

       117
       117
       +
         (** Eio integration *)

     

       118
       118
       +
         val of_eio : Eio.Fiber.t -> log_context

     

       119
       119
       +
         val current_fiber_id : unit -> string option

     

       120
       120
       +
       end

     

       121
       121
       +
       

     

       122
       122
       +
       (** {1 Pattern Detection} *)

     

       123
       123
       +
       

     

       124
       124
       +
       module Pattern : sig

     

       125
       125
       +
         type t = {

     

       126
       126
       +
           id : int;

     

       127
       127
       +
           pattern_hash : string;

     

       128
       128
       +
           pattern : string;

     

       129
       129
       +
           first_seen : float;

     

       130
       130
       +
           last_seen : float;

     

       131
       131
       +
           occurrence_count : int;

     

       132
       132
       +
           severity_score : float option;

     

       133
       133
       +
         }

     

       134
       134
       +
         

     

       135
       135
       +
         val detect : t -> unit

     

       136
       136
       +
         (** Run pattern detection on recent logs *)

     

       137
       137
       +
         

     

       138
       138
       +
         val list : t -> ?min_count:int -> ?since:float -> unit -> t list

     

       139
       139
       +
         (** List detected patterns *)

     

       140
       140
       +
         

     

       141
       141
       +
         val mark_consulted : t -> pattern_hash:string -> unit

     

       142
       142
       +
         (** Mark a pattern as consulted by Claude *)

     

       143
       143
       +
       end

     

       144
       144
       +
       

     

       145
       145
       +
       (** {1 Queries} *)

     

       146
       146
       +
       

     

       147
       147
       +
       module Query : sig

     

       148
       148
       +
         type filter = {

     

       149
       149
       +
           level : Logs.level option;

     

       150
       150
       +
           source : string option;

     

       151
       151
       +
           since : float option;  (** Unix timestamp *)

     

       152
       152
       +
           until : float option;  (** Unix timestamp *)

     

       153
       153
       +
           session_id : string option;

     

       154
       154
       +
           request_id : string option;

     

       155
       155
       +
           trace_id : string option;

     

       156
       156
       +
           user_id : string option;

     

       157
       157
       +
           error_type : string option;

     

       158
       158
       +
           min_duration_ms : float option;

     

       159
       159
       +
           max_duration_ms : float option;

     

       160
       160
       +
           limit : int;

     

       161
       161
       +
         }

     

       162
       162
       +
         

     

       163
       163
       +
         val default_filter : filter

     

       164
       164
       +
         

     

       165
       165
       +
         val list : t -> filter -> Sqlite3.row list

     

       166
       166
       +
         (** List logs matching the filter *)

     

       167
       167
       +
         

     

       168
       168
       +
         val search : t -> string -> ?limit:int -> unit -> Sqlite3.row list

     

       169
       169
       +
         (** Full-text search *)

     

       170
       170
       +
         

     

       171
       171
       +
         val count_by_level : t -> ?since:float -> unit -> (Logs.level * int) list

     

       172
       172
       +
         (** Count logs by level *)

     

       173
       173
       +
         

     

       174
       174
       +
         val recent_errors : t -> ?limit:int -> unit -> Sqlite3.row list

     

       175
       175
       +
         (** Get recent errors with aggregation *)

     

       176
       176
       +
         

     

       177
       177
       +
         val slow_operations : t -> ?threshold_ms:float -> unit -> Sqlite3.row list

     

       178
       178
       +
         (** Find slow operations *)

     

       179
       179
       +
         

     

       180
       180
       +
         val error_trends : t -> ?days:int -> unit -> Sqlite3.row list

     

       181
       181
       +
         (** Get error trends over time *)

     

       182
       182
       +
         

     

       183
       183
       +
         val session_summary : t -> session_id:string -> Sqlite3.row list

     

       184
       184
       +
         (** Get summary for a session *)

     

       185
       185
       +
       end

     

       186
       186
       +
       

     

       187
       187
       +
       (** {1 Metrics} *)

     

       188
       188
       +
       

     

       189
       189
       +
       module Metrics : sig

     

       190
       190
       +
         type system_snapshot = {

     

       191
       191
       +
           timestamp : float;

     

       192
       192
       +
           cpu_usage_percent : float option;

     

       193
       193
       +
           memory_used_bytes : int option;

     

       194
       194
       +
           memory_available_bytes : int option;

     

       195
       195
       +
           disk_usage_percent : float option;

     

       196
       196
       +
           network_rx_bytes_per_sec : float option;

     

       197
       197
       +
           network_tx_bytes_per_sec : float option;

     

       198
       198
       +
           gc_minor_collections : int option;

     

       199
       199
       +
           gc_major_collections : int option;

     

       200
       200
       +
           gc_heap_words : int option;

     

       201
       201
       +
         }

     

       202
       202
       +
         

     

       203
       203
       +
         val snapshot : unit -> system_snapshot

     

       204
       204
       +
         (** Take a system metrics snapshot *)

     

       205
       205
       +
         

     

       206
       206
       +
         val record : t -> system_snapshot -> unit

     

       207
       207
       +
         (** Record metrics to database *)

     

       208
       208
       +
         

     

       209
       209
       +
         val get_range : t -> since:float -> until:float -> system_snapshot list

     

       210
       210
       +
         (** Get metrics in time range *)

     

       211
       211
       +
       end

     

       212
       212
       +
       

     

       213
       213
       +
       (** {1 Export} *)

     

       214
       214
       +
       

     

       215
       215
       +
       module Export : sig

     

       216
       216
       +
         val to_json : Sqlite3.row list -> Yojson.Safe.t

     

       217
       217
       +
         (** Export rows to JSON *)

     

       218
       218
       +
         

     

       219
       219
       +
         val to_csv : Sqlite3.row list -> string

     

       220
       220
       +
         (** Export rows to CSV *)

     

       221
       221
       +
         

     

       222
       222
       +
         val for_claude : t -> ?limit:int -> ?context_size:int -> unit -> string

     

       223
       223
       +
         (** Prepare context for Claude analysis *)

     

       224
       224
       +
       end

     

       225
       225
       +
       

     

       226
       226
       +
       (** {1 Maintenance} *)

     

       227
       227
       +
       

     

       228
       228
       +
       val vacuum : t -> unit

     

       229
       229
       +
       (** Optimize the database *)

     

       230
       230
       +
       

     

       231
       231
       +
       val stats : t -> (string * string) list

     

       232
       232
       +
       (** Get database statistics *)

     

       233
       233
       +
       

     

       234
       234
       +
       val prune : t -> older_than:float -> ?dry_run:bool -> unit -> int

     

       235
       235
       +
       (** Prune old logs. Returns number of deleted rows *)

dancer/lib/dancer_logs/dune

···

       1
       1
       +
       (library

     

       2
       2
       +
        (name dancer_logs)

     

       3
       3
       +
        (public_name dancer-logs)

     

       4
       4
       +
        (libraries eio eio_main logs sqlite3 fmt ptime ptime.clock.os digestif yojson str)

     

       5
       5
       +
        (preprocess (pps ppx_deriving.show ppx_deriving.eq)))

+316

dancer/lib/dancer_logs/schema.sql

···

       1
       1
       +
       -- Main log entries table with comprehensive structured fields

     

       2
       2
       +
       CREATE TABLE IF NOT EXISTS logs (

     

       3
       3
       +
           id INTEGER PRIMARY KEY AUTOINCREMENT,

     

       4
       4
       +
           

     

       5
       5
       +
           -- Core fields

     

       6
       6
       +
           timestamp REAL NOT NULL,              -- Unix timestamp with microseconds

     

       7
       7
       +
           level INTEGER NOT NULL,               -- 0=App, 1=Error, 2=Warning, 3=Info, 4=Debug

     

       8
       8
       +
           source TEXT NOT NULL,                 -- Module/source name (e.g., 'myapp.db.connection')

     

       9
       9
       +
           message TEXT NOT NULL,                -- Full log message, no truncation

     

       10
       10
       +
           

     

       11
       11
       +
           -- Error classification

     

       12
       12
       +
           error_type TEXT,                      -- Exception name or error classification

     

       13
       13
       +
           error_code TEXT,                      -- Specific error code if applicable

     

       14
       14
       +
           error_hash TEXT,                      -- Hash for grouping similar errors

     

       15
       15
       +
           stack_trace TEXT,                     -- Full stack trace

     

       16
       16
       +
           

     

       17
       17
       +
           -- Execution context

     

       18
       18
       +
           pid INTEGER,                          -- Process ID

     

       19
       19
       +
           ppid INTEGER,                         -- Parent process ID

     

       20
       20
       +
           thread_id TEXT,                       -- Thread/fiber ID for Eio

     

       21
       21
       +
           fiber_id TEXT,                        -- Specific Eio fiber ID

     

       22
       22
       +
           domain_id INTEGER,                    -- OCaml domain ID for multicore

     

       23
       23
       +
           

     

       24
       24
       +
           -- Request/Session tracking

     

       25
       25
       +
           session_id TEXT,                      -- Session identifier

     

       26
       26
       +
           request_id TEXT,                      -- Request identifier

     

       27
       27
       +
           trace_id TEXT,                        -- Distributed tracing ID

     

       28
       28
       +
           span_id TEXT,                         -- Span ID for tracing

     

       29
       29
       +
           parent_span_id TEXT,                  -- Parent span for trace hierarchy

     

       30
       30
       +
           

     

       31
       31
       +
           -- Source location

     

       32
       32
       +
           file_path TEXT,                       -- Full file path

     

       33
       33
       +
           file_name TEXT,                       -- Just the filename

     

       34
       34
       +
           line_number INTEGER,                  -- Line number in source

     

       35
       35
       +
           column_number INTEGER,                -- Column number if available

     

       36
       36
       +
           function_name TEXT,                   -- Function/method name

     

       37
       37
       +
           module_path TEXT,                     -- Full module path (e.g., 'Myapp.Db.Connection')

     

       38
       38
       +
           

     

       39
       39
       +
           -- Performance metrics

     

       40
       40
       +
           duration_ms REAL,                     -- Operation duration in milliseconds

     

       41
       41
       +
           memory_before INTEGER,                -- Memory usage before operation (bytes)

     

       42
       42
       +
           memory_after INTEGER,                 -- Memory usage after operation (bytes)

     

       43
       43
       +
           memory_delta INTEGER,                 -- Memory change

     

       44
       44
       +
           cpu_time_ms REAL,                     -- CPU time consumed

     

       45
       45
       +
           

     

       46
       46
       +
           -- Network context

     

       47
       47
       +
           client_ip TEXT,                       -- Client IP address

     

       48
       48
       +
           server_ip TEXT,                       -- Server IP address

     

       49
       49
       +
           host_name TEXT,                       -- Hostname

     

       50
       50
       +
           port INTEGER,                         -- Port number

     

       51
       51
       +
           protocol TEXT,                        -- Protocol (HTTP, gRPC, etc.)

     

       52
       52
       +
           method TEXT,                          -- HTTP method or RPC method

     

       53
       53
       +
           path TEXT,                            -- Request path or endpoint

     

       54
       54
       +
           status_code INTEGER,                  -- Response status code

     

       55
       55
       +
           bytes_sent INTEGER,                   -- Bytes sent

     

       56
       56
       +
           bytes_received INTEGER,               -- Bytes received

     

       57
       57
       +
           

     

       58
       58
       +
           -- User context

     

       59
       59
       +
           user_id TEXT,                         -- User identifier

     

       60
       60
       +
           tenant_id TEXT,                       -- Tenant ID for multi-tenant apps

     

       61
       61
       +
           organization_id TEXT,                 -- Organization ID

     

       62
       62
       +
           

     

       63
       63
       +
           -- System context

     

       64
       64
       +
           os_name TEXT,                         -- Operating system name

     

       65
       65
       +
           os_version TEXT,                      -- OS version

     

       66
       66
       +
           ocaml_version TEXT,                   -- OCaml compiler version

     

       67
       67
       +
           app_version TEXT,                     -- Application version

     

       68
       68
       +
           environment TEXT,                     -- Environment (dev, staging, prod)

     

       69
       69
       +
           region TEXT,                          -- Deployment region

     

       70
       70
       +
           container_id TEXT,                    -- Docker/container ID

     

       71
       71
       +
           k8s_pod TEXT,                         -- Kubernetes pod name

     

       72
       72
       +
           k8s_namespace TEXT,                   -- Kubernetes namespace

     

       73
       73
       +
           

     

       74
       74
       +
           -- Metadata fields (JSON for flexibility)

     

       75
       75
       +
           tags TEXT,                            -- JSON array of tags

     

       76
       76
       +
           labels TEXT,                          -- JSON object of key-value labels

     

       77
       77
       +
           context TEXT,                         -- JSON object with arbitrary context

     

       78
       78
       +
           custom_fields TEXT,                   -- JSON object for app-specific fields

     

       79
       79
       +
           

     

       80
       80
       +
           -- Audit fields

     

       81
       81
       +
           created_at REAL DEFAULT (julianday('now')),

     

       82
       82
       +
           indexed_at REAL                       -- When added to FTS index

     

       83
       83
       +
       );

     

       84
       84
       +
       

     

       85
       85
       +
       -- Full-text search index on all text fields

     

       86
       86
       +
       CREATE VIRTUAL TABLE IF NOT EXISTS logs_fts USING fts5(

     

       87
       87
       +
           message,

     

       88
       88
       +
           error_type,

     

       89
       89
       +
           error_code,

     

       90
       90
       +
           source,

     

       91
       91
       +
           file_path,

     

       92
       92
       +
           function_name,

     

       93
       93
       +
           module_path,

     

       94
       94
       +
           path,

     

       95
       95
       +
           method,

     

       96
       96
       +
           stack_trace,

     

       97
       97
       +
           tags,

     

       98
       98
       +
           labels,

     

       99
       99
       +
           context,

     

       100
       100
       +
           custom_fields,

     

       101
       101
       +
           content=logs,

     

       102
       102
       +
           content_rowid=id,

     

       103
       103
       +
           tokenize='porter unicode61'

     

       104
       104
       +
       );

     

       105
       105
       +
       

     

       106
       106
       +
       -- Pattern tracking for error analysis

     

       107
       107
       +
       CREATE TABLE IF NOT EXISTS patterns (

     

       108
       108
       +
           id INTEGER PRIMARY KEY AUTOINCREMENT,

     

       109
       109
       +
           pattern_hash TEXT UNIQUE NOT NULL,    -- Hash of normalized pattern

     

       110
       110
       +
           pattern TEXT NOT NULL,                -- Normalized pattern template

     

       111
       111
       +
           pattern_type TEXT,                    -- Type of pattern (error, warning, anomaly)

     

       112
       112
       +
           

     

       113
       113
       +
           -- Occurrence tracking

     

       114
       114
       +
           first_seen REAL NOT NULL,

     

       115
       115
       +
           last_seen REAL NOT NULL,

     

       116
       116
       +
           occurrence_count INTEGER DEFAULT 1,

     

       117
       117
       +
           unique_sources INTEGER DEFAULT 1,     -- Number of unique sources

     

       118
       118
       +
           unique_sessions INTEGER DEFAULT 1,    -- Number of unique sessions affected

     

       119
       119
       +
           

     

       120
       120
       +
           -- Statistical analysis

     

       121
       121
       +
           avg_duration_ms REAL,                 -- Average duration when this occurs

     

       122
       122
       +
           max_duration_ms REAL,                 -- Maximum duration observed

     

       123
       123
       +
           min_duration_ms REAL,                 -- Minimum duration observed

     

       124
       124
       +
           

     

       125
       125
       +
           -- Trend analysis

     

       126
       126
       +
           hourly_rate REAL,                     -- Occurrences per hour

     

       127
       127
       +
           daily_rate REAL,                      -- Occurrences per day

     

       128
       128
       +
           acceleration_rate REAL,               -- Rate of increase/decrease

     

       129
       129
       +
           

     

       130
       130
       +
           -- Claude consultation tracking

     

       131
       131
       +
           severity_score REAL,                  -- Calculated severity (0-10)

     

       132
       132
       +
           impact_score REAL,                    -- Business impact score

     

       133
       133
       +
           last_consultation REAL,               -- When Claude last reviewed

     

       134
       134
       +
           consultation_count INTEGER DEFAULT 0, -- Number of consultations

     

       135
       135
       +
           fix_attempted BOOLEAN DEFAULT 0,      -- Whether a fix was attempted

     

       136
       136
       +
           fix_successful BOOLEAN DEFAULT 0,     -- Whether the fix worked

     

       137
       137
       +
           

     

       138
       138
       +
           -- Metadata

     

       139
       139
       +
           sample_log_ids TEXT,                  -- JSON array of example log IDs

     

       140
       140
       +
           related_patterns TEXT,                -- JSON array of related pattern IDs

     

       141
       141
       +
           notes TEXT,                           -- Human or Claude notes

     

       142
       142
       +
           

     

       143
       143
       +
           created_at REAL DEFAULT (julianday('now')),

     

       144
       144
       +
           updated_at REAL DEFAULT (julianday('now'))

     

       145
       145
       +
       );

     

       146
       146
       +
       

     

       147
       147
       +
       -- Time-series data for pattern analysis

     

       148
       148
       +
       CREATE TABLE IF NOT EXISTS pattern_metrics (

     

       149
       149
       +
           pattern_id INTEGER REFERENCES patterns(id) ON DELETE CASCADE,

     

       150
       150
       +
           timestamp REAL NOT NULL,              -- Bucket timestamp (e.g., hour boundary)

     

       151
       151
       +
           bucket_size INTEGER NOT NULL,         -- Bucket size in seconds (3600 for hourly)

     

       152
       152
       +
           

     

       153
       153
       +
           -- Metrics for this time bucket

     

       154
       154
       +
           occurrence_count INTEGER DEFAULT 0,

     

       155
       155
       +
           unique_sources INTEGER DEFAULT 0,

     

       156
       156
       +
           unique_sessions INTEGER DEFAULT 0,

     

       157
       157
       +
           avg_duration_ms REAL,

     

       158
       158
       +
           max_duration_ms REAL,

     

       159
       159
       +
           error_rate REAL,                      -- Errors per second in this bucket

     

       160
       160
       +
           

     

       161
       161
       +
           PRIMARY KEY (pattern_id, timestamp, bucket_size)

     

       162
       162
       +
       );

     

       163
       163
       +
       

     

       164
       164
       +
       -- Session tracking for correlation

     

       165
       165
       +
       CREATE TABLE IF NOT EXISTS sessions (

     

       166
       166
       +
           session_id TEXT PRIMARY KEY,

     

       167
       167
       +
           start_time REAL NOT NULL,

     

       168
       168
       +
           end_time REAL,

     

       169
       169
       +
           duration_ms REAL,

     

       170
       170
       +
           

     

       171
       171
       +
           -- Session metadata

     

       172
       172
       +
           user_id TEXT,

     

       173
       173
       +
           tenant_id TEXT,

     

       174
       174
       +
           client_ip TEXT,

     

       175
       175
       +
           user_agent TEXT,

     

       176
       176
       +
           

     

       177
       177
       +
           -- Session metrics

     

       178
       178
       +
           log_count INTEGER DEFAULT 0,

     

       179
       179
       +
           error_count INTEGER DEFAULT 0,

     

       180
       180
       +
           warning_count INTEGER DEFAULT 0,

     

       181
       181
       +
           

     

       182
       182
       +
           -- Performance metrics

     

       183
       183
       +
           total_duration_ms REAL,

     

       184
       184
       +
           total_memory_bytes INTEGER,

     

       185
       185
       +
           

     

       186
       186
       +
           created_at REAL DEFAULT (julianday('now'))

     

       187
       187
       +
       );

     

       188
       188
       +
       

     

       189
       189
       +
       -- System metrics snapshot table

     

       190
       190
       +
       CREATE TABLE IF NOT EXISTS system_metrics (

     

       191
       191
       +
           timestamp REAL PRIMARY KEY,

     

       192
       192
       +
           

     

       193
       193
       +
           -- CPU metrics

     

       194
       194
       +
           cpu_usage_percent REAL,

     

       195
       195
       +
           cpu_user_percent REAL,

     

       196
       196
       +
           cpu_system_percent REAL,

     

       197
       197
       +
           cpu_idle_percent REAL,

     

       198
       198
       +
           load_avg_1m REAL,

     

       199
       199
       +
           load_avg_5m REAL,

     

       200
       200
       +
           load_avg_15m REAL,

     

       201
       201
       +
           

     

       202
       202
       +
           -- Memory metrics

     

       203
       203
       +
           memory_total_bytes INTEGER,

     

       204
       204
       +
           memory_used_bytes INTEGER,

     

       205
       205
       +
           memory_free_bytes INTEGER,

     

       206
       206
       +
           memory_available_bytes INTEGER,

     

       207
       207
       +
           swap_used_bytes INTEGER,

     

       208
       208
       +
           swap_free_bytes INTEGER,

     

       209
       209
       +
           

     

       210
       210
       +
           -- Disk metrics

     

       211
       211
       +
           disk_read_bytes_per_sec REAL,

     

       212
       212
       +
           disk_write_bytes_per_sec REAL,

     

       213
       213
       +
           disk_usage_percent REAL,

     

       214
       214
       +
           

     

       215
       215
       +
           -- Network metrics

     

       216
       216
       +
           network_rx_bytes_per_sec REAL,

     

       217
       217
       +
           network_tx_bytes_per_sec REAL,

     

       218
       218
       +
           network_connections INTEGER,

     

       219
       219
       +
           

     

       220
       220
       +
           -- Process metrics

     

       221
       221
       +
           process_count INTEGER,

     

       222
       222
       +
           thread_count INTEGER,

     

       223
       223
       +
           fiber_count INTEGER,

     

       224
       224
       +
           

     

       225
       225
       +
           -- GC metrics (OCaml specific)

     

       226
       226
       +
           gc_minor_collections INTEGER,

     

       227
       227
       +
           gc_major_collections INTEGER,

     

       228
       228
       +
           gc_heap_words INTEGER,

     

       229
       229
       +
           gc_live_words INTEGER,

     

       230
       230
       +
           

     

       231
       231
       +
           created_at REAL DEFAULT (julianday('now'))

     

       232
       232
       +
       );

     

       233
       233
       +
       

     

       234
       234
       +
       -- Comprehensive indexes for efficient querying

     

       235
       235
       +
       CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs(timestamp DESC);

     

       236
       236
       +
       CREATE INDEX IF NOT EXISTS idx_logs_level_timestamp ON logs(level, timestamp DESC);

     

       237
       237
       +
       CREATE INDEX IF NOT EXISTS idx_logs_source_timestamp ON logs(source, timestamp DESC);

     

       238
       238
       +
       CREATE INDEX IF NOT EXISTS idx_logs_error_hash ON logs(error_hash) WHERE error_hash IS NOT NULL;

     

       239
       239
       +
       CREATE INDEX IF NOT EXISTS idx_logs_session_id ON logs(session_id) WHERE session_id IS NOT NULL;

     

       240
       240
       +
       CREATE INDEX IF NOT EXISTS idx_logs_request_id ON logs(request_id) WHERE request_id IS NOT NULL;

     

       241
       241
       +
       CREATE INDEX IF NOT EXISTS idx_logs_trace_id ON logs(trace_id) WHERE trace_id IS NOT NULL;

     

       242
       242
       +
       CREATE INDEX IF NOT EXISTS idx_logs_user_id ON logs(user_id) WHERE user_id IS NOT NULL;

     

       243
       243
       +
       CREATE INDEX IF NOT EXISTS idx_logs_error_type ON logs(error_type) WHERE error_type IS NOT NULL;

     

       244
       244
       +
       CREATE INDEX IF NOT EXISTS idx_logs_duration ON logs(duration_ms) WHERE duration_ms IS NOT NULL;

     

       245
       245
       +
       CREATE INDEX IF NOT EXISTS idx_logs_status_code ON logs(status_code) WHERE status_code IS NOT NULL;

     

       246
       246
       +
       

     

       247
       247
       +
       CREATE INDEX IF NOT EXISTS idx_patterns_last_seen ON patterns(last_seen DESC);

     

       248
       248
       +
       CREATE INDEX IF NOT EXISTS idx_patterns_severity ON patterns(severity_score DESC) WHERE severity_score IS NOT NULL;

     

       249
       249
       +
       CREATE INDEX IF NOT EXISTS idx_patterns_count ON patterns(occurrence_count DESC);

     

       250
       250
       +
       CREATE INDEX IF NOT EXISTS idx_pattern_metrics_timestamp ON pattern_metrics(timestamp DESC);

     

       251
       251
       +
       

     

       252
       252
       +
       -- Triggers to maintain FTS and update timestamps

     

       253
       253
       +
       CREATE TRIGGER IF NOT EXISTS logs_fts_insert AFTER INSERT ON logs

     

       254
       254
       +
       BEGIN

     

       255
       255
       +
           INSERT INTO logs_fts(rowid, message, error_type, error_code, source, 

     

       256
       256
       +
                               file_path, function_name, module_path, path, method,

     

       257
       257
       +
                               stack_trace, tags, labels, context, custom_fields)

     

       258
       258
       +
           VALUES (new.id, new.message, new.error_type, new.error_code, new.source,

     

       259
       259
       +
                   new.file_path, new.function_name, new.module_path, new.path, new.method,

     

       260
       260
       +
                   new.stack_trace, new.tags, new.labels, new.context, new.custom_fields);

     

       261
       261
       +
           

     

       262
       262
       +
           UPDATE logs SET indexed_at = julianday('now') WHERE id = new.id;

     

       263
       263
       +
       END;

     

       264
       264
       +
       

     

       265
       265
       +
       CREATE TRIGGER IF NOT EXISTS patterns_update_timestamp AFTER UPDATE ON patterns

     

       266
       266
       +
       BEGIN

     

       267
       267
       +
           UPDATE patterns SET updated_at = julianday('now') WHERE id = new.id;

     

       268
       268
       +
       END;

     

       269
       269
       +
       

     

       270
       270
       +
       -- Views for common queries

     

       271
       271
       +
       CREATE VIEW IF NOT EXISTS recent_errors AS

     

       272
       272
       +
       SELECT 

     

       273
       273
       +
           error_hash,

     

       274
       274
       +
           error_type,

     

       275
       275
       +
           error_code,

     

       276
       276
       +
           COUNT(*) as count,

     

       277
       277
       +
           COUNT(DISTINCT session_id) as affected_sessions,

     

       278
       278
       +
           COUNT(DISTINCT user_id) as affected_users,

     

       279
       279
       +
           MAX(timestamp) as last_seen,

     

       280
       280
       +
           MIN(timestamp) as first_seen,

     

       281
       281
       +
           GROUP_CONCAT(DISTINCT source) as sources,

     

       282
       282
       +
           AVG(duration_ms) as avg_duration_ms,

     

       283
       283
       +
           message as sample_message

     

       284
       284
       +
       FROM logs

     

       285
       285
       +
       WHERE level <= 2  -- Error and Warning

     

       286
       286
       +
           AND timestamp > julianday('now') - 1  -- Last 24 hours

     

       287
       287
       +
       GROUP BY error_hash

     

       288
       288
       +
       ORDER BY count DESC;

     

       289
       289
       +
       

     

       290
       290
       +
       CREATE VIEW IF NOT EXISTS error_trends AS

     

       291
       291
       +
       SELECT 

     

       292
       292
       +
           date(timestamp) as day,

     

       293
       293
       +
           level,

     

       294
       294
       +
           COUNT(*) as count,

     

       295
       295
       +
           COUNT(DISTINCT error_hash) as unique_errors,

     

       296
       296
       +
           COUNT(DISTINCT session_id) as affected_sessions,

     

       297
       297
       +
           AVG(duration_ms) as avg_duration_ms

     

       298
       298
       +
       FROM logs

     

       299
       299
       +
       WHERE level <= 2

     

       300
       300
       +
       GROUP BY date(timestamp), level

     

       301
       301
       +
       ORDER BY day DESC, level;

     

       302
       302
       +
       

     

       303
       303
       +
       CREATE VIEW IF NOT EXISTS slow_operations AS

     

       304
       304
       +
       SELECT 

     

       305
       305
       +
           source,

     

       306
       306
       +
           function_name,

     

       307
       307
       +
           COUNT(*) as count,

     

       308
       308
       +
           AVG(duration_ms) as avg_duration_ms,

     

       309
       309
       +
           MAX(duration_ms) as max_duration_ms,

     

       310
       310
       +
           MIN(duration_ms) as min_duration_ms,

     

       311
       311
       +
           SUM(duration_ms) as total_duration_ms

     

       312
       312
       +
       FROM logs

     

       313
       313
       +
       WHERE duration_ms IS NOT NULL

     

       314
       314
       +
       GROUP BY source, function_name

     

       315
       315
       +
       HAVING avg_duration_ms > 100  -- Operations slower than 100ms

     

       316
       316
       +
       ORDER BY avg_duration_ms DESC;