commit 96a54b793439582cf265bd2c6bc544e820ae3e00 · anil.recoil.org/slop

rec-converter/.gitignore

···

       1
       1
       +
       _build

rec-converter/bin/dune

···

       1
       1
       +
       (executable

     

       2
       2
       +
        (public_name owntracks2clickhouse)

     

       3
       3
       +
        (name main)

     

       4
       4
       +
        (libraries owntracks_to_clickhouse cmdliner))

+106

rec-converter/bin/main.ml

···

       1
       1
       +
       open Cmdliner

     

       2
       2
       +
       open Owntracks_to_clickhouse

     

       3
       3
       +
       

     

       4
       4
       +
       let process_file input_file output_file =

     

       5
       5
       +
         try

     

       6
       6
       +
           let records = Owntracks_parser.parse_file input_file in

     

       7
       7
       +
           match output_file with

     

       8
       8
       +
           | Some out_path ->

     

       9
       9
       +
             Clickhouse_formatter.write_jsonl_file out_path records;

     

       10
       10
       +
             Printf.printf "Converted %d records from %s to %s\n" 

     

       11
       11
       +
               (List.length records) input_file out_path

     

       12
       12
       +
           | None ->

     

       13
       13
       +
             Clickhouse_formatter.print_jsonl records

     

       14
       14
       +
         with

     

       15
       15
       +
         | Sys_error msg ->

     

       16
       16
       +
           Printf.eprintf "Error: %s\n" msg;

     

       17
       17
       +
           exit 1

     

       18
       18
       +
         | e ->

     

       19
       19
       +
           Printf.eprintf "Unexpected error: %s\n" (Printexc.to_string e);

     

       20
       20
       +
           exit 1

     

       21
       21
       +
       

     

       22
       22
       +
       let process_directory dir_path output_file recursive =

     

       23
       23
       +
         let rec find_rec_files dir =

     

       24
       24
       +
           let entries = Sys.readdir dir in

     

       25
       25
       +
           Array.fold_left (fun acc entry ->

     

       26
       26
       +
             let full_path = Filename.concat dir entry in

     

       27
       27
       +
             if Sys.is_directory full_path then

     

       28
       28
       +
               if recursive then

     

       29
       29
       +
                 acc @ find_rec_files full_path

     

       30
       30
       +
               else

     

       31
       31
       +
                 acc

     

       32
       32
       +
             else if Filename.check_suffix entry ".rec" then

     

       33
       33
       +
               full_path :: acc

     

       34
       34
       +
             else

     

       35
       35
       +
               acc

     

       36
       36
       +
           ) [] entries

     

       37
       37
       +
         in

     

       38
       38
       +
         

     

       39
       39
       +
         let rec_files = find_rec_files dir_path in

     

       40
       40
       +
         let all_records = List.fold_left (fun acc file ->

     

       41
       41
       +
           try

     

       42
       42
       +
             let records = Owntracks_parser.parse_file file in

     

       43
       43
       +
             Printf.printf "Processed %s: %d records\n" file (List.length records);

     

       44
       44
       +
             acc @ records

     

       45
       45
       +
           with e ->

     

       46
       46
       +
             Printf.eprintf "Warning: Failed to process %s: %s\n" 

     

       47
       47
       +
               file (Printexc.to_string e);

     

       48
       48
       +
             acc

     

       49
       49
       +
         ) [] rec_files in

     

       50
       50
       +
         

     

       51
       51
       +
         match output_file with

     

       52
       52
       +
         | Some out_path ->

     

       53
       53
       +
           Clickhouse_formatter.write_jsonl_file out_path all_records;

     

       54
       54
       +
           Printf.printf "Total: Converted %d records from %d files to %s\n" 

     

       55
       55
       +
             (List.length all_records) (List.length rec_files) out_path

     

       56
       56
       +
         | None ->

     

       57
       57
       +
           Clickhouse_formatter.print_jsonl all_records

     

       58
       58
       +
       

     

       59
       59
       +
       let main input output recursive =

     

       60
       60
       +
         if Sys.is_directory input then

     

       61
       61
       +
           process_directory input output recursive

     

       62
       62
       +
         else

     

       63
       63
       +
           process_file input output

     

       64
       64
       +
       

     

       65
       65
       +
       let input_arg =

     

       66
       66
       +
         let doc = "Input OwnTracks .rec file or directory containing .rec files" in

     

       67
       67
       +
         Arg.(required & pos 0 (some string) None & info [] ~docv:"INPUT" ~doc)

     

       68
       68
       +
       

     

       69
       69
       +
       let output_arg =

     

       70
       70
       +
         let doc = "Output JSON lines file (if not specified, outputs to stdout)" in

     

       71
       71
       +
         Arg.(value & opt (some string) None & info ["o"; "output"] ~docv:"OUTPUT" ~doc)

     

       72
       72
       +
       

     

       73
       73
       +
       let recursive_arg =

     

       74
       74
       +
         let doc = "Recursively process directories for .rec files" in

     

       75
       75
       +
         Arg.(value & flag & info ["r"; "recursive"] ~doc)

     

       76
       76
       +
       

     

       77
       77
       +
       let main_term =

     

       78
       78
       +
         Term.(const main $ input_arg $ output_arg $ recursive_arg)

     

       79
       79
       +
       

     

       80
       80
       +
       let info =

     

       81
       81
       +
         let doc = "Convert OwnTracks .rec files to ClickHouse JSON lines" in

     

       82
       82
       +
         let man = [

     

       83
       83
       +
           `S Manpage.s_description;

     

       84
       84
       +
           `P "$(tname) converts OwnTracks recorder files (.rec) to JSON lines format suitable for importing into ClickHouse with geo data types.";

     

       85
       85
       +
           `P "Each location record is converted to a JSON object with the following fields:";

     

       86
       86
       +
           `P "- timestamp: ISO 8601 formatted timestamp";

     

       87
       87
       +
           `P "- timestamp_epoch: Unix timestamp";

     

       88
       88
       +
           `P "- point: [longitude, latitude] array for ClickHouse Point type";

     

       89
       89
       +
           `P "- latitude, longitude: Individual coordinate fields";

     

       90
       90
       +
           `P "- altitude, accuracy, battery: Optional fields from OwnTracks";

     

       91
       91
       +
           `P "- tracker_id: Device identifier from OwnTracks";

     

       92
       92
       +
           `S Manpage.s_examples;

     

       93
       93
       +
           `P "Convert a single file to stdout:";

     

       94
       94
       +
           `Pre "  $(tname) path/to/file.rec";

     

       95
       95
       +
           `P "Convert a single file to output file:";

     

       96
       96
       +
           `Pre "  $(tname) path/to/file.rec -o output.jsonl";

     

       97
       97
       +
           `P "Process all .rec files in a directory:";

     

       98
       98
       +
           `Pre "  $(tname) path/to/directory -o output.jsonl";

     

       99
       99
       +
           `P "Process directory recursively:";

     

       100
       100
       +
           `Pre "  $(tname) path/to/directory -r -o output.jsonl";

     

       101
       101
       +
         ] in

     

       102
       102
       +
         Cmd.info "owntracks2clickhouse" ~version:"1.0.0" ~doc ~man

     

       103
       103
       +
       

     

       104
       104
       +
       let cmd = Cmd.v info main_term

     

       105
       105
       +
       

     

       106
       106
       +
       let () = exit (Cmd.eval cmd)

+14

rec-converter/dune-project

···

       1
       1
       +
       (lang dune 3.0)

     

       2
       2
       +
       (name owntracks_to_clickhouse)

     

       3
       3
       +
       

     

       4
       4
       +
       (generate_opam_files true)

     

       5
       5
       +
       

     

       6
       6
       +
       (package

     

       7
       7
       +
        (name owntracks_to_clickhouse)

     

       8
       8
       +
        (synopsis "Convert OwnTracks .rec files to ClickHouse JSON lines")

     

       9
       9
       +
        (description "A library and CLI tool to convert OwnTracks recorder files to JSON lines suitable for ClickHouse import with geo data types")

     

       10
       10
       +
        (depends

     

       11
       11
       +
         ocaml

     

       12
       12
       +
         dune

     

       13
       13
       +
         ezjsonm

     

       14
       14
       +
         cmdliner))

+54

rec-converter/example.sh

···

       1
       1
       +
       #!/bin/bash

     

       2
       2
       +
       

     

       3
       3
       +
       # Build the project

     

       4
       4
       +
       dune build

     

       5
       5
       +
       

     

       6
       6
       +
       # Example 1: Convert a single .rec file to stdout

     

       7
       7
       +
       echo "Converting single file to stdout:"

     

       8
       8
       +
       dune exec owntracks2clickhouse avsm/avsm-ip15/2025-08.rec | head -5

     

       9
       9
       +
       

     

       10
       10
       +
       # Example 2: Convert a single .rec file to output file

     

       11
       11
       +
       echo -e "\nConverting single file to output.jsonl:"

     

       12
       12
       +
       dune exec owntracks2clickhouse avsm/avsm-ip15/2025-08.rec -o single_output.jsonl

     

       13
       13
       +
       echo "Created single_output.jsonl"

     

       14
       14
       +
       

     

       15
       15
       +
       # Example 3: Process all .rec files in a directory recursively

     

       16
       16
       +
       echo -e "\nProcessing all .rec files recursively:"

     

       17
       17
       +
       dune exec owntracks2clickhouse avsm -r -o all_records.jsonl

     

       18
       18
       +
       

     

       19
       19
       +
       # Example 4: Create ClickHouse table and import data

     

       20
       20
       +
       cat << 'EOF'

     

       21
       21
       +
       

     

       22
       22
       +
       To import into ClickHouse, create a table like this:

     

       23
       23
       +
       

     

       24
       24
       +
       CREATE TABLE owntracks_locations (

     

       25
       25
       +
           timestamp DateTime64(3),

     

       26
       26
       +
           timestamp_epoch UInt32,

     

       27
       27
       +
           point Point,

     

       28
       28
       +
           latitude Float64,

     

       29
       29
       +
           longitude Float64,

     

       30
       30
       +
           altitude Nullable(Float64),

     

       31
       31
       +
           accuracy Nullable(Float64),

     

       32
       32
       +
           battery Nullable(UInt8),

     

       33
       33
       +
           tracker_id Nullable(String)

     

       34
       34
       +
       ) ENGINE = MergeTree()

     

       35
       35
       +
       ORDER BY (tracker_id, timestamp);

     

       36
       36
       +
       

     

       37
       37
       +
       Then import the JSON lines file:

     

       38
       38
       +
       

     

       39
       39
       +
       clickhouse-client --query="INSERT INTO owntracks_locations FORMAT JSONEachRow" < all_records.jsonl

     

       40
       40
       +
       

     

       41
       41
       +
       Or using clickhouse-local for testing:

     

       42
       42
       +
       

     

       43
       43
       +
       clickhouse-local --query="

     

       44
       44
       +
           SELECT 

     

       45
       45
       +
               tracker_id,

     

       46
       46
       +
               toDate(timestamp) as date,

     

       47
       47
       +
               count() as points,

     

       48
       48
       +
               round(avg(battery), 2) as avg_battery

     

       49
       49
       +
           FROM file('all_records.jsonl', 'JSONEachRow')

     

       50
       50
       +
           GROUP BY tracker_id, date

     

       51
       51
       +
           ORDER BY date DESC

     

       52
       52
       +
           LIMIT 10

     

       53
       53
       +
       "

     

       54
       54
       +
       EOF

+43

rec-converter/lib/clickhouse_formatter.ml

···

       1
       1
       +
       open Owntracks_parser

     

       2
       2
       +
       

     

       3
       3
       +
       let option_to_json = function

     

       4
       4
       +
         | Some v -> v

     

       5
       5
       +
         | None -> `Null

     

       6
       6
       +
       

     

       7
       7
       +
       let location_to_clickhouse_json record =

     

       8
       8
       +
         let point = `A [`Float record.lon; `Float record.lat] in

     

       9
       9
       +
         let timestamp_epoch = record.tst in

     

       10
       10
       +
         let timestamp_iso = record.timestamp in

     

       11
       11
       +
         

     

       12
       12
       +
         `O [

     

       13
       13
       +
           ("timestamp", `String timestamp_iso);

     

       14
       14
       +
           ("timestamp_epoch", `Float (float_of_int timestamp_epoch));

     

       15
       15
       +
           ("point", point);

     

       16
       16
       +
           ("latitude", `Float record.lat);

     

       17
       17
       +
           ("longitude", `Float record.lon);

     

       18
       18
       +
           ("altitude", option_to_json (Option.map (fun x -> `Float x) record.alt));

     

       19
       19
       +
           ("accuracy", option_to_json (Option.map (fun x -> `Float x) record.acc));

     

       20
       20
       +
           ("battery", option_to_json (Option.map (fun x -> `Float (float_of_int x)) record.batt));

     

       21
       21
       +
           ("tracker_id", option_to_json (Option.map (fun x -> `String x) record.tid));

     

       22
       22
       +
         ]

     

       23
       23
       +
       

     

       24
       24
       +
       let to_jsonl records =

     

       25
       25
       +
         List.map (fun record ->

     

       26
       26
       +
           let json = location_to_clickhouse_json record in

     

       27
       27
       +
           Ezjsonm.to_string json

     

       28
       28
       +
         ) records

     

       29
       29
       +
       

     

       30
       30
       +
       let write_jsonl_file path records =

     

       31
       31
       +
         let oc = open_out path in

     

       32
       32
       +
         List.iter (fun record ->

     

       33
       33
       +
           let json = location_to_clickhouse_json record in

     

       34
       34
       +
           output_string oc (Ezjsonm.to_string json);

     

       35
       35
       +
           output_char oc '\n'

     

       36
       36
       +
         ) records;

     

       37
       37
       +
         close_out oc

     

       38
       38
       +
       

     

       39
       39
       +
       let print_jsonl records =

     

       40
       40
       +
         List.iter (fun record ->

     

       41
       41
       +
           let json = location_to_clickhouse_json record in

     

       42
       42
       +
           print_endline (Ezjsonm.to_string json)

     

       43
       43
       +
         ) records

rec-converter/lib/dune

···

       1
       1
       +
       (library

     

       2
       2
       +
        (public_name owntracks_to_clickhouse)

     

       3
       3
       +
        (name owntracks_to_clickhouse)

     

       4
       4
       +
        (libraries ezjsonm))

+78

rec-converter/lib/owntracks_parser.ml

···

       1
       1
       +
       type location_record = {

     

       2
       2
       +
         timestamp: string;

     

       3
       3
       +
         lat: float;

     

       4
       4
       +
         lon: float;

     

       5
       5
       +
         alt: float option;

     

       6
       6
       +
         acc: float option;

     

       7
       7
       +
         batt: int option;

     

       8
       8
       +
         tid: string option;

     

       9
       9
       +
         tst: int;

     

       10
       10
       +
       }

     

       11
       11
       +
       

     

       12
       12
       +
       let parse_timestamp line =

     

       13
       13
       +
         match String.split_on_char '\t' line with

     

       14
       14
       +
         | timestamp :: _ -> Some timestamp

     

       15
       15
       +
         | [] -> None

     

       16
       16
       +
       

     

       17
       17
       +
       let parse_json_payload line =

     

       18
       18
       +
         match String.split_on_char '\t' line with

     

       19
       19
       +
         | _ :: _ :: json :: _ ->

     

       20
       20
       +
           (try Some (Ezjsonm.from_string json) with _ -> None)

     

       21
       21
       +
         | _ -> None

     

       22
       22
       +
       

     

       23
       23
       +
       let get_string json key =

     

       24
       24
       +
         try Some (Ezjsonm.get_string (Ezjsonm.find json [key]))

     

       25
       25
       +
         with _ -> None

     

       26
       26
       +
       

     

       27
       27
       +
       let get_float json key =

     

       28
       28
       +
         try Some (Ezjsonm.get_float (Ezjsonm.find json [key]))

     

       29
       29
       +
         with _ -> None

     

       30
       30
       +
       

     

       31
       31
       +
       let get_int json key =

     

       32
       32
       +
         try Some (Ezjsonm.get_int (Ezjsonm.find json [key]))

     

       33
       33
       +
         with _ -> None

     

       34
       34
       +
       

     

       35
       35
       +
       let extract_location_from_json json =

     

       36
       36
       +
         try

     

       37
       37
       +
           let dict = Ezjsonm.get_dict json in

     

       38
       38
       +
           match List.assoc_opt "_type" dict with

     

       39
       39
       +
           | Some (`String "location") ->

     

       40
       40
       +
             let lat = Ezjsonm.get_float (List.assoc "lat" dict) in

     

       41
       41
       +
             let lon = Ezjsonm.get_float (List.assoc "lon" dict) in

     

       42
       42
       +
             let tst = Ezjsonm.get_int (List.assoc "tst" dict) in

     

       43
       43
       +
             Some {

     

       44
       44
       +
               timestamp = "";

     

       45
       45
       +
               lat;

     

       46
       46
       +
               lon;

     

       47
       47
       +
               alt = get_float json "alt";

     

       48
       48
       +
               acc = get_float json "acc";

     

       49
       49
       +
               batt = get_int json "batt";

     

       50
       50
       +
               tid = get_string json "tid";

     

       51
       51
       +
               tst;

     

       52
       52
       +
             }

     

       53
       53
       +
           | _ -> None

     

       54
       54
       +
         with _ -> None

     

       55
       55
       +
       

     

       56
       56
       +
       let parse_line line =

     

       57
       57
       +
         match parse_timestamp line, parse_json_payload line with

     

       58
       58
       +
         | Some timestamp, Some json ->

     

       59
       59
       +
           (match extract_location_from_json json with

     

       60
       60
       +
            | Some record -> Some { record with timestamp }

     

       61
       61
       +
            | None -> None)

     

       62
       62
       +
         | _ -> None

     

       63
       63
       +
       

     

       64
       64
       +
       let parse_file path =

     

       65
       65
       +
         let ic = open_in path in

     

       66
       66
       +
         let rec read_lines acc =

     

       67
       67
       +
           try

     

       68
       68
       +
             let line = input_line ic in

     

       69
       69
       +
             let acc' = match parse_line line with

     

       70
       70
       +
               | Some record -> record :: acc

     

       71
       71
       +
               | None -> acc

     

       72
       72
       +
             in

     

       73
       73
       +
             read_lines acc'

     

       74
       74
       +
           with End_of_file ->

     

       75
       75
       +
             close_in ic;

     

       76
       76
       +
             List.rev acc

     

       77
       77
       +
         in

     

       78
       78
       +
         read_lines []

+26

rec-converter/owntracks_to_clickhouse.opam

···

       1
       1
       +
       # This file is generated by dune, edit dune-project instead

     

       2
       2
       +
       opam-version: "2.0"

     

       3
       3
       +
       synopsis: "Convert OwnTracks .rec files to ClickHouse JSON lines"

     

       4
       4
       +
       description:

     

       5
       5
       +
         "A library and CLI tool to convert OwnTracks recorder files to JSON lines suitable for ClickHouse import with geo data types"

     

       6
       6
       +
       depends: [

     

       7
       7
       +
         "ocaml"

     

       8
       8
       +
         "dune" {>= "3.0"}

     

       9
       9
       +
         "ezjsonm"

     

       10
       10
       +
         "cmdliner"

     

       11
       11
       +
         "odoc" {with-doc}

     

       12
       12
       +
       ]

     

       13
       13
       +
       build: [

     

       14
       14
       +
         ["dune" "subst"] {dev}

     

       15
       15
       +
         [

     

       16
       16
       +
           "dune"

     

       17
       17
       +
           "build"

     

       18
       18
       +
           "-p"

     

       19
       19
       +
           name

     

       20
       20
       +
           "-j"

     

       21
       21
       +
           jobs

     

       22
       22
       +
           "@install"

     

       23
       23
       +
           "@runtest" {with-test}

     

       24
       24
       +
           "@doc" {with-doc}

     

       25
       25
       +
         ]

     

       26
       26
       +
       ]