Geotessera library for OCaml

Initial commit

Patrick Ferris a033d7f0

+2
.gitignore
···
+
_build
+
+6
.ocamlformat
···
+
version=0.28.1
+
profile = conventional
+
+
ocaml-version = 4.08.0
+
break-infix = fit-or-vertical
+
parse-docstrings = true
+2
dune-project
···
+
(lang dune 3.17)
+
+3
example/dune
···
+
(executable
+
(name main)
+
(libraries mirage-crypto-rng.unix eio_main geotessera))
+20
example/main.ml
···
+
module Gt = Geotessera
+
+
let belfast =
+
Gt.Bbox.v ~min_lat:54.48 ~min_lon:(-6.11) ~max_lat:54.66 ~max_lon:(-5.78)
+
+
let pp_result fmt (crs, (point : Gt.point), emb) =
+
Fmt.pf fmt "Embedding at (%.2f, %.2f) with CRS:%i: %a" point.lat point.lon crs
+
Nx.pp_shape (Nx.shape emb)
+
+
let () =
+
Mirage_crypto_rng_unix.use_default ();
+
Eio_main.run @@ fun env ->
+
let embeddings = Gt.fetch env ~year:2024 belfast in
+
Eio.traceln "Dequantizing embeddings...";
+
let dequantize =
+
List.map (fun (crs, p, _lm, e) -> (crs, p, Gt.scale e)) embeddings
+
in
+
Eio.traceln "Embeddings: %a"
+
Fmt.(list ~sep:(Fmt.any "\n") pp_result)
+
dequantize
+44
geotessera.opam
···
+
# This file is generated by dune, edit dune-project instead
+
opam-version: "2.0"
+
version: "dev"
+
synopsis: "A short synopsis"
+
description: "A longer description"
+
maintainer: ["Maintainer Name <maintainer@example.com>"]
+
authors: ["Author Name <author@example.com>"]
+
license: "LICENSE"
+
tags: ["add topics" "to describe" "your" "project"]
+
homepage: "https://github.com/username/reponame"
+
doc: "https://url/to/documentation"
+
bug-reports: "https://github.com/username/reponame/issues"
+
depends: [
+
"dune" {>= "3.19"}
+
"ocaml"
+
"tiff"
+
"cohttp-eio"
+
"xdg"
+
"camlzip"
+
"nx" {>= "1.0.0~alpha2"}
+
"progress"
+
"ca-certs"
+
"tls-eio"
+
"odoc" {with-doc}
+
]
+
build: [
+
["dune" "subst"] {dev}
+
[
+
"dune"
+
"build"
+
"-p"
+
name
+
"-j"
+
jobs
+
"@install"
+
"@runtest" {with-test}
+
"@doc" {with-doc}
+
]
+
]
+
dev-repo: "git+https://github.com/username/reponame.git"
+
x-maintenance-intent: ["(latest)"]
+
pin-depends:[
+
[ "tiff.dev" "git+https://github.com/geocaml/ocaml-tiff#a64bc028f1a09452ccefd93632ac59a3488d279b" ]
+
]
+59
src/client.ml
···
+
open Cohttp_eio
+
+
type t = Cohttp_eio.Client.t
+
+
let authenticator =
+
match Ca_certs.authenticator () with
+
| Ok x -> x
+
| Error (`Msg m) ->
+
Fmt.failwith "Failed to create system store X509 authenticator: %s" m
+
+
let https ~authenticator =
+
let tls_config =
+
match Tls.Config.client ~authenticator () with
+
| Error (`Msg msg) -> failwith ("tls configuration problem: " ^ msg)
+
| Ok tls_config -> tls_config
+
in
+
fun uri raw ->
+
let host =
+
Uri.host uri
+
|> Option.map (fun x -> Domain_name.(host_exn (of_string_exn x)))
+
in
+
Tls_eio.client_of_flow ?host tls_config raw
+
+
let v net = Client.make ~https:(Some (https ~authenticator)) net
+
+
module Progress = struct
+
type t = { flow : Eio.Flow.source_ty Eio.Resource.t; progress : int -> unit }
+
+
let read_methods = []
+
+
let single_read (t : t) buf =
+
let i = Eio.Flow.single_read t.flow buf in
+
t.progress i;
+
i
+
end
+
+
let progress_handler = Eio.Flow.Pi.source (module Progress)
+
+
let with_progress ~progress flow =
+
Eio.Resource.T (Progress.{ flow; progress }, progress_handler)
+
+
let with_body ?(show = true) t display ~default ~uri fn =
+
Eio.Switch.run @@ fun sw ->
+
let uri = Uri.of_string uri in
+
let response, body = Client.get t uri ~sw in
+
match response.status with
+
| `OK ->
+
let total = Http.Response.content_length response |> Option.get in
+
let bar =
+
Display.line
+
~total:(Optint.Int63.of_int total)
+
(Uri.path uri |> Filename.basename)
+
in
+
Display.with_line ~display ~show (fun _ -> bar) @@ fun r ->
+
let progress = Display.report_int r in
+
fn (with_progress ~progress body)
+
| s ->
+
Eio.traceln "%a: %a" Uri.pp uri Http.Status.pp s;
+
default
+120
src/display.ml
···
+
module Int63 = Optint.Int63
+
+
(* FIXME: the None type is probably not needed with switch cancellation *)
+
type t = {
+
stream : (unit -> unit) option Eio.Stream.t;
+
display : ((unit -> unit) -> unit, unit) Progress.Display.t;
+
}
+
+
type line = Int63.t Progress.Line.t
+
+
type reporter = {
+
stream : (unit -> unit) option Eio.Stream.t;
+
reporter : Int63.t Progress.Reporter.t option;
+
}
+
+
let report r i =
+
match r.reporter with
+
| None -> ()
+
| Some reporter ->
+
Eio.Stream.add r.stream
+
(Some (fun () -> Progress.Reporter.report reporter i))
+
+
let report_int r i = report r (Int63.of_int i)
+
+
let line ~color ~total message =
+
let message = String.sub message 0 (min 21 (String.length message)) in
+
let open Progress.Line.Using_int63 in
+
list
+
[
+
rpad 22 (const message);
+
bytes;
+
bytes_per_sec;
+
bar ~color ~style:`UTF8 total;
+
percentage_of total ++ const " ";
+
]
+
+
let colors =
+
let a =
+
[
+
"#1996f3";
+
"#06aeed";
+
"#10c6e6";
+
"#27dade";
+
"#3dead5";
+
"#52f5cb";
+
"#66fcc2";
+
"#7dffb6";
+
"#92fda9";
+
"#a8f79c";
+
"#bced8f";
+
"#d2de81";
+
"#e8cb72";
+
"#feb562";
+
"#ff9b52";
+
"#ff8143";
+
"#ff6232";
+
"#ff4121";
+
]
+
in
+
Array.map Progress.Color.hex (Array.of_list (a @ List.rev a))
+
+
let next_color i = colors.(i mod Array.length colors)
+
+
let line ~total file =
+
let color = next_color (Random.int (Array.length colors)) in
+
line ~color ~total file
+
+
let rec apply_stream ~sw stream =
+
Eio.Switch.check sw;
+
match Eio.Stream.take stream with
+
| Some f ->
+
f ();
+
apply_stream ~sw stream
+
| None -> ()
+
+
let init ?platform ~sw image : t =
+
let image_name =
+
Progress.Line.(
+
spacer 4
+
++ constf "🐫 Fetching %a" Fmt.(styled `Bold string) image
+
++
+
match platform with
+
| None -> const ""
+
| Some p -> constf "%a" Fmt.(styled `Faint (brackets string)) p)
+
in
+
let stream = Eio.Stream.create max_int in
+
let display = Progress.Display.start Progress.Multi.(line image_name) in
+
Eio.Fiber.fork ~sw (fun () -> apply_stream ~sw stream);
+
{ stream; display }
+
+
let rec empty_stream stream =
+
match Eio.Stream.take_nonblocking stream with
+
| None | Some None -> ()
+
| Some (Some f) ->
+
f ();
+
empty_stream stream
+
+
let finalise { stream; display } =
+
Eio.Stream.add stream None;
+
empty_stream stream;
+
Progress.Display.finalise display
+
+
let lines = ref 0
+
+
let with_line ~display ?(show = true) bar f =
+
let reporter =
+
if show then (
+
let r = Progress.Display.add_line display.display (bar !lines) in
+
incr lines;
+
Some r)
+
else None
+
in
+
let finally () =
+
match reporter with
+
| None -> ()
+
| Some r ->
+
Eio.Stream.add display.stream
+
(Some (fun () -> Progress.Reporter.finalise r))
+
in
+
Fun.protect ~finally (fun () -> f { reporter; stream = display.stream })
+4
src/dune
···
+
(library
+
(name geotessera)
+
(public_name geotessera)
+
(libraries tiff eio nx progress cohttp-eio ca-certs tls-eio xdg camlzip))
+333
src/geotessera.ml
···
+
open Eio
+
+
let ( / ) = Eio.Path.( / )
+
+
type point = { lat : float; lon : float }
+
+
let pp_point fmt { lat; lon } = Fmt.pf fmt "{ lat: %.3f, lon: %.3f }" lat lon
+
+
module Bbox = struct
+
type t = float array
+
+
let v ~min_lon ~min_lat ~max_lon ~max_lat =
+
[| min_lat; min_lon; max_lat; max_lon |]
+
+
let min_lon t = t.(1)
+
let min_lat t = t.(0)
+
let max_lon t = t.(3)
+
let max_lat t = t.(2)
+
end
+
+
module Registry = struct
+
type t = {
+
git_url : string;
+
data_url : string;
+
git : Eio.Fs.dir_ty Eio.Path.t;
+
data : Eio.Fs.dir_ty Eio.Path.t;
+
client : Client.t;
+
version : [ `v1 ];
+
}
+
+
let version_to_string = function `v1 -> "v1"
+
let block_size = 5.
+
+
let block_of_point point =
+
{
+
lon = Float.floor (point.lon /. block_size) *. block_size;
+
lat = Float.floor (point.lat /. block_size) *. block_size;
+
}
+
+
let blocks_for_region (bbox : Bbox.t) =
+
let min_block_lon = floor (Bbox.min_lon bbox /. block_size) *. block_size in
+
let min_block_lat = floor (Bbox.min_lat bbox /. block_size) *. block_size in
+
let max_block_lon = floor (Bbox.max_lon bbox /. block_size) *. block_size in
+
let max_block_lat = floor (Bbox.max_lat bbox /. block_size) *. block_size in
+
+
let blocks = ref [] in
+
+
let lon = ref min_block_lon in
+
while !lon <= max_block_lon do
+
let lat = ref min_block_lat in
+
while !lat <= max_block_lat do
+
(* Blocks are referenced by their centre *)
+
blocks := { lat = !lat +. 2.5; lon = !lon +. 2.5 } :: !blocks;
+
lat := !lat +. block_size
+
done;
+
lon := !lon +. block_size
+
done;
+
+
List.rev !blocks
+
+
let clone env ~into url =
+
Eio.Process.run env#process_mgr [ "git"; "clone"; url; into ]
+
+
let with_registry ?(git_url = "https://github.com/ucam-eo/tessera-manifests")
+
?(data_url = "https://dl-2.tessera.wiki") ?(version = `v1) env fn =
+
let t =
+
Xdg.create
+
~env:(fun s -> try Some (Unix.getenv s) with Not_found -> None)
+
()
+
in
+
let cache = Xdg.cache_dir t in
+
let dir = env#fs / cache / "ocaml-geotessera" in
+
let git = dir / "tessera-manifests" in
+
let data = dir / "downloads" in
+
let client = Client.v env#net in
+
let v = { git_url; data_url; git; client; data; version } in
+
(match Eio.Path.kind ~follow:false git with
+
| `Directory -> ()
+
| `Not_found ->
+
Eio.Path.mkdirs ~perm:0o755 data;
+
clone env ~into:(Eio.Path.native_exn git) v.git_url;
+
()
+
| _ -> Fmt.failwith "%a is not a directory or non-existent" Eio.Path.pp git);
+
fn v
+
+
let with_display ~sw label fn =
+
let display = ref None in
+
let mux = Mutex.create () in
+
let rec get_display () =
+
match !display with
+
| Some d -> d
+
| None ->
+
Mutex.use_rw ~protect:true mux (fun () ->
+
display := Some (Display.init ~sw label));
+
get_display ()
+
in
+
let finalise_display () = Option.iter Display.finalise !display in
+
Fun.protect ~finally:finalise_display (fun () -> fn get_display)
+
+
let extract_lat_lon_from_grid_name s =
+
match String.split_on_char '_' (Filename.basename s) with
+
| [ "grid"; lon; lat_with_ext ] ->
+
let lat = Filename.chop_extension lat_with_ext in
+
{ lon = float_of_string lon; lat = float_of_string lat }
+
| [ "grid"; lon; lat; "scales.npy" ] ->
+
{ lon = float_of_string lon; lat = float_of_string lat }
+
| _ -> Fmt.invalid_arg "Failed to extract lat/lon from %s" s
+
+
let parse_manifest path =
+
let module R = Eio.Buf_read in
+
Eio.Path.with_open_in path @@ fun f ->
+
let r = R.of_flow ~max_size:max_int f in
+
let lines = R.lines r in
+
Seq.map (String.split_on_char ' ') lines
+
|> Seq.map (function
+
| [ s; h ] ->
+
(extract_lat_lon_from_grid_name s, s, Digestif.SHA256.of_hex h)
+
| _ -> failwith "Malformed manifest")
+
|> List.of_seq
+
+
let find_manifest t ~year point =
+
let b = block_of_point point in
+
let name =
+
Fmt.str "embeddings_%i_lon%i_lat%i.txt" year (Float.to_int b.lon)
+
(Float.to_int b.lat)
+
in
+
let path = t.git / "registry" / "embeddings" / name in
+
parse_manifest path
+
+
let find_landmasks _t point =
+
let name = Fmt.str "grid_%.2f_%.2f.tiff" point.lon point.lat in
+
(point, name, Digestif.SHA256.empty)
+
+
let download_embedding t v =
+
Eio.Switch.run @@ fun sw ->
+
with_display ~sw "embeddings" @@ fun get_display ->
+
Eio.traceln "Downloading and checking embeddings...";
+
let paths =
+
Fiber.List.map ~max_fibers:10
+
(fun (_point, name, _hash) ->
+
let uri =
+
Fmt.str "%s/%s/global_0.1_degree_representation/%s" t.data_url
+
(version_to_string t.version)
+
name
+
in
+
let data_dir = t.data / "embeddings" / Filename.dirname name in
+
Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 data_dir;
+
let path = data_dir / Filename.basename name in
+
let download =
+
try
+
let stat = Eio.Path.stat ~follow:false path in
+
if Optint.Int63.(equal stat.size zero) then `Delete_and_download
+
else `Check_hash stat.kind
+
with Eio.Exn.Io (Eio.Fs.E (Not_found _), _) -> `Download
+
in
+
match download with
+
| `Check_hash `Regular_file ->
+
let _disk_hash =
+
Digestif.SHA256.digest_string @@ Eio.Path.load path
+
in
+
(* assert (Digestif.SHA256.equal disk_hash hash); *)
+
path
+
| `Delete_and_download | `Download ->
+
let () =
+
match download with
+
| `Delete_and_download -> Eio.Path.unlink path
+
| _ -> ()
+
in
+
let display = get_display () in
+
let () =
+
Eio.Path.with_open_out ~create:(`If_missing 0o644) path
+
@@ fun w ->
+
Client.with_body ~default:() t.client display ~uri
+
@@ fun body -> Flow.copy body w
+
in
+
path
+
| `Check_hash _ -> Fmt.failwith "%s exists but is not a file!" name)
+
v
+
in
+
List.fold_left
+
(fun (points, emb, sca) p ->
+
let point = extract_lat_lon_from_grid_name (Eio.Path.native_exn p) in
+
match String.split_on_char '_' (Eio.Path.native_exn p) |> List.rev with
+
| "scales.npy" :: _ -> (points, emb, p :: sca)
+
| _ -> (point :: points, p :: emb, sca))
+
([], [], []) paths
+
|> fun (ps, s, t) -> List.combine ps (List.combine s t)
+
+
let crs_of_landmask lm =
+
Eio.Path.with_open_in lm @@ fun r ->
+
let ro = Eio.File.pread_exact r in
+
let tiff = Tiff.from_file Tiff.Float32 ro in
+
let geos = Tiff.ifd tiff |> Tiff.Ifd.geo_key_directory in
+
Tiff.Ifd.GeoKeys.projected_crs geos
+
+
let download_landmasks t v =
+
Eio.Switch.run @@ fun sw ->
+
with_display ~sw "landmasks" @@ fun get_display ->
+
Eio.traceln "Downloading and checking landmasks...";
+
let paths =
+
Fiber.List.map ~max_fibers:10
+
(fun (_point, name, _hash) ->
+
let uri =
+
Fmt.str "%s/%s/global_0.1_degree_tiff_all/%s" t.data_url
+
(version_to_string t.version)
+
name
+
in
+
let data_dir = t.data / "landmasks" / Filename.dirname name in
+
Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 data_dir;
+
let path = data_dir / Filename.basename name in
+
let download =
+
try
+
let stat = Eio.Path.stat ~follow:false path in
+
if Optint.Int63.(equal stat.size zero) then `Delete_and_download
+
else `Check_hash stat.kind
+
with Eio.Exn.Io (Eio.Fs.E (Not_found _), _) -> `Download
+
in
+
match download with
+
| `Check_hash `Regular_file ->
+
let _disk_hash =
+
Digestif.SHA256.digest_string @@ Eio.Path.load path
+
in
+
(* Eio.traceln "%a\n%a" Digestif.SHA256.pp disk_hash *)
+
(* Digestif.SHA256.pp hash; *)
+
(* assert (Digestif.SHA256.equal disk_hash hash); *)
+
let crs = crs_of_landmask path in
+
(crs, path)
+
| `Delete_and_download | `Download ->
+
let () =
+
match download with
+
| `Delete_and_download -> Eio.Path.unlink path
+
| _ -> ()
+
in
+
let display = get_display () in
+
let () =
+
Eio.Path.with_open_out ~create:(`If_missing 0o644) path
+
@@ fun w ->
+
Client.with_body ~default:() t.client display ~uri
+
@@ fun body -> Flow.copy body w
+
in
+
let crs = crs_of_landmask path in
+
(crs, path)
+
| `Check_hash _ -> Fmt.failwith "%s exists but is not a file!" name)
+
v
+
in
+
List.fold_left
+
(fun acc (crs, p) ->
+
let point = extract_lat_lon_from_grid_name (Eio.Path.native_exn p) in
+
(point, crs, p) :: acc)
+
[] paths
+
|> List.rev
+
end
+
+
type 'a env =
+
< fs : Eio.Fs.dir_ty Eio.Path.t
+
; process_mgr : [> `Generic ] Eio.Process.mgr_ty Eio.Process.mgr
+
; net : [> `Generic ] Eio.Net.ty Eio.Net.t
+
; .. >
+
as
+
'a
+
+
type embedding = (int, Bigarray.int8_signed_elt) Nx.t
+
type scales = (float, Bigarray.float32_elt) Nx.t
+
+
let fetch_embedding registry ~year (bbox : Bbox.t) =
+
let blocks = Registry.blocks_for_region bbox in
+
Fiber.List.map
+
(fun point ->
+
let manifest = Registry.find_manifest registry ~year point in
+
+
(* We need to check the tiles available are within the bbox supplied. *)
+
let manifest =
+
List.filter
+
(fun (p, _, _) ->
+
(* Tiles span 0.1 degree *)
+
let tile_min_lon = p.lon -. 0.05 and tile_max_lon = p.lon +. 0.05 in
+
let tile_min_lat = p.lat -. 0.05 and tile_max_lat = p.lat +. 0.05 in
+
tile_min_lon < Bbox.max_lon bbox
+
&& tile_min_lat < Bbox.max_lat bbox
+
&& tile_max_lat > Bbox.min_lat bbox
+
&& tile_max_lon > Bbox.min_lon bbox)
+
manifest
+
in
+
+
let downloads = Registry.download_embedding registry manifest in
+
List.map
+
(fun (point, (emb, scale)) ->
+
let emb =
+
Npy.read_copy (Eio.Path.native_exn emb)
+
|> Npy.to_bigarray C_layout Bigarray.Int8_signed
+
|> Option.get
+
|> Nx.of_bigarray
+
in
+
let scales =
+
Npy.read_copy (Eio.Path.native_exn scale)
+
|> Npy.to_bigarray C_layout Bigarray.Float32
+
|> Option.get
+
|> Nx.of_bigarray
+
in
+
(point, (emb, scales)))
+
downloads)
+
blocks
+
|> List.concat
+
+
let fetch_landmask registry points =
+
let landmask_files = List.map (Registry.find_landmasks registry) points in
+
Registry.download_landmasks registry landmask_files
+
+
let scale (emb, sca) =
+
match (Array.length (Nx.dims emb), Array.length (Nx.dims sca)) with
+
| 3, 2 ->
+
let dim0 = Nx.dim 0 sca and dim1 = Nx.dim 1 sca in
+
let bscales =
+
Nx.broadcast_to (Nx.dims emb) (Nx.reshape [| dim0; dim1; 1 |] sca)
+
in
+
Nx.mul (Nx.cast Nx.float32 emb) bscales
+
| 3, 3 -> Nx.mul (Nx.cast Nx.float32 emb) sca
+
| a, b -> Fmt.invalid_arg "Wrong dimensions %i %i" a b
+
+
let check_points p1 p2 =
+
if p1 <> p2 then
+
Fmt.invalid_arg "Different points: %a and %a" pp_point p1 pp_point p2
+
+
let fetch (env : _ env) ~year (bbox : Bbox.t) =
+
Registry.with_registry env @@ fun registry ->
+
let embs = fetch_embedding registry ~year bbox in
+
Eio.traceln "Depickling and scaling embeddings...";
+
let landmask = fetch_landmask registry (List.map fst embs) in
+
List.map2
+
(fun (p1, emb) (p2, crs, lm) ->
+
check_points p1 p2;
+
(crs, p1, lm, emb))
+
embs landmask
+43
src/geotessera.mli
···
+
(* {1 Tessera bindings for OCaml}
+
+
Access and use the Tessera bindings directly in OCaml. *)
+
+
module Bbox : sig
+
type t
+
(** A bounding box *)
+
+
val v : min_lon:float -> min_lat:float -> max_lon:float -> max_lat:float -> t
+
(** Construct a bounding box *)
+
end
+
+
type point = { lat : float; lon : float }
+
(** Points *)
+
+
type 'a env =
+
< fs : Eio.Fs.dir_ty Eio.Path.t
+
; process_mgr : [> `Generic ] Eio.Process.mgr_ty Eio.Process.mgr
+
; net : [> `Generic ] Eio.Net.ty Eio.Net.t
+
; .. >
+
as
+
'a
+
(** The required environment for {! fetch}ing embeddings. Filesystem access is
+
needed for caching the downloads, network access for fetching the downloads
+
and the process manager is needed for running git commands. *)
+
+
type embedding = (int, Bigarray.int8_signed_elt) Nx.t
+
(** Quantised embeddings *)
+
+
type scales = (float, Bigarray.float32_elt) Nx.t
+
(** Scales for embeddings *)
+
+
val fetch :
+
_ env ->
+
year:int ->
+
Bbox.t ->
+
(int * point * Eio.Fs.dir_ty Eio.Path.t * (embedding * scales)) list
+
(** [fetch env ~year bbox] returns a list of
+
[(crs, centre, landmask_path, scaled_embeddings)] for the tiles available in
+
[bbox] for [year]. *)
+
+
val scale : embedding * scales -> (float, Bigarray.float32_elt) Nx.t
+
(** Scale up the embeddings *)
+693
src/npy.ml
···
+
(* See the end of the file for the license *)
+
exception Cannot_write
+
exception Read_error of string
+
+
let read_error fmt = Printf.ksprintf (fun s -> raise (Read_error s)) fmt
+
let magic_string = "\147NUMPY"
+
let magic_string_len = String.length magic_string
+
+
type packed_kind = P : (_, _) Bigarray.kind -> packed_kind
+
+
let dtype ~packed_kind =
+
let endianness =
+
match packed_kind with
+
| P Bigarray.Char -> "|"
+
| P _ -> if Sys.big_endian then ">" else "<"
+
in
+
let kind =
+
match packed_kind with
+
| P Bigarray.Int32 -> "i4"
+
| P Bigarray.Int64 -> "i8"
+
| P Bigarray.Float16 -> "f16"
+
| P Bigarray.Float32 -> "f4"
+
| P Bigarray.Float64 -> "f8"
+
| P Bigarray.Int8_unsigned -> "u1"
+
| P Bigarray.Int8_signed -> "i1"
+
| P Bigarray.Int16_unsigned -> "u2"
+
| P Bigarray.Int16_signed -> "i2"
+
| P Bigarray.Char -> "S1"
+
| P Bigarray.Complex32 -> "c8" (* 2 32bits float. *)
+
| P Bigarray.Complex64 -> "c16" (* 2 64bits float. *)
+
| P Bigarray.Int -> failwith "Int is not supported"
+
| P Bigarray.Nativeint -> failwith "Nativeint is not supported."
+
in
+
endianness ^ kind
+
+
let map_file file_descr ~pos kind layout shared shape =
+
let is_scalar = Array.length shape = 0 in
+
let array =
+
Unix.map_file file_descr ~pos kind layout shared
+
(if is_scalar then [| 1 |] else shape)
+
in
+
if is_scalar then Bigarray.reshape array [||] else array
+
+
let fortran_order (type a) ~(layout : a Bigarray.layout) =
+
match layout with
+
| Bigarray.C_layout -> "False"
+
| Bigarray.Fortran_layout -> "True"
+
+
let shape ~dims =
+
match dims with
+
| [| dim1 |] -> Printf.sprintf "%d," dim1
+
| dims -> Array.to_list dims |> List.map string_of_int |> String.concat ", "
+
+
let full_header ?header_len ~layout ~packed_kind ~dims () =
+
let header =
+
Printf.sprintf "{'descr': '%s', 'fortran_order': %s, 'shape': (%s), }"
+
(dtype ~packed_kind) (fortran_order ~layout) (shape ~dims)
+
in
+
let padding_len =
+
let total_len = String.length header + magic_string_len + 4 + 1 in
+
match header_len with
+
| None -> if total_len mod 16 = 0 then 0 else 16 - (total_len mod 16)
+
| Some header_len ->
+
if header_len mod 16 <> 0 then
+
failwith "header_len has to be divisible by 16";
+
if header_len < total_len then
+
failwith "header_len is smaller than total_len";
+
header_len - total_len
+
in
+
let total_header_len = String.length header + padding_len + 1 in
+
Printf.sprintf "%s\001\000%c%c%s%s\n" magic_string
+
(total_header_len mod 256 |> Char.chr)
+
(total_header_len / 256 |> Char.chr)
+
header
+
(String.make padding_len ' ')
+
+
let with_file filename flags mask ~f =
+
let file_descr = Unix.openfile filename flags mask in
+
try
+
let result = f file_descr in
+
Unix.close file_descr;
+
result
+
with exn ->
+
Unix.close file_descr;
+
raise exn
+
+
let write ?header_len bigarray filename =
+
with_file filename [ O_CREAT; O_TRUNC; O_RDWR ] 0o640 ~f:(fun file_descr ->
+
let full_header =
+
full_header () ?header_len
+
~layout:(Bigarray.Genarray.layout bigarray)
+
~packed_kind:(P (Bigarray.Genarray.kind bigarray))
+
~dims:(Bigarray.Genarray.dims bigarray)
+
in
+
let full_header_len = String.length full_header in
+
if
+
Unix.write_substring file_descr full_header 0 full_header_len
+
<> full_header_len
+
then raise Cannot_write;
+
let file_array =
+
map_file
+
~pos:(Int64.of_int full_header_len)
+
file_descr
+
(Bigarray.Genarray.kind bigarray)
+
(Bigarray.Genarray.layout bigarray)
+
true
+
(Bigarray.Genarray.dims bigarray)
+
in
+
Bigarray.Genarray.blit bigarray file_array)
+
+
let write1 array1 filename = write (Bigarray.genarray_of_array1 array1) filename
+
let write2 array2 filename = write (Bigarray.genarray_of_array2 array2) filename
+
let write3 array3 filename = write (Bigarray.genarray_of_array3 array3) filename
+
+
module Batch_writer = struct
+
let header_len = 128
+
+
type t = {
+
file_descr : Unix.file_descr;
+
mutable bytes_written_so_far : int;
+
mutable dims_and_packed_kind : (int array * packed_kind) option;
+
}
+
+
let append t bigarray =
+
let file_array =
+
map_file
+
~pos:(Int64.of_int t.bytes_written_so_far)
+
t.file_descr
+
(Bigarray.Genarray.kind bigarray)
+
(Bigarray.Genarray.layout bigarray)
+
true
+
(Bigarray.Genarray.dims bigarray)
+
in
+
Bigarray.Genarray.blit bigarray file_array;
+
let size_in_bytes = Bigarray.Genarray.size_in_bytes bigarray in
+
t.bytes_written_so_far <- t.bytes_written_so_far + size_in_bytes;
+
match t.dims_and_packed_kind with
+
| None ->
+
let dims = Bigarray.Genarray.dims bigarray in
+
let kind = Bigarray.Genarray.kind bigarray in
+
t.dims_and_packed_kind <- Some (dims, P kind)
+
| Some (dims, _kind) ->
+
let dims' = Bigarray.Genarray.dims bigarray in
+
let incorrect_dimensions =
+
match (Array.to_list dims, Array.to_list dims') with
+
| [], _ | _, [] -> true
+
| _ :: d, _ :: d' -> d <> d'
+
in
+
if incorrect_dimensions then
+
Printf.sprintf "Incorrect dimensions %s vs %s." (shape ~dims)
+
(shape ~dims:dims')
+
|> failwith;
+
dims.(0) <- dims.(0) + dims'.(0)
+
+
let create filename =
+
let file_descr =
+
Unix.openfile filename [ O_CREAT; O_TRUNC; O_RDWR ] 0o640
+
in
+
{
+
file_descr;
+
bytes_written_so_far = header_len;
+
dims_and_packed_kind = None;
+
}
+
+
let close t =
+
assert (Unix.lseek t.file_descr 0 SEEK_SET = 0);
+
let header =
+
match t.dims_and_packed_kind with
+
| None -> failwith "Nothing to write"
+
| Some (dims, packed_kind) ->
+
full_header ~header_len ~layout:C_layout ~dims ~packed_kind ()
+
in
+
if Unix.write_substring t.file_descr header 0 header_len <> header_len then
+
raise Cannot_write;
+
Unix.close t.file_descr
+
end
+
+
let really_read fd len =
+
let buffer = Bytes.create len in
+
let rec loop offset =
+
let read = Unix.read fd buffer offset (len - offset) in
+
if read + offset < len then loop (read + offset)
+
else if read = 0 then read_error "unexpected eof"
+
in
+
loop 0;
+
Bytes.to_string buffer
+
+
module Header = struct
+
type packed_kind = P : (_, _) Bigarray.kind -> packed_kind
+
type t = { kind : packed_kind; fortran_order : bool; shape : int array }
+
+
let split str ~on =
+
let parens = ref 0 in
+
let indexes = ref [] in
+
for i = 0 to String.length str - 1 do
+
match str.[i] with
+
| '(' -> incr parens
+
| ')' -> decr parens
+
| c when !parens = 0 && c = on -> indexes := i :: !indexes
+
| _ -> ()
+
done;
+
List.fold_left
+
(fun (prev_p, acc) index ->
+
(index, String.sub str (index + 1) (prev_p - index - 1) :: acc))
+
(String.length str, [])
+
!indexes
+
|> fun (first_pos, acc) -> String.sub str 0 first_pos :: acc
+
+
let trim str ~on =
+
let rec loopr start len =
+
if len = 0 then (start, len)
+
else if List.mem str.[start + len - 1] on then loopr start (len - 1)
+
else (start, len)
+
in
+
let rec loopl start len =
+
if len = 0 then (start, len)
+
else if List.mem str.[start] on then loopl (start + 1) (len - 1)
+
else loopr start len
+
in
+
let start, len = loopl 0 (String.length str) in
+
String.sub str start len
+
+
let parse header =
+
let header_fields =
+
trim header ~on:[ '{'; ' '; '}'; '\n' ]
+
|> split ~on:','
+
|> List.map String.trim
+
|> List.filter (fun s -> String.length s > 0)
+
|> List.map (fun header_field ->
+
match split header_field ~on:':' with
+
| [ name; value ] ->
+
( trim name ~on:[ '\''; ' ' ],
+
trim value ~on:[ '\''; ' '; '('; ')' ] )
+
| _ -> read_error "unable to parse field %s" header_field)
+
in
+
let find_field field =
+
try List.assoc field header_fields
+
with Not_found -> read_error "cannot find field %s" field
+
in
+
let kind =
+
let kind = find_field "descr" in
+
(match kind.[0] with
+
| '|' | '=' -> ()
+
| '>' ->
+
if not Sys.big_endian then
+
read_error "big endian data but arch is little endian"
+
| '<' ->
+
if Sys.big_endian then
+
read_error "little endian data but arch is big endian"
+
| otherwise -> read_error "incorrect endianness %c" otherwise);
+
match String.sub kind 1 (String.length kind - 1) with
+
| "f2" -> P Float16
+
| "f4" -> P Float32
+
| "f8" -> P Float64
+
| "i4" -> P Int32
+
| "i8" -> P Int64
+
| "u1" -> P Int8_unsigned
+
| "i1" -> P Int8_signed
+
| "u2" -> P Int16_unsigned
+
| "i2" -> P Int16_signed
+
| "S1" -> P Char
+
| "c8" -> P Complex32
+
| "c16" -> P Complex64
+
| otherwise -> read_error "incorrect descr %s" otherwise
+
in
+
let fortran_order =
+
match find_field "fortran_order" with
+
| "False" -> false
+
| "True" -> true
+
| otherwise -> read_error "incorrect fortran_order %s" otherwise
+
in
+
let shape =
+
find_field "shape"
+
|> split ~on:','
+
|> List.map String.trim
+
|> List.filter (fun s -> String.length s > 0)
+
|> List.map int_of_string
+
|> Array.of_list
+
in
+
{ kind; fortran_order; shape }
+
end
+
+
type packed_array = P : (_, _, _) Bigarray.Genarray.t -> packed_array
+
type packed_array1 = P1 : (_, _, _) Bigarray.Array1.t -> packed_array1
+
type packed_array2 = P2 : (_, _, _) Bigarray.Array2.t -> packed_array2
+
type packed_array3 = P3 : (_, _, _) Bigarray.Array3.t -> packed_array3
+
+
let read_mmap filename ~shared =
+
let access = if shared then Unix.O_RDWR else O_RDONLY in
+
let file_descr = Unix.openfile filename [ access ] 0 in
+
let pos, header =
+
try
+
let magic_string' = really_read file_descr magic_string_len in
+
if magic_string <> magic_string' then read_error "magic string mismatch";
+
let version = really_read file_descr 2 |> fun v -> v.[0] |> Char.code in
+
let header_len_len =
+
match version with
+
| 1 -> 2
+
| 2 -> 4
+
| _ -> read_error "unsupported version %d" version
+
in
+
let header, header_len =
+
really_read file_descr header_len_len |> fun str ->
+
let header_len = ref 0 in
+
for i = String.length str - 1 downto 0 do
+
header_len := (256 * !header_len) + Char.code str.[i]
+
done;
+
(really_read file_descr !header_len, !header_len)
+
in
+
let header = Header.parse header in
+
(Int64.of_int (header_len + header_len_len + magic_string_len + 2), header)
+
with exn ->
+
Unix.close file_descr;
+
raise exn
+
in
+
let (Header.P kind) = header.kind in
+
let build layout =
+
let array = map_file file_descr ~pos kind layout shared header.shape in
+
Gc.finalise (fun _ -> Unix.close file_descr) array;
+
P array
+
in
+
if header.fortran_order then build Fortran_layout else build C_layout
+
+
let read_mmap1 filename ~shared =
+
let (P array) = read_mmap filename ~shared in
+
P1 (Bigarray.array1_of_genarray array)
+
+
let read_mmap2 filename ~shared =
+
let (P array) = read_mmap filename ~shared in
+
P2 (Bigarray.array2_of_genarray array)
+
+
let read_mmap3 filename ~shared =
+
let (P array) = read_mmap filename ~shared in
+
P3 (Bigarray.array3_of_genarray array)
+
+
let read_copy filename =
+
let (P array) = read_mmap filename ~shared:false in
+
let result =
+
Bigarray.Genarray.create
+
(Bigarray.Genarray.kind array)
+
(Bigarray.Genarray.layout array)
+
(Bigarray.Genarray.dims array)
+
in
+
Bigarray.Genarray.blit array result;
+
P result
+
+
let read_copy1 filename =
+
let (P array) = read_copy filename in
+
P1 (Bigarray.array1_of_genarray array)
+
+
let read_copy2 filename =
+
let (P array) = read_copy filename in
+
P2 (Bigarray.array2_of_genarray array)
+
+
let read_copy3 filename =
+
let (P array) = read_copy filename in
+
P3 (Bigarray.array3_of_genarray array)
+
+
module Npz = struct
+
let npy_suffix = ".npy"
+
+
let maybe_add_suffix array_name ~suffix =
+
let suffix =
+
match suffix with None -> npy_suffix | Some suffix -> suffix
+
in
+
array_name ^ suffix
+
+
type in_file = Zip.in_file
+
+
let open_in = Zip.open_in
+
+
let entries t =
+
Zip.entries t
+
|> List.map (fun entry ->
+
let filename = entry.Zip.filename in
+
if String.length filename < String.length npy_suffix then filename
+
else
+
let start_pos = String.length filename - String.length npy_suffix in
+
if
+
String.sub filename start_pos (String.length npy_suffix)
+
= npy_suffix
+
then String.sub filename 0 start_pos
+
else filename)
+
+
let close_in = Zip.close_in
+
+
let read ?suffix t array_name =
+
let array_name = maybe_add_suffix array_name ~suffix in
+
let entry =
+
try Zip.find_entry t array_name
+
with Not_found ->
+
raise (Invalid_argument ("unable to find " ^ array_name))
+
in
+
let tmp_file = Filename.temp_file "ocaml-npz" ".tmp" in
+
Zip.copy_entry_to_file t entry tmp_file;
+
let data = read_copy tmp_file in
+
Sys.remove tmp_file;
+
data
+
+
type out_file = Zip.out_file
+
+
let open_out filename = Zip.open_out filename
+
let close_out = Zip.close_out
+
+
let write ?suffix t array_name array =
+
let array_name = maybe_add_suffix array_name ~suffix in
+
let tmp_file = Filename.temp_file "ocaml-npz" ".tmp" in
+
write array tmp_file;
+
Zip.copy_file_to_entry tmp_file t array_name;
+
Sys.remove tmp_file
+
end
+
+
(** Type equalities module, used in conversion function *)
+
module Eq = struct
+
(** An equality type to extract type equalities *)
+
type ('a, 'b) t = W : ('a, 'a) t
+
+
open Bigarray
+
+
(** Type equalities for bigarray kinds *)
+
module Kind = struct
+
let ( === ) : type a b c d.
+
(a, b) kind -> (c, d) kind -> ((a, b) kind, (c, d) kind) t option =
+
fun x y ->
+
match (x, y) with
+
| Float32, Float32 -> Some W
+
| Float64, Float64 -> Some W
+
| Int8_signed, Int8_signed -> Some W
+
| Int8_unsigned, Int8_unsigned -> Some W
+
| Int16_signed, Int16_signed -> Some W
+
| Int16_unsigned, Int16_unsigned -> Some W
+
| Int32, Int32 -> Some W
+
| Int64, Int64 -> Some W
+
| Int, Int -> Some W
+
| Nativeint, Nativeint -> Some W
+
| Complex32, Complex32 -> Some W
+
| Complex64, Complex64 -> Some W
+
| Char, Char -> Some W
+
| _ -> None
+
end
+
+
(** Type equalities for layout *)
+
module Layout = struct
+
let ( === ) : type a b.
+
a layout -> b layout -> (a layout, b layout) t option =
+
fun x y ->
+
match (x, y) with
+
| Fortran_layout, Fortran_layout -> Some W
+
| C_layout, C_layout -> Some W
+
| _, _ -> None
+
end
+
end
+
+
(** Conversion functions from packed arrays to bigarrays *)
+
+
let to_bigarray (type a b c) (layout : c Bigarray.layout)
+
(kind : (a, b) Bigarray.kind) (P x) =
+
match Eq.Layout.(Bigarray.Genarray.layout x === layout) with
+
| None -> None
+
| Some Eq.W -> (
+
match Eq.Kind.(Bigarray.Genarray.kind x === kind) with
+
| None -> None
+
| Some Eq.W -> Some (x : (a, b, c) Bigarray.Genarray.t))
+
+
let to_bigarray1 (type a b c) (layout : c Bigarray.layout)
+
(kind : (a, b) Bigarray.kind) (P1 x) =
+
match Eq.Layout.(Bigarray.Array1.layout x === layout) with
+
| None -> None
+
| Some Eq.W -> (
+
match Eq.Kind.(Bigarray.Array1.kind x === kind) with
+
| None -> None
+
| Some Eq.W -> Some (x : (a, b, c) Bigarray.Array1.t))
+
+
let to_bigarray2 (type a b c) (layout : c Bigarray.layout)
+
(kind : (a, b) Bigarray.kind) (P2 x) =
+
match Eq.Layout.(Bigarray.Array2.layout x === layout) with
+
| None -> None
+
| Some Eq.W -> (
+
match Eq.Kind.(Bigarray.Array2.kind x === kind) with
+
| None -> None
+
| Some Eq.W -> Some (x : (a, b, c) Bigarray.Array2.t))
+
+
let to_bigarray3 (type a b c) (layout : c Bigarray.layout)
+
(kind : (a, b) Bigarray.kind) (P3 x) =
+
match Eq.Layout.(Bigarray.Array3.layout x === layout) with
+
| None -> None
+
| Some Eq.W -> (
+
match Eq.Kind.(Bigarray.Array3.kind x === kind) with
+
| None -> None
+
| Some Eq.W -> Some (x : (a, b, c) Bigarray.Array3.t))
+
+
(*
+
Apache License
+
Version 2.0, January 2004
+
http://www.apache.org/licenses/
+
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+
1. Definitions.
+
+
"License" shall mean the terms and conditions for use, reproduction,
+
and distribution as defined by Sections 1 through 9 of this document.
+
+
"Licensor" shall mean the copyright owner or entity authorized by
+
the copyright owner that is granting the License.
+
+
"Legal Entity" shall mean the union of the acting entity and all
+
other entities that control, are controlled by, or are under common
+
control with that entity. For the purposes of this definition,
+
"control" means (i) the power, direct or indirect, to cause the
+
direction or management of such entity, whether by contract or
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
+
outstanding shares, or (iii) beneficial ownership of such entity.
+
+
"You" (or "Your") shall mean an individual or Legal Entity
+
exercising permissions granted by this License.
+
+
"Source" form shall mean the preferred form for making modifications,
+
including but not limited to software source code, documentation
+
source, and configuration files.
+
+
"Object" form shall mean any form resulting from mechanical
+
transformation or translation of a Source form, including but
+
not limited to compiled object code, generated documentation,
+
and conversions to other media types.
+
+
"Work" shall mean the work of authorship, whether in Source or
+
Object form, made available under the License, as indicated by a
+
copyright notice that is included in or attached to the work
+
(an example is provided in the Appendix below).
+
+
"Derivative Works" shall mean any work, whether in Source or Object
+
form, that is based on (or derived from) the Work and for which the
+
editorial revisions, annotations, elaborations, or other modifications
+
represent, as a whole, an original work of authorship. For the purposes
+
of this License, Derivative Works shall not include works that remain
+
separable from, or merely link (or bind by name) to the interfaces of,
+
the Work and Derivative Works thereof.
+
+
"Contribution" shall mean any work of authorship, including
+
the original version of the Work and any modifications or additions
+
to that Work or Derivative Works thereof, that is intentionally
+
submitted to Licensor for inclusion in the Work by the copyright owner
+
or by an individual or Legal Entity authorized to submit on behalf of
+
the copyright owner. For the purposes of this definition, "submitted"
+
means any form of electronic, verbal, or written communication sent
+
to the Licensor or its representatives, including but not limited to
+
communication on electronic mailing lists, source code control systems,
+
and issue tracking systems that are managed by, or on behalf of, the
+
Licensor for the purpose of discussing and improving the Work, but
+
excluding communication that is conspicuously marked or otherwise
+
designated in writing by the copyright owner as "Not a Contribution."
+
+
"Contributor" shall mean Licensor and any individual or Legal Entity
+
on behalf of whom a Contribution has been received by Licensor and
+
subsequently incorporated within the Work.
+
+
2. Grant of Copyright License. Subject to the terms and conditions of
+
this License, each Contributor hereby grants to You a perpetual,
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+
copyright license to reproduce, prepare Derivative Works of,
+
publicly display, publicly perform, sublicense, and distribute the
+
Work and such Derivative Works in Source or Object form.
+
+
3. Grant of Patent License. Subject to the terms and conditions of
+
this License, each Contributor hereby grants to You a perpetual,
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+
(except as stated in this section) patent license to make, have made,
+
use, offer to sell, sell, import, and otherwise transfer the Work,
+
where such license applies only to those patent claims licensable
+
by such Contributor that are necessarily infringed by their
+
Contribution(s) alone or by combination of their Contribution(s)
+
with the Work to which such Contribution(s) was submitted. If You
+
institute patent litigation against any entity (including a
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
+
or a Contribution incorporated within the Work constitutes direct
+
or contributory patent infringement, then any patent licenses
+
granted to You under this License for that Work shall terminate
+
as of the date such litigation is filed.
+
+
4. Redistribution. You may reproduce and distribute copies of the
+
Work or Derivative Works thereof in any medium, with or without
+
modifications, and in Source or Object form, provided that You
+
meet the following conditions:
+
+
(a) You must give any other recipients of the Work or
+
Derivative Works a copy of this License; and
+
+
(b) You must cause any modified files to carry prominent notices
+
stating that You changed the files; and
+
+
(c) You must retain, in the Source form of any Derivative Works
+
that You distribute, all copyright, patent, trademark, and
+
attribution notices from the Source form of the Work,
+
excluding those notices that do not pertain to any part of
+
the Derivative Works; and
+
+
(d) If the Work includes a "NOTICE" text file as part of its
+
distribution, then any Derivative Works that You distribute must
+
include a readable copy of the attribution notices contained
+
within such NOTICE file, excluding those notices that do not
+
pertain to any part of the Derivative Works, in at least one
+
of the following places: within a NOTICE text file distributed
+
as part of the Derivative Works; within the Source form or
+
documentation, if provided along with the Derivative Works; or,
+
within a display generated by the Derivative Works, if and
+
wherever such third-party notices normally appear. The contents
+
of the NOTICE file are for informational purposes only and
+
do not modify the License. You may add Your own attribution
+
notices within Derivative Works that You distribute, alongside
+
or as an addendum to the NOTICE text from the Work, provided
+
that such additional attribution notices cannot be construed
+
as modifying the License.
+
+
You may add Your own copyright statement to Your modifications and
+
may provide additional or different license terms and conditions
+
for use, reproduction, or distribution of Your modifications, or
+
for any such Derivative Works as a whole, provided Your use,
+
reproduction, and distribution of the Work otherwise complies with
+
the conditions stated in this License.
+
+
5. Submission of Contributions. Unless You explicitly state otherwise,
+
any Contribution intentionally submitted for inclusion in the Work
+
by You to the Licensor shall be under the terms and conditions of
+
this License, without any additional terms or conditions.
+
Notwithstanding the above, nothing herein shall supersede or modify
+
the terms of any separate license agreement you may have executed
+
with Licensor regarding such Contributions.
+
+
6. Trademarks. This License does not grant permission to use the trade
+
names, trademarks, service marks, or product names of the Licensor,
+
except as required for reasonable and customary use in describing the
+
origin of the Work and reproducing the content of the NOTICE file.
+
+
7. Disclaimer of Warranty. Unless required by applicable law or
+
agreed to in writing, Licensor provides the Work (and each
+
Contributor provides its Contributions) on an "AS IS" BASIS,
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+
implied, including, without limitation, any warranties or conditions
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+
PARTICULAR PURPOSE. You are solely responsible for determining the
+
appropriateness of using or redistributing the Work and assume any
+
risks associated with Your exercise of permissions under this License.
+
+
8. Limitation of Liability. In no event and under no legal theory,
+
whether in tort (including negligence), contract, or otherwise,
+
unless required by applicable law (such as deliberate and grossly
+
negligent acts) or agreed to in writing, shall any Contributor be
+
liable to You for damages, including any direct, indirect, special,
+
incidental, or consequential damages of any character arising as a
+
result of this License or out of the use or inability to use the
+
Work (including but not limited to damages for loss of goodwill,
+
work stoppage, computer failure or malfunction, or any and all
+
other commercial damages or losses), even if such Contributor
+
has been advised of the possibility of such damages.
+
+
9. Accepting Warranty or Additional Liability. While redistributing
+
the Work or Derivative Works thereof, You may choose to offer,
+
and charge a fee for, acceptance of support, warranty, indemnity,
+
or other liability obligations and/or rights consistent with this
+
License. However, in accepting such obligations, You may act only
+
on Your own behalf and on Your sole responsibility, not on behalf
+
of any other Contributor, and only if You agree to indemnify,
+
defend, and hold each Contributor harmless for any liability
+
incurred by, or claims asserted against, such Contributor by reason
+
of your accepting any such warranty or additional liability.
+
+
END OF TERMS AND CONDITIONS
+
+
APPENDIX: How to apply the Apache License to your work.
+
+
To apply the Apache License to your work, attach the following
+
boilerplate notice, with the fields enclosed by brackets "{}"
+
replaced with your own identifying information. (Don't include
+
the brackets!) The text should be enclosed in the appropriate
+
comment syntax for the file format. We also recommend that a
+
file or class name and description of purpose be included on the
+
same "printed page" as the copyright notice for easier
+
identification within third-party archives.
+
+
Copyright {yyyy} {name of copyright owner}
+
+
Licensed under the Apache License, Version 2.0 (the "License");
+
you may not use this file except in compliance with the License.
+
You may obtain a copy of the License at
+
+
http://www.apache.org/licenses/LICENSE-2.0
+
+
Unless required by applicable law or agreed to in writing, software
+
distributed under the License is distributed on an "AS IS" BASIS,
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
See the License for the specific language governing permissions and
+
limitations under the License. *)