From 2af6e61b78dd03524aa2794c0a80149fd3c0241c Mon Sep 17 00:00:00 2001 From: Rizo I Date: Tue, 27 Apr 2021 12:46:27 +0100 Subject: [PATCH 1/4] Add `make bench`. --- Makefile | 4 +- bench/Bench_main.ml | 89 +++++++++++++++++++++++++++++++++++++++++++++ bench/dune | 10 +++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 bench/Bench_main.ml create mode 100644 bench/dune diff --git a/Makefile b/Makefile index 572a6bb..8c2d758 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,6 @@ docs-update: git-check-uncommited docs-build fi git checkout - - +.PHONY: bench +bench: + @dune exec -- ./bench/Bench_main.exe --format=json \ No newline at end of file diff --git a/bench/Bench_main.ml b/bench/Bench_main.ml new file mode 100644 index 0000000..21b5ccb --- /dev/null +++ b/bench/Bench_main.ml @@ -0,0 +1,89 @@ +open Bechamel +open Toolkit + +let make_all n = + let open Streaming in + Staged.stage @@ fun () -> + Stream.(0 -- n) + |> Stream.map (fun x -> x + 1) + |> Stream.filter (fun x -> x mod 3 = 0) + |> Stream.take (n / 2) + |> Stream.flat_map (fun x -> Stream.(x -- (x + 30))) + |> Stream.fold ( + ) 0 + + +let test = + Test.make_indexed ~name:"all" ~fmt:"%s %d" + ~args:[ 10; 100; 1_000; 10_000; 100_000; 1_000_000 ] + make_all + + +let benchmark () = + let ols = + Analyze.ols ~bootstrap:0 ~r_square:true ~predictors:Measure.[| run |] + in + let instances = + Instance.[ minor_allocated; major_allocated; monotonic_clock ] + in + let cfg = + Benchmark.cfg ~limit:2000 ~quota:(Time.second 0.5) ~kde:(Some 1000) () + in + let raw_results = Benchmark.all cfg instances test in + let results = + List.map (fun instance -> Analyze.all ols instance raw_results) instances + in + let results = Analyze.merge ols instances results in + (results, raw_results) + + +let output_current_bench () = + let results, _ = benchmark () in + let json : Yojson.Safe.t = + Current_bench_bechamel.json_of_ols_results results + in + Yojson.Safe.pretty_to_channel stdout json; + print_newline () + + +let output_notty () = + List.iter + (fun v -> Bechamel_notty.Unit.add v (Measure.unit v)) + Instance.[ minor_allocated; major_allocated; monotonic_clock ]; + + let img (window, results) = + Bechamel_notty.Multiple.image_of_ols_results ~rect:window + ~predictor:Measure.run results + in + + let open Notty_unix in + let window = + match winsize Unix.stdout with + | Some (w, h) -> { Bechamel_notty.w; h } + | None -> { Bechamel_notty.w = 80; h = 1 } + in + let results, _ = benchmark () in + img (window, results) |> eol |> output_image + + +let main format = + match format with + | `Json -> output_current_bench () + | `Table -> output_notty () + + +let () = + let open Cmdliner in + let format = + let doc = "Benchmark output format." in + Arg.( + required + & opt (some (enum [ ("json", `Json); ("table", `Table) ])) None + & info [ "format" ] ~doc) + in + + let info = + let doc = "Run the main streaming benchmark." in + Term.info "streaming-bench-main" ~doc + in + let term = Term.(const main $ format) in + Term.exit @@ Term.eval (term, info) diff --git a/bench/dune b/bench/dune new file mode 100644 index 0000000..7ebdf04 --- /dev/null +++ b/bench/dune @@ -0,0 +1,10 @@ +(executable + (public_name streaming-bench-main) + (name Bench_main) + (libraries + cmdliner + streaming + current-bench-bechamel + bechamel + bechamel-notty + notty.unix)) From 80887232a8f465de24fcede3075bf1a006b99a08 Mon Sep 17 00:00:00 2001 From: Rizo I Date: Tue, 27 Apr 2021 13:25:02 +0100 Subject: [PATCH 2/4] Vendor current_bench converter. --- bench/Current_bench_bechamel.ml | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 bench/Current_bench_bechamel.ml diff --git a/bench/Current_bench_bechamel.ml b/bench/Current_bench_bechamel.ml new file mode 100644 index 0000000..6b6b71c --- /dev/null +++ b/bench/Current_bench_bechamel.ml @@ -0,0 +1,52 @@ +type 'a result = (string, 'a) Hashtbl.t + +type 'a results = (string, 'a result) Hashtbl.t + +let process_results results = + let metrics_by_test = Hashtbl.create 16 in + Hashtbl.iter + (fun metric_name result -> + Hashtbl.iter + (fun test_name ols -> + let metrics = + try Hashtbl.find metrics_by_test test_name + with Not_found -> Hashtbl.create 16 + in + Hashtbl.add metrics metric_name ols; + Hashtbl.replace metrics_by_test test_name metrics) + result) + results; + metrics_by_test + + +let json_of_ols ols = + match Bechamel.Analyze.OLS.estimates ols with + | Some [ x ] -> `Float x + | Some estimates -> `List (List.map (fun x -> `Float x) estimates) + | None -> `List [] + + +let json_of_ols_results ?name (results : Bechamel.Analyze.OLS.t results) : + Yojson.Safe.t = + let metrics_by_test = process_results results in + let results = + metrics_by_test |> Hashtbl.to_seq + |> Seq.map (fun (test_name, metrics) -> + let metrics = + metrics |> Hashtbl.to_seq + |> Seq.map (fun (metric_name, ols) -> + (metric_name, json_of_ols ols)) + |> List.of_seq + |> fun bindings -> `Assoc bindings + in + `Assoc [ ("name", `String test_name); ("metrics", metrics) ]) + |> List.of_seq + |> fun items -> `List items + in + let bindings = [ ("results", results) ] in + let bindings = + match name with + | Some name -> ("name", `String name) :: bindings + | None -> bindings + in + `Assoc bindings From e13f0e08358c6445238026d0e3b9d374ed8a3bd4 Mon Sep 17 00:00:00 2001 From: Rizo I Date: Tue, 27 Apr 2021 13:25:18 +0100 Subject: [PATCH 3/4] Add separate package for benchmarks. --- bench/dune | 9 ++------- streaming-bench.opam | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 streaming-bench.opam diff --git a/bench/dune b/bench/dune index 7ebdf04..792d4df 100644 --- a/bench/dune +++ b/bench/dune @@ -1,10 +1,5 @@ (executable (public_name streaming-bench-main) + (package streaming-bench) (name Bench_main) - (libraries - cmdliner - streaming - current-bench-bechamel - bechamel - bechamel-notty - notty.unix)) + (libraries cmdliner yojson streaming bechamel bechamel-notty notty.unix)) diff --git a/streaming-bench.opam b/streaming-bench.opam new file mode 100644 index 0000000..9e3b3c4 --- /dev/null +++ b/streaming-bench.opam @@ -0,0 +1,26 @@ +opam-version: "2.0" + +homepage: "http://github.com/odis-labs/streaming" +doc: "https://odis-labs.github.io/streaming" +bug-reports: "https://github.com/odis-labs/streaming/issues" +license: "ISC" + +authors: [ + "Rizo I " +] +maintainer: "Rizo I " +dev-repo: "git+https://github.com/odis-labs/streaming.git" + +synopsis: "Benchmarks for streaming." +depends: [ + "dune" {>= "2.0"} + "ocaml" {>= "4.08.0"} + "streaming" + "bechamel" + "bechamel-notty" + "yojson" +] + +build: [ + ["dune" "build" "-p" name "-j" jobs "@install" "@doc" {with-doc}] +] From 4db34288c7025bb60907353779fbc0b030e63a30 Mon Sep 17 00:00:00 2001 From: Rizo I Date: Tue, 27 Apr 2021 13:30:54 +0100 Subject: [PATCH 4/4] Add missing cmdliner dep. --- streaming-bench.opam | 1 + 1 file changed, 1 insertion(+) diff --git a/streaming-bench.opam b/streaming-bench.opam index 9e3b3c4..666da64 100644 --- a/streaming-bench.opam +++ b/streaming-bench.opam @@ -15,6 +15,7 @@ synopsis: "Benchmarks for streaming." depends: [ "dune" {>= "2.0"} "ocaml" {>= "4.08.0"} + "cmdliner" "streaming" "bechamel" "bechamel-notty"