plots: hacky plots (#355)

* plots: hacky plots * gha: lint: show diff on failure Co-authored-by: Ruslan Kuprieiev <[email protected]>
iterative · Jun 30, 2022 · 545e160 · 545e160
1 parent c38a90b
commit 545e160
Show file tree

Hide file tree

Showing 36 changed files with 333 additions and 82 deletions.
diff --git a/.dvc/plot/confusion.json b/.dvc/plot/confusion.json
diff --git a/.dvc/plot/default.json b/.dvc/plot/default.json
diff --git a/.dvc/plot/scatter.json b/.dvc/plot/scatter.json
diff --git a/.dvc/plots/linear_versions.json b/.dvc/plots/linear_versions.json
@@ -0,0 +1,117 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "width": 300,
+    "height": 300,
+    "layer": [
+        {
+            "encoding": {
+                "x": {
+                    "field": "<DVC_METRIC_X>",
+                    "title": "<DVC_METRIC_X_LABEL>",
+                    "sort": "index"
+                },
+                "y": {
+                    "field": "<DVC_METRIC_Y>",
+                    "type": "quantitative",
+                    "title": "<DVC_METRIC_Y_LABEL>"
+                },
+                "color": {
+                    "field": "rev",
+                    "type": "nominal"
+                },
+                "order": {
+                    "field": "index",
+                    "type": "ordinal"
+                }
+            },
+            "layer": [
+                {
+                    "mark": "line"
+                },
+                {
+                    "selection": {
+                        "label": {
+                            "type": "single",
+                            "nearest": true,
+                            "on": "mouseover",
+                            "encodings": [
+                                "x"
+                            ],
+                            "empty": "none",
+                            "clear": "mouseout"
+                        }
+                    },
+                    "mark": "point",
+                    "encoding": {
+                        "opacity": {
+                            "condition": {
+                                "selection": "label",
+                                "value": 1
+                            },
+                            "value": 0
+                        }
+                    }
+                }
+            ]
+        },
+        {
+            "transform": [
+                {
+                    "filter": {
+                        "selection": "label"
+                    }
+                }
+            ],
+            "layer": [
+                {
+                    "mark": {
+                        "type": "rule",
+                        "color": "gray"
+                    },
+                    "encoding": {
+                        "x": {
+                            "field": "<DVC_METRIC_X>",
+                            "sort": "index"
+                        }
+                    }
+                },
+                {
+                    "encoding": {
+                        "text": {
+                            "type": "quantitative",
+                            "field": "<DVC_METRIC_Y>"
+                        },
+                        "x": {
+                            "field": "<DVC_METRIC_X>",
+                            "sort": "index"
+                        },
+                        "y": {
+                            "field": "<DVC_METRIC_Y>",
+                            "type": "quantitative"
+                        }
+                    },
+                    "layer": [
+                        {
+                            "mark": {
+                                "type": "text",
+                                "align": "left",
+                                "dx": 5,
+                                "dy": -5
+                            },
+                            "encoding": {
+                                "color": {
+                                    "type": "nominal",
+                                    "field": "rev"
+                                }
+                            }
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -37,7 +37,7 @@ jobs:
           pip install wheel
           pip install -r requirements.txt
       - name: check project styling
-        run: pre-commit run --all-files
+        run: pre-commit run --all-files --show-diff-on-failure
   gen:
     runs-on: ubuntu-latest
     outputs:

diff --git a/dvc.lock b/dvc.lock
@@ -0,0 +1,16 @@
+schema: '2.0'
+stages:
+  gen_plots:
+    cmd: python plots.py
+    deps:
+    - path: plots.py
+      md5: d14581b088a19976c87b41f39a1200d6
+      size: 558
+    - path: results.csv
+      md5: 538b4993f552f45b3ac20732ccb95217
+      size: 26070
+    outs:
+    - path: plots
+      md5: 698fe1027e6ba06b6f37073d15204285.dir
+      size: 32329
+      nfiles: 27
diff --git a/dvc.yaml b/dvc.yaml
@@ -0,0 +1,14 @@
+stages:
+  gen_plots:
+    cmd: python plots.py
+    deps:
+    - plots.py
+    - results.csv
+    plots:
+    - plots:
+        cache: false
+        x: 'param:dvc_rev'
+        x_label: 'DVC Revision'
+        y: mean
+        y_label: 'Mean Time'
+        template: linear_versions
diff --git a/plots.py b/plots.py
@@ -0,0 +1,22 @@
+import os
+
+import pandas as pd
+from distutils.version import StrictVersion
+
+df = pd.read_csv("results.csv")
+df["test"] = df["name"].str.extract(r"::(.*)\[")
+
+
+def version(x):
+    try:
+        return StrictVersion(x)
+    except ValueError:
+        return StrictVersion("99.99.99")
+
+
+os.makedirs("plots", exist_ok=True)
+for test, test_df in df.groupby("test"):
+    test_df["ver_sort"] = test_df["param:dvc_rev"].apply(version)
+    test_df = test_df.sort_values("ver_sort").reset_index(drop=True)
+    test_df.index.name = "index"
+    test_df.to_csv(f"plots/{test}.csv")
diff --git a/plots/test_add-add.csv b/plots/test_add-add.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_add.py::test_add-add[2.6.3],2.6.3,332.2735612050001,332.2735612050001,332.2735612050001,0.0,332.2735612050001,0.0,0;0,0.0030095683700306,1,1,test_add-add,2.6.3
+1,tests/benchmarks/cli/commands/test_add.py::test_add-add[2.8.3],2.8.3,357.0697099250001,357.0697099250001,357.0697099250001,0.0,357.0697099250001,0.0,0;0,0.0028005735916665,1,1,test_add-add,2.8.3
+2,tests/benchmarks/cli/commands/test_add.py::test_add-add[2.9.5],2.9.5,358.93439585700025,358.93439585700025,358.93439585700025,0.0,358.93439585700025,0.0,0;0,0.0027860244421891,1,1,test_add-add,2.9.5
+3,tests/benchmarks/cli/commands/test_add.py::test_add-add[2.10.0],2.10.0,352.3659208939998,352.3659208939998,352.3659208939998,0.0,352.3659208939998,0.0,0;0,0.0028379588964303,1,1,test_add-add,2.10
+4,tests/benchmarks/cli/commands/test_add.py::test_add-add[main],main,293.172158291,293.172158291,293.172158291,0.0,293.172158291,0.0,0;0,0.0034109650992418,1,1,test_add-add,99.99.99
diff --git a/plots/test_checkout-checkout-noop.csv b/plots/test_checkout-checkout-noop.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-noop[2.6.3],2.6.3,2.500541410000096,2.500541410000096,2.500541410000096,0.0,2.500541410000096,0.0,0;0,0.3999133931559092,1,1,test_checkout-checkout-noop,2.6.3
+1,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-noop[2.8.3],2.8.3,6.217971195000246,6.217971195000246,6.217971195000246,0.0,6.217971195000246,0.0,0;0,0.1608241609102469,1,1,test_checkout-checkout-noop,2.8.3
+2,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-noop[2.9.5],2.9.5,5.44425583400016,5.44425583400016,5.44425583400016,0.0,5.44425583400016,0.0,0;0,0.1836798325594577,1,1,test_checkout-checkout-noop,2.9.5
+3,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-noop[2.10.0],2.10.0,5.306598285999826,5.306598285999826,5.306598285999826,0.0,5.306598285999826,0.0,0;0,0.1884446393159734,1,1,test_checkout-checkout-noop,2.10
+4,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-noop[main],main,5.234277915000121,5.234277915000121,5.234277915000121,0.0,5.234277915000121,0.0,0;0,0.1910483196037895,1,1,test_checkout-checkout-noop,99.99.99
diff --git a/plots/test_checkout-checkout-update.csv b/plots/test_checkout-checkout-update.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-update[2.6.3],2.6.3,94.88412419399992,94.88412419399992,94.88412419399992,0.0,94.88412419399992,0.0,0;0,0.0105391708939147,1,1,test_checkout-checkout-update,2.6.3
+1,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-update[2.8.3],2.8.3,38.00906510200002,38.00906510200002,38.00906510200002,0.0,38.00906510200002,0.0,0;0,0.0263095131994546,1,1,test_checkout-checkout-update,2.8.3
+2,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-update[2.9.5],2.9.5,34.66704759100003,34.66704759100003,34.66704759100003,0.0,34.66704759100003,0.0,0;0,0.0288458368822735,1,1,test_checkout-checkout-update,2.9.5
+3,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-update[2.10.0],2.10.0,34.74823923799977,34.74823923799977,34.74823923799977,0.0,34.74823923799977,0.0,0;0,0.0287784366036718,1,1,test_checkout-checkout-update,2.10
+4,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout-update[main],main,40.349377904,40.349377904,40.349377904,0.0,40.349377904,0.0,0;0,0.0247835295597176,1,1,test_checkout-checkout-update,99.99.99
diff --git a/plots/test_checkout-checkout.csv b/plots/test_checkout-checkout.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout[2.6.3],2.6.3,63.1818875099998,63.1818875099998,63.1818875099998,0.0,63.1818875099998,0.0,0;0,0.0158273207624848,1,1,test_checkout-checkout,2.6.3
+1,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout[2.8.3],2.8.3,83.62453495900036,83.62453495900036,83.62453495900036,0.0,83.62453495900036,0.0,0;0,0.0119582129872564,1,1,test_checkout-checkout,2.8.3
+2,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout[2.9.5],2.9.5,102.08011368900044,102.08011368900044,102.08011368900044,0.0,102.08011368900044,0.0,0;0,0.0097962273342153,1,1,test_checkout-checkout,2.9.5
+3,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout[2.10.0],2.10.0,103.2896065829998,103.2896065829998,103.2896065829998,0.0,103.2896065829998,0.0,0;0,0.0096815162055674,1,1,test_checkout-checkout,2.10
+4,tests/benchmarks/cli/commands/test_checkout.py::test_checkout-checkout[main],main,88.98860477499989,88.98860477499989,88.98860477499989,0.0,88.98860477499989,0.0,0;0,0.0112373938497902,1,1,test_checkout-checkout,99.99.99
diff --git a/plots/test_diff-diff-changed-noop.csv b/plots/test_diff-diff-changed-noop.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed-noop[2.6.3],2.6.3,90.92673303200036,90.92673303200036,90.92673303200036,0.0,90.92673303200036,0.0,0;0,0.010997865717314,1,1,test_diff-diff-changed-noop,2.6.3
+1,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed-noop[2.8.3],2.8.3,41.53946576699991,41.53946576699991,41.53946576699991,0.0,41.53946576699991,0.0,0;0,0.0240734920764057,1,1,test_diff-diff-changed-noop,2.8.3
+2,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed-noop[2.9.5],2.9.5,34.86566686700007,34.86566686700007,34.86566686700007,0.0,34.86566686700007,0.0,0;0,0.0286815107771963,1,1,test_diff-diff-changed-noop,2.9.5
+3,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed-noop[2.10.0],2.10.0,35.437922630999765,35.437922630999765,35.437922630999765,0.0,35.437922630999765,0.0,0;0,0.0282183583505325,1,1,test_diff-diff-changed-noop,2.10
+4,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed-noop[main],main,3.174017637999896,3.174017637999896,3.174017637999896,0.0,3.174017637999896,0.0,0;0,0.3150581105876113,1,1,test_diff-diff-changed-noop,99.99.99
diff --git a/plots/test_diff-diff-changed.csv b/plots/test_diff-diff-changed.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed[2.6.3],2.6.3,89.31212003100018,89.31212003100018,89.31212003100018,0.0,89.31212003100018,0.0,0;0,0.0111966886426265,1,1,test_diff-diff-changed,2.6.3
+1,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed[2.8.3],2.8.3,42.19566154599988,42.19566154599988,42.19566154599988,0.0,42.19566154599988,0.0,0;0,0.0236991188989854,1,1,test_diff-diff-changed,2.8.3
+2,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed[2.9.5],2.9.5,35.37422902499975,35.37422902499975,35.37422902499975,0.0,35.37422902499975,0.0,0;0,0.0282691673447717,1,1,test_diff-diff-changed,2.9.5
+3,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed[2.10.0],2.10.0,34.44541783700015,34.44541783700015,34.44541783700015,0.0,34.44541783700015,0.0,0;0,0.0290314376423627,1,1,test_diff-diff-changed,2.10
+4,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-changed[main],main,33.68136617900018,33.68136617900018,33.68136617900018,0.0,33.68136617900018,0.0,0;0,0.0296900070705411,1,1,test_diff-diff-changed,99.99.99
diff --git a/plots/test_diff-diff-noop.csv b/plots/test_diff-diff-noop.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-noop[2.6.3],2.6.3,57.93432388000019,57.93432388000019,57.93432388000019,0.0,57.93432388000019,0.0,0;0,0.0172609246648205,1,1,test_diff-diff-noop,2.6.3
+1,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-noop[2.8.3],2.8.3,8.561673233999954,8.561673233999954,8.561673233999954,0.0,8.561673233999954,0.0,0;0,0.1167995989415735,1,1,test_diff-diff-noop,2.8.3
+2,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-noop[2.9.5],2.9.5,3.8153779999997823,3.8153779999997823,3.8153779999997823,0.0,3.8153779999997823,0.0,0;0,0.2620972286363388,1,1,test_diff-diff-noop,2.9.5
+3,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-noop[2.10.0],2.10.0,3.503529090000029,3.503529090000029,3.503529090000029,0.0,3.503529090000029,0.0,0;0,0.2854264869254994,1,1,test_diff-diff-noop,2.10
+4,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff-noop[main],main,3.509945867000169,3.509945867000169,3.509945867000169,0.0,3.509945867000169,0.0,0;0,0.2849046788447099,1,1,test_diff-diff-noop,99.99.99
diff --git a/plots/test_diff-diff.csv b/plots/test_diff-diff.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff[2.6.3],2.6.3,116.16352329499978,116.16352329499978,116.16352329499978,0.0,116.16352329499978,0.0,0;0,0.0086085543175242,1,1,test_diff-diff,2.6.3
+1,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff[2.8.3],2.8.3,71.45220055499976,71.45220055499976,71.45220055499976,0.0,71.45220055499976,0.0,0;0,0.0139953702227863,1,1,test_diff-diff,2.8.3
+2,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff[2.9.5],2.9.5,62.29177433399991,62.29177433399991,62.29177433399991,0.0,62.29177433399991,0.0,0;0,0.0160534839582211,1,1,test_diff-diff,2.9.5
+3,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff[2.10.0],2.10.0,60.79699943600008,60.79699943600008,60.79699943600008,0.0,60.79699943600008,0.0,0;0,0.016448180161468,1,1,test_diff-diff,2.10
+4,tests/benchmarks/cli/commands/test_diff.py::test_diff-diff[main],main,55.25879571899986,55.25879571899986,55.25879571899986,0.0,55.25879571899986,0.0,0;0,0.0180966665485285,1,1,test_diff-diff,99.99.99
diff --git a/plots/test_gc-gc.csv b/plots/test_gc-gc.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_gc.py::test_gc-gc[2.6.3],2.6.3,4.08804847600004,4.08804847600004,4.08804847600004,0.0,4.08804847600004,0.0,0;0,0.2446154946231098,1,1,test_gc-gc,2.6.3
+1,tests/benchmarks/cli/commands/test_gc.py::test_gc-gc[2.8.3],2.8.3,4.260282059000019,4.260282059000019,4.260282059000019,0.0,4.260282059000019,0.0,0;0,0.2347262425705967,1,1,test_gc-gc,2.8.3
+2,tests/benchmarks/cli/commands/test_gc.py::test_gc-gc[2.9.5],2.9.5,3.660101109999914,3.660101109999914,3.660101109999914,0.0,3.660101109999914,0.0,0;0,0.2732164959235302,1,1,test_gc-gc,2.9.5
+3,tests/benchmarks/cli/commands/test_gc.py::test_gc-gc[2.10.0],2.10.0,3.5930902279999373,3.5930902279999373,3.5930902279999373,0.0,3.5930902279999373,0.0,0;0,0.2783119645054506,1,1,test_gc-gc,2.10
+4,tests/benchmarks/cli/commands/test_gc.py::test_gc-gc[main],main,3.237301605000084,3.237301605000084,3.237301605000084,0.0,3.237301605000084,0.0,0;0,0.308899238321038,1,1,test_gc-gc,99.99.99
diff --git a/plots/test_get-get.csv b/plots/test_get-get.csv
@@ -0,0 +1,6 @@
+index,name,param:dvc_rev,min,max,mean,stddev,median,iqr,outliers,ops,rounds,iterations,test,ver_sort
+0,tests/benchmarks/cli/commands/test_get.py::test_get-get[2.6.3],2.6.3,120.5367786429997,120.5367786429997,120.5367786429997,0.0,120.5367786429997,0.0,0;0,0.0082962230387934,1,1,test_get-get,2.6.3
+1,tests/benchmarks/cli/commands/test_get.py::test_get-get[2.8.3],2.8.3,99.47031682399984,99.47031682399984,99.47031682399984,0.0,99.47031682399984,0.0,0;0,0.0100532503758822,1,1,test_get-get,2.8.3
+2,tests/benchmarks/cli/commands/test_get.py::test_get-get[2.9.5],2.9.5,103.95698615099992,103.95698615099992,103.95698615099992,0.0,103.95698615099992,0.0,0;0,0.0096193631330123,1,1,test_get-get,2.9.5
+3,tests/benchmarks/cli/commands/test_get.py::test_get-get[2.10.0],2.10.0,103.30797336599994,103.30797336599994,103.30797336599994,0.0,103.30797336599994,0.0,0;0,0.0096797949608129,1,1,test_get-get,2.10
+4,tests/benchmarks/cli/commands/test_get.py::test_get-get[main],main,169.58718436499998,169.58718436499998,169.58718436499998,0.0,169.58718436499998,0.0,0;0,0.0058966719905421,1,1,test_get-get,99.99.99