Skip to content

Commit 271134b

Browse files
johnkerljohnkerl
committed
csv/csvlite perf timings
1 parent 81fe4ed commit 271134b

File tree

1 file changed

+73
-53
lines changed

1 file changed

+73
-53
lines changed

‎perf/do-timings.rb

Lines changed: 73 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,15 @@ def run(desc, cmd)
2222
# catc real 0.55
2323

2424
# ----------------------------------------------------------------
25-
run("(cat)", "cat < ../data/big > /dev/null")
26-
run("catc0", "catc0 < ../data/big > /dev/null")
27-
run("catc0", "catc0 ../data/big > /dev/null")
28-
run("catc", "catc < ../data/big > /dev/null")
29-
run("catc", "catc ../data/big > /dev/null")
30-
run("catm", "catm ../data/big > /dev/null")
31-
run("catawk", "awk -F, '{print}' ../data/big > /dev/null")
32-
run("CATMLR", "mlr --no-mmap cat ../data/big > /dev/null")
33-
run("CATMLRM", "mlr --mmap cat ../data/big > /dev/null")
25+
run("(cat)", "cat < ../data/big.dkvp > /dev/null")
26+
run("catc0", "catc0 < ../data/big.dkvp > /dev/null")
27+
run("catc0", "catc0 ../data/big.dkvp > /dev/null")
28+
run("catc", "catc < ../data/big.dkvp > /dev/null")
29+
run("catc", "catc ../data/big.dkvp > /dev/null")
30+
run("catm", "catm ../data/big.dkvp > /dev/null")
31+
run("catawk", "awk -F, '{print}' ../data/big.dkvp > /dev/null")
32+
run("CATMLR", "mlr --no-mmap cat ../data/big.dkvp > /dev/null")
33+
run("CATMLRM", "mlr --mmap cat ../data/big.dkvp > /dev/null")
3434
puts
3535

3636
run("(catv)", "cat < ../data/big.csv > /dev/null")
@@ -40,84 +40,104 @@ def run(desc, cmd)
4040
run("catcv", "catc ../data/big.csv > /dev/null")
4141
run("catmv", "catm ../data/big.csv > /dev/null")
4242
run("catawkv", "awk -F, '{print}' ../data/big.csv > /dev/null")
43-
run("CATMLRV", "mlr --no-mmap --csv cat ../data/big.csv > /dev/null")
44-
run("CATMLRVM", "mlr --mmap --csv cat ../data/big.csv > /dev/null")
43+
run("CATMLRV", "mlr --no-mmap --csv --rs lf cat ../data/big.csv > /dev/null")
44+
run("CATMLRVM", "mlr --mmap --csv --rs lf cat ../data/big.csv > /dev/null")
45+
run("CATMLRV", "mlr --no-mmap --csvlite cat ../data/big.csv > /dev/null")
46+
run("CATMLRVM", "mlr --mmap --csvlite cat ../data/big.csv > /dev/null")
4547
puts
4648
puts
4749

48-
run("cutcut", "cut -d , -f 1,4 ../data/big > /dev/null")
49-
run("cutawk", "awk -F, '{print $1,$4}' ../data/big > /dev/null")
50-
run("CUTMLR", "mlr --no-mmap cut -f a,x ../data/big > /dev/null")
51-
run("CUTMLRM", "mlr --mmap cut -f a,x ../data/big > /dev/null")
52-
run("CUTMLRX", "mlr --no-mmap cut -x -f a,x ../data/big > /dev/null")
53-
run("CUTMLRXM", "mlr --mmap cut -x -f a,x ../data/big > /dev/null")
50+
run("cutcut", "cut -d , -f 1,4 ../data/big.dkvp > /dev/null")
51+
run("cutawk", "awk -F, '{print $1,$4}' ../data/big.dkvp > /dev/null")
52+
run("CUTMLR", "mlr --no-mmap cut -f a,x ../data/big.dkvp > /dev/null")
53+
run("CUTMLRM", "mlr --mmap cut -f a,x ../data/big.dkvp > /dev/null")
54+
run("CUTMLRX", "mlr --no-mmap cut -x -f a,x ../data/big.dkvp > /dev/null")
55+
run("CUTMLRXM", "mlr --mmap cut -x -f a,x ../data/big.dkvp > /dev/null")
5456
puts
5557

5658
run("cutcutv", "cut -d , -f 1,4 ../data/big.csv > /dev/null")
5759
run("cutawkv", "awk -F, '{print $1,$4}' ../data/big.csv > /dev/null")
58-
run("CUTMLRV", "mlr --no-mmap --csv cut -f a,x ../data/big.csv > /dev/null")
59-
run("CUTMLRVM", "mlr --mmap --csv cut -f a,x ../data/big.csv > /dev/null")
60-
run("CUTMLRXV", "mlr --no-mmap --csv cut -x -f a,x ../data/big.csv > /dev/null")
61-
run("CUTMLRXVM", "mlr --mmap --csv cut -x -f a,x ../data/big.csv > /dev/null")
60+
run("CUTMLRV", "mlr --no-mmap --csv --rs lf cut -f a,x ../data/big.csv > /dev/null")
61+
run("CUTMLRVM", "mlr --mmap --csv --rs lf cut -f a,x ../data/big.csv > /dev/null")
62+
run("CUTMLRXV", "mlr --no-mmap --csv --rs lf cut -x -f a,x ../data/big.csv > /dev/null")
63+
run("CUTMLRXVM", "mlr --mmap --csv --rs lf cut -x -f a,x ../data/big.csv > /dev/null")
64+
run("CUTMLRV", "mlr --no-mmap --csvlite cut -f a,x ../data/big.csv > /dev/null")
65+
run("CUTMLRVM", "mlr --mmap --csvlite cut -f a,x ../data/big.csv > /dev/null")
66+
run("CUTMLRXV", "mlr --no-mmap --csvlite cut -x -f a,x ../data/big.csv > /dev/null")
67+
run("CUTMLRXVM", "mlr --mmap --csvlite cut -x -f a,x ../data/big.csv > /dev/null")
6268
puts
6369
puts
6470

65-
run("rensed", "sed -e 's/x=/EKS=/' -e 's/b=/BEE=/' ../data/big > /dev/null")
66-
run("RENMLR", "mlr --no-mmap rename x,EKS,b,BEE ../data/big > /dev/null")
67-
run("RENMLRM", "mlr --mmap rename x,EKS,b,BEE ../data/big > /dev/null")
71+
run("rensed", "sed -e 's/x=/EKS=/' -e 's/b=/BEE=/' ../data/big.dkvp > /dev/null")
72+
run("RENMLR", "mlr --no-mmap rename x,EKS,b,BEE ../data/big.dkvp > /dev/null")
73+
run("RENMLRM", "mlr --mmap rename x,EKS,b,BEE ../data/big.dkvp > /dev/null")
6874
puts
6975

7076
run("rensedv", "sed -e 's/x=/EKS=/' -e 's/b=/BEE=/' ../data/big.csv > /dev/null")
71-
run("RENMLRV", "mlr --no-mmap --csv rename x,EKS,b,BEE ../data/big.csv > /dev/null")
72-
run("RENMLRVM", "mlr --mmap --csv rename x,EKS,b,BEE ../data/big.csv > /dev/null")
77+
run("RENMLRV", "mlr --no-mmap --csv --rs lf rename x,EKS,b,BEE ../data/big.csv > /dev/null")
78+
run("RENMLRVM", "mlr --mmap --csv --rs lf rename x,EKS,b,BEE ../data/big.csv > /dev/null")
79+
run("RENMLRV", "mlr --no-mmap --csvlite rename x,EKS,b,BEE ../data/big.csv > /dev/null")
80+
run("RENMLRVM", "mlr --mmap --csvlite rename x,EKS,b,BEE ../data/big.csv > /dev/null")
7381
puts
7482
puts
7583

76-
run("addawk", "awk -F, '{gsub(\"x=\",\"\",$4);gsub(\"y=\",\"\",$5);print $4+$5}' ../data/big > /dev/null")
77-
run("ADDMLR", "mlr --no-mmap put '$z=$x+$y' ../data/big > /dev/null")
78-
run("ADDMLRM", "mlr --mmap put '$z=$x+$y' ../data/big > /dev/null")
84+
run("addawk", "awk -F, '{gsub(\"x=\",\"\",$4);gsub(\"y=\",\"\",$5);print $4+$5}' ../data/big.dkvp > /dev/null")
85+
run("ADDMLR", "mlr --no-mmap put '$z=$x+$y' ../data/big.dkvp > /dev/null")
86+
run("ADDMLRM", "mlr --mmap put '$z=$x+$y' ../data/big.dkvp > /dev/null")
7987
puts
8088

8189
run("addawkv", "awk -F, '{gsub(\"x=\",\"\",$4);gsub(\"y=\",\"\",$5);print $4+$5}' ../data/big.csv > /dev/null")
82-
run("ADDMLRV", "mlr --no-mmap --csv put '$z=$x+$y' ../data/big.csv > /dev/null")
83-
run("ADDMLRVM", "mlr --mmap --csv put '$z=$x+$y' ../data/big.csv > /dev/null")
90+
run("ADDMLRV", "mlr --no-mmap --csv --rs lf put '$z=$x+$y' ../data/big.csv > /dev/null")
91+
run("ADDMLRVM", "mlr --mmap --csv --rs lf put '$z=$x+$y' ../data/big.csv > /dev/null")
92+
run("ADDMLRV", "mlr --no-mmap --csvlite put '$z=$x+$y' ../data/big.csv > /dev/null")
93+
run("ADDMLRVM", "mlr --mmap --csvlite put '$z=$x+$y' ../data/big.csv > /dev/null")
8494
puts
8595
puts
8696

87-
run("MEANMLR", "mlr --no-mmap stats1 -a mean -f x,y -g a,b ../data/big > /dev/null")
88-
run("MEANMLRM", "mlr --mmap stats1 -a mean -f x,y -g a,b ../data/big > /dev/null")
89-
run('CORRMLR', "mlr --no-mmap stats2 -a corr -f x,y -g a,b ../data/big > /dev/null")
90-
run('CORRMLRM', "mlr --mmap stats2 -a corr -f x,y -g a,b ../data/big > /dev/null")
91-
run('LINREGMLR', "mlr --no-mmap stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big > /dev/null")
92-
run('LINREGMLRM', "mlr --mmap stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big > /dev/null")
97+
run("MEANMLR", "mlr --no-mmap stats1 -a mean -f x,y -g a,b ../data/big.dkvp > /dev/null")
98+
run("MEANMLRM", "mlr --mmap stats1 -a mean -f x,y -g a,b ../data/big.dkvp > /dev/null")
99+
run('CORRMLR', "mlr --no-mmap stats2 -a corr -f x,y -g a,b ../data/big.dkvp > /dev/null")
100+
run('CORRMLRM', "mlr --mmap stats2 -a corr -f x,y -g a,b ../data/big.dkvp > /dev/null")
101+
run('LINREGMLR', "mlr --no-mmap stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.dkvp > /dev/null")
102+
run('LINREGMLRM', "mlr --mmap stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.dkvp > /dev/null")
93103
puts
94104

95-
run("MEANMLRV", "mlr --no-mmap --csv stats1 -a mean -f x,y -g a,b ../data/big.csv > /dev/null")
96-
run("MEANMLRVM", "mlr --mmap --csv stats1 -a mean -f x,y -g a,b ../data/big.csv > /dev/null")
97-
run('CORRMLRV', "mlr --no-mmap --csv stats2 -a corr -f x,y -g a,b ../data/big.csv > /dev/null")
98-
run('CORRMLRVM', "mlr --mmap --csv stats2 -a corr -f x,y -g a,b ../data/big.csv > /dev/null")
99-
run('LINREGMLRV', "mlr --no-mmap --csv stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.csv > /dev/null")
100-
run('LINREGMLRVM', "mlr --mmap --csv stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.csv > /dev/null")
105+
run("MEANMLRV", "mlr --no-mmap --csv --rs lf stats1 -a mean -f x,y -g a,b ../data/big.csv > /dev/null")
106+
run("MEANMLRVM", "mlr --mmap --csv --rs lf stats1 -a mean -f x,y -g a,b ../data/big.csv > /dev/null")
107+
run('CORRMLRV', "mlr --no-mmap --csv --rs lf stats2 -a corr -f x,y -g a,b ../data/big.csv > /dev/null")
108+
run('CORRMLRVM', "mlr --mmap --csv --rs lf stats2 -a corr -f x,y -g a,b ../data/big.csv > /dev/null")
109+
run('LINREGMLRV', "mlr --no-mmap --csv --rs lf stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.csv > /dev/null")
110+
run('LINREGMLRVM', "mlr --mmap --csv --rs lf stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.csv > /dev/null")
111+
run("MEANMLRV", "mlr --no-mmap --csvlite stats1 -a mean -f x,y -g a,b ../data/big.csv > /dev/null")
112+
run("MEANMLRVM", "mlr --mmap --csvlite stats1 -a mean -f x,y -g a,b ../data/big.csv > /dev/null")
113+
run('CORRMLRV', "mlr --no-mmap --csvlite stats2 -a corr -f x,y -g a,b ../data/big.csv > /dev/null")
114+
run('CORRMLRVM', "mlr --mmap --csvlite stats2 -a corr -f x,y -g a,b ../data/big.csv > /dev/null")
115+
run('LINREGMLRV', "mlr --no-mmap --csvlite stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.csv > /dev/null")
116+
run('LINREGMLRVM', "mlr --mmap --csvlite stats2 -a linreg-ols,linreg-pca -f x,y -g a,b ../data/big.csv > /dev/null")
101117
puts
102118
puts
103119
puts
104120

105-
run("sortsort1", "sort -t= -k 1,2 ../data/big > /dev/null")
106-
run("SORTMLR1", "mlr --no-mmap sort -f a,b ../data/big > /dev/null")
107-
run("SORTMLR1M", "mlr --mmap sort -f a,b ../data/big > /dev/null")
121+
run("sortsort1", "sort -t= -k 1,2 ../data/big.dkvp > /dev/null")
122+
run("SORTMLR1", "mlr --no-mmap sort -f a,b ../data/big.dkvp > /dev/null")
123+
run("SORTMLR1M", "mlr --mmap sort -f a,b ../data/big.dkvp > /dev/null")
108124
puts
109125

110-
run("sortsort2", "sort -t, -k 4,5 ../data/big > /dev/null")
111-
run("SORTMLR2", "mlr --no-mmap sort -n x,y ../data/big > /dev/null")
112-
run("SORTMLR2M", "mlr --mmap sort -n x,y ../data/big > /dev/null")
126+
run("sortsort2", "sort -t, -k 4,5 ../data/big.dkvp > /dev/null")
127+
run("SORTMLR2", "mlr --no-mmap sort -n x,y ../data/big.dkvp > /dev/null")
128+
run("SORTMLR2M", "mlr --mmap sort -n x,y ../data/big.dkvp > /dev/null")
113129
puts
114130
puts
115131

116132
run("sortsort1v", "sort -t, -k 1,4 ../data/big.csv > /dev/null")
117-
run("SORTMLR1V", "mlr --no-mmap --csv sort -f a,x ../data/big.csv > /dev/null")
118-
run("SORTMLR1VM", "mlr --mmap --csv sort -f a,x ../data/big.csv > /dev/null")
133+
run("SORTMLR1V", "mlr --no-mmap --csv --rs lf sort -f a,x ../data/big.csv > /dev/null")
134+
run("SORTMLR1VM", "mlr --mmap --csv --rs lf sort -f a,x ../data/big.csv > /dev/null")
135+
run("SORTMLR1V", "mlr --no-mmap --csvlite sort -f a,x ../data/big.csv > /dev/null")
136+
run("SORTMLR1VM", "mlr --mmap --csvlite sort -f a,x ../data/big.csv > /dev/null")
119137
puts
120138

121139
run("sortsort2v", "sort -t, -n -k 4,5 ../data/big.csv > /dev/null")
122-
run("SORTMLR2V", "mlr --no-mmap --csv sort -n x,y ../data/big.csv > /dev/null")
123-
run("SORTMLR2VM", "mlr --mmap --csv sort -n x,y ../data/big.csv > /dev/null")
140+
run("SORTMLR2V", "mlr --no-mmap --csv --rs lf sort -n x,y ../data/big.csv > /dev/null")
141+
run("SORTMLR2VM", "mlr --mmap --csv --rs lf sort -n x,y ../data/big.csv > /dev/null")
142+
run("SORTMLR2V", "mlr --no-mmap --csvlite sort -n x,y ../data/big.csv > /dev/null")
143+
run("SORTMLR2VM", "mlr --mmap --csvlite sort -n x,y ../data/big.csv > /dev/null")

0 commit comments

Comments
 (0)