1- /** dude, is my code constant time?
2- *
3- * This file measures the execution time of a given function many times with
4- * different inputs and performs a Welch's t-test to determine if the function
5- * runs in constant time or not. This is essentially leakage detection, and
6- * not a timing attack.
7- *
8- * Notes:
9- *
10- * - the execution time distribution tends to be skewed towards large
11- * timings, leading to a fat right tail. Most executions take little time,
12- * some of them take a lot. We try to speed up the test process by
13- * throwing away those measurements with large cycle count. (For example,
14- * those measurements could correspond to the execution being interrupted
15- * by the OS.) Setting a threshold value for this is not obvious; we just
16- * keep the x% percent fastest timings, and repeat for several values of x.
17- *
18- * - the previous observation is highly heuristic. We also keep the uncropped
19- * measurement time and do a t-test on that.
20- *
21- * - we also test for unequal variances (second order test), but this is
22- * probably redundant since we're doing as well a t-test on cropped
23- * measurements (non-linear transform)
24- *
25- * - as long as any of the different test fails, the code will be deemed
26- * variable time.
27- */
28-
29- #include <assert.h>
30- #include <math.h>
31- #include <stdint.h>
32- #include <stdio.h>
33- #include <stdlib.h>
34- #include <string.h>
35-
36- #include "../console.h"
37- #include "../random.h"
38-
39- #include "constant.h"
40- #include "fixture.h"
41- #include "ttest.h"
42-
43- #define ENOUGH_MEASURE 10000
44- #define TEST_TRIES 10
45-
46- static t_context_t * t ;
47-
48- /* threshold values for Welch's t-test */
49- enum {
50- t_threshold_bananas = 500 , /* Test failed with overwhelming probability */
51- t_threshold_moderate = 10 , /* Test failed */
52- };
53-
54- static void __attribute__((noreturn )) die (void )
55- {
56- exit (111 );
57- }
58-
59- static void differentiate (int64_t * exec_times ,
60- const int64_t * before_ticks ,
61- const int64_t * after_ticks )
62- {
63- for (size_t i = 0 ; i < N_MEASURES ; i ++ )
64- exec_times [i ] = after_ticks [i ] - before_ticks [i ];
65- }
66-
67- static void update_statistics (const int64_t * exec_times , uint8_t * classes )
68- {
69- for (size_t i = 0 ; i < N_MEASURES ; i ++ ) {
70- int64_t difference = exec_times [i ];
71- /* CPU cycle counter overflowed or dropped measurement */
72- if (difference <= 0 )
73- continue ;
74-
75- /* do a t-test on the execution time */
76- t_push (t , difference , classes [i ]);
77- }
78- }
79-
80- static bool report (void )
81- {
82- double max_t = fabs (t_compute (t ));
83- double number_traces_max_t = t -> n [0 ] + t -> n [1 ];
84- double max_tau = max_t / sqrt (number_traces_max_t );
85-
86- printf ("\033[A\033[2K" );
87- printf ("measure: %7.2lf M, " , (number_traces_max_t / 1e6 ));
88- if (number_traces_max_t < ENOUGH_MEASURE ) {
89- printf ("not enough measurements (%.0f still to go).\n" ,
90- ENOUGH_MEASURE - number_traces_max_t );
91- return false;
92- }
93-
94- /* max_t: the t statistic value
95- * max_tau: a t value normalized by sqrt(number of measurements).
96- * this way we can compare max_tau taken with different
97- * number of measurements. This is sort of "distance
98- * between distributions", independent of number of
99- * measurements.
100- * (5/tau)^2: how many measurements we would need to barely
101- * detect the leak, if present. "barely detect the
102- * leak" = have a t value greater than 5.
103- */
104- printf ("max t: %+7.2f, max tau: %.2e, (5/tau)^2: %.2e.\n" , max_t , max_tau ,
105- (double ) (5 * 5 ) / (double ) (max_tau * max_tau ));
106-
107- /* Definitely not constant time */
108- if (max_t > t_threshold_bananas )
109- return false;
110-
111- /* Probably not constant time. */
112- if (max_t > t_threshold_moderate )
113- return false;
114-
115- /* For the moment, maybe constant time. */
116- return true;
117- }
118-
119- static bool doit (int mode )
120- {
121- int64_t * before_ticks = calloc (N_MEASURES + 1 , sizeof (int64_t ));
122- int64_t * after_ticks = calloc (N_MEASURES + 1 , sizeof (int64_t ));
123- int64_t * exec_times = calloc (N_MEASURES , sizeof (int64_t ));
124- uint8_t * classes = calloc (N_MEASURES , sizeof (uint8_t ));
125- uint8_t * input_data = calloc (N_MEASURES * CHUNK_SIZE , sizeof (uint8_t ));
126-
127- if (!before_ticks || !after_ticks || !exec_times || !classes ||
128- !input_data ) {
129- die ();
130- }
131-
132- prepare_inputs (input_data , classes );
133-
134- bool ret = measure (before_ticks , after_ticks , input_data , mode );
135- differentiate (exec_times , before_ticks , after_ticks );
136- update_statistics (exec_times , classes );
137- ret &= report ();
138-
139- free (before_ticks );
140- free (after_ticks );
141- free (exec_times );
142- free (classes );
143- free (input_data );
144-
145- return ret ;
146- }
147-
148- static void init_once (void )
149- {
150- init_dut ();
151- t_init (t );
152- }
153-
154- static bool test_const (char * text , int mode )
155- {
156- bool result = false;
157- t = malloc (sizeof (t_context_t ));
158-
159- for (int cnt = 0 ; cnt < TEST_TRIES ; ++ cnt ) {
160- printf ("Testing %s...(%d/%d)\n\n" , text , cnt , TEST_TRIES );
161- init_once ();
162- for (int i = 0 ; i < ENOUGH_MEASURE / (N_MEASURES - DROP_SIZE * 2 ) + 1 ;
163- ++ i )
164- result = doit (mode );
165- printf ("\033[A\033[2K\033[A\033[2K" );
166- if (result )
167- break ;
168- }
169- free (t );
170- return result ;
171- }
172-
173- #define DUT_FUNC_IMPL (op ) \
174- bool is_##op##_const(void) \
175- { \
176- return test_const(#op, DUT(op)); \
177- }
178-
179- #define _ (x ) DUT_FUNC_IMPL(x)
180- DUT_FUNCS
181- #undef _
1+ /** dude, is my code constant time?
2+ *
3+ * This file measures the execution time of a given function many times with
4+ * different inputs and performs a Welch's t-test to determine if the function
5+ * runs in constant time or not. This is essentially leakage detection, and
6+ * not a timing attack.
7+ *
8+ * Notes:
9+ *
10+ * - the execution time distribution tends to be skewed towards large
11+ * timings, leading to a fat right tail. Most executions take little time,
12+ * some of them take a lot. We try to speed up the test process by
13+ * throwing away those measurements with large cycle count. (For example,
14+ * those measurements could correspond to the execution being interrupted
15+ * by the OS.) Setting a threshold value for this is not obvious; we just
16+ * keep the x% percent fastest timings, and repeat for several values of x.
17+ *
18+ * - the previous observation is highly heuristic. We also keep the uncropped
19+ * measurement time and do a t-test on that.
20+ *
21+ * - we also test for unequal variances (second order test), but this is
22+ * probably redundant since we're doing as well a t-test on cropped
23+ * measurements (non-linear transform)
24+ *
25+ * - as long as any of the different test fails, the code will be deemed
26+ * variable time.
27+ */
28+
29+ #include <assert.h>
30+ #include <math.h>
31+ #include <stdint.h>
32+ #include <stdio.h>
33+ #include <stdlib.h>
34+ #include <string.h>
35+
36+ #include "../console.h"
37+ #include "../random.h"
38+
39+ #include "constant.h"
40+ #include "fixture.h"
41+ #include "ttest.h"
42+
43+ #define ENOUGH_MEASURE 10000
44+ #define TEST_TRIES 10
45+
46+ /* Number of percentiles to calculate */
47+ #define NUM_PERCENTILES (100)
48+ #define DUDECT_TESTS (NUM_PERCENTILES + 1)
49+
50+ static t_context_t * ctxs [DUDECT_TESTS ];
51+
52+ /* threshold values for Welch's t-test */
53+ enum {
54+ t_threshold_bananas = 500 , /* Test failed with overwhelming probability */
55+ t_threshold_moderate = 10 , /* Test failed */
56+ };
57+
58+ static void __attribute__((noreturn )) die (void )
59+ {
60+ exit (111 );
61+ }
62+
63+ static int64_t percentile (const int64_t * a_sorted , double which , size_t size )
64+ {
65+ assert (which >= 0 && which <= 1.0 );
66+ size_t pos = (size_t ) (which * size );
67+ return a_sorted [pos ];
68+ }
69+
70+ /* leverages the fact that comparison expressions return 1 or 0. */
71+ static int cmp (const void * aa , const void * bb )
72+ {
73+ int64_t a = * (const int64_t * ) aa , b = * (const int64_t * ) bb ;
74+ return (a > b ) - (a < b );
75+ }
76+
77+ /* This function is used to set different thresholds for cropping measurements.
78+ * To filter out slow measurements, we keep only the fastest ones by a
79+ * complementary exponential decay scale as thresholds for cropping
80+ * measurements: threshold(x) = 1 - 0.5^(10 * x / N_MEASURES), where x is the
81+ * counter of the measurement.
82+ */
83+ static void prepare_percentiles (int64_t * exec_times , int64_t * percentiles )
84+ {
85+ qsort (exec_times , N_MEASURES , sizeof (int64_t ), cmp );
86+
87+ for (size_t i = 0 ; i < NUM_PERCENTILES ; i ++ ) {
88+ percentiles [i ] = percentile (
89+ exec_times , 1 - (pow (0.5 , 10 * (double ) (i + 1 ) / NUM_PERCENTILES )),
90+ N_MEASURES );
91+ }
92+ }
93+
94+ static void differentiate (int64_t * exec_times ,
95+ const int64_t * before_ticks ,
96+ const int64_t * after_ticks )
97+ {
98+ for (size_t i = 0 ; i < N_MEASURES ; i ++ )
99+ exec_times [i ] = after_ticks [i ] - before_ticks [i ];
100+ }
101+
102+ static void update_statistics (const int64_t * exec_times ,
103+ uint8_t * classes ,
104+ int64_t * percentiles )
105+ {
106+ for (size_t i = 0 ; i < N_MEASURES ; i ++ ) {
107+ int64_t difference = exec_times [i ];
108+ /* CPU cycle counter overflowed or dropped measurement */
109+ if (difference <= 0 )
110+ continue ;
111+
112+ /* do a t-test on the execution time */
113+ t_push (ctxs [0 ], difference , classes [i ]);
114+
115+ /* t-test on cropped execution times, for several cropping thresholds.
116+ */
117+ for (size_t j = 0 ; j < NUM_PERCENTILES ; j ++ ) {
118+ if (difference < percentiles [j ]) {
119+ t_push (ctxs [j + 1 ], difference , classes [i ]);
120+ }
121+ }
122+ }
123+ }
124+
125+ static t_context_t * max_test ()
126+ {
127+ size_t max_idx = 0 ;
128+ double max_t = 0.0f ;
129+ for (size_t i = 0 ; i < NUM_PERCENTILES + 1 ; i ++ ) {
130+ double t = fabs (t_compute (ctxs [i ]));
131+ if (t > max_t ) {
132+ max_t = t ;
133+ max_idx = i ;
134+ }
135+ }
136+ return ctxs [max_idx ];
137+ }
138+
139+ static bool report (void )
140+ {
141+ t_context_t * t = max_test ();
142+ double number_traces_max_t = t -> n [0 ] + t -> n [1 ];
143+
144+ printf ("\033[A\033[2K" );
145+ printf ("measure: %7.2lf M, " , (number_traces_max_t / 1e6 ));
146+ if (number_traces_max_t < ENOUGH_MEASURE ) {
147+ printf ("not enough measurements (%.0f still to go).\n" ,
148+ ENOUGH_MEASURE - number_traces_max_t );
149+ return false;
150+ }
151+
152+ double max_t = fabs (t_compute (t ));
153+ double max_tau = max_t / sqrt (number_traces_max_t );
154+
155+ /* max_t: the t statistic value
156+ * max_tau: a t value normalized by sqrt(number of measurements).
157+ * this way we can compare max_tau taken with different
158+ * number of measurements. This is sort of "distance
159+ * between distributions", independent of number of
160+ * measurements.
161+ * (5/tau)^2: how many measurements we would need to barely
162+ * detect the leak, if present. "barely detect the
163+ * leak" = have a t value greater than 5.
164+ */
165+ printf ("max t: %+7.2f, max tau: %.2e, (5/tau)^2: %.2e.\n" , max_t , max_tau ,
166+ (double ) (5 * 5 ) / (double ) (max_tau * max_tau ));
167+
168+ /* Definitely not constant time */
169+ if (max_t > t_threshold_bananas )
170+ return false;
171+
172+ /* Probably not constant time. */
173+ if (max_t > t_threshold_moderate )
174+ return false;
175+
176+ /* For the moment, maybe constant time. */
177+ return true;
178+ }
179+
180+ static bool doit (int mode )
181+ {
182+ int64_t * before_ticks = calloc (N_MEASURES + 1 , sizeof (int64_t ));
183+ int64_t * after_ticks = calloc (N_MEASURES + 1 , sizeof (int64_t ));
184+ int64_t * exec_times = calloc (N_MEASURES , sizeof (int64_t ));
185+ uint8_t * classes = calloc (N_MEASURES , sizeof (uint8_t ));
186+ uint8_t * input_data = calloc (N_MEASURES * CHUNK_SIZE , sizeof (uint8_t ));
187+ int64_t * percentiles = calloc (NUM_PERCENTILES , sizeof (int64_t ));
188+
189+ if (!before_ticks || !after_ticks || !exec_times || !classes ||
190+ !input_data ) {
191+ die ();
192+ }
193+
194+ prepare_inputs (input_data , classes );
195+
196+ bool ret = measure (before_ticks , after_ticks , input_data , mode );
197+ differentiate (exec_times , before_ticks , after_ticks );
198+ prepare_percentiles (exec_times , percentiles );
199+ update_statistics (exec_times , classes , percentiles );
200+ ret &= report ();
201+
202+ free (before_ticks );
203+ free (after_ticks );
204+ free (exec_times );
205+ free (classes );
206+ free (input_data );
207+ free (percentiles );
208+
209+ return ret ;
210+ }
211+
212+ static void init_once (void )
213+ {
214+ init_dut ();
215+ for (size_t i = 0 ; i < DUDECT_TESTS ; i ++ ) {
216+ ctxs [i ] = malloc (sizeof (t_context_t ));
217+ t_init (ctxs [i ]);
218+ }
219+ }
220+
221+ static bool test_const (char * text , int mode )
222+ {
223+ bool result = false;
224+
225+ for (int cnt = 0 ; cnt < TEST_TRIES ; ++ cnt ) {
226+ printf ("Testing %s...(%d/%d)\n\n" , text , cnt , TEST_TRIES );
227+ init_once ();
228+ for (int i = 0 ; i < ENOUGH_MEASURE / (N_MEASURES - DROP_SIZE * 2 ) + 1 ;
229+ ++ i )
230+ result = doit (mode );
231+ printf ("\033[A\033[2K\033[A\033[2K" );
232+ if (result )
233+ break ;
234+ }
235+
236+ for (size_t i = 0 ; i < DUDECT_TESTS ; i ++ ) {
237+ free (ctxs [i ]);
238+ }
239+
240+ return result ;
241+ }
242+
243+ #define DUT_FUNC_IMPL (op ) \
244+ bool is_##op##_const(void) \
245+ { \
246+ return test_const(#op, DUT(op)); \
247+ }
248+
249+ #define _ (x ) DUT_FUNC_IMPL(x)
250+ DUT_FUNCS
251+ #undef _
0 commit comments