Ahmet Göksu pushed to branch GSoC-2023-Ahmet at FreeType / FreeType
Commits:
-
35531481
by goksu at 2023-08-18T02:04:38+03:00
3 changed files:
Changes:
... | ... | @@ -3,7 +3,7 @@ FTBENCH_DIR = $(TOP_DIR)/src/tools/ftbench |
3 | 3 | FTBENCH_SRC = $(FTBENCH_DIR)/ftbench.c
|
4 | 4 | FTBENCH_OBJ = $(OBJ_DIR)/bench.$(SO)
|
5 | 5 | FTBENCH_BIN = $(OBJ_DIR)/bench$E
|
6 | -FTBENCH_FLAG ?= -c 1000 -w 100
|
|
6 | +FTBENCH_FLAG ?= -c 550 -w 50
|
|
7 | 7 | INCLUDES = $(TOP_DIR)/include
|
8 | 8 | FONTS = $(wildcard $(FTBENCH_DIR)/fonts/*.ttf)
|
9 | 9 | BASELINE_DIR = $(OBJ_DIR)/baseline/
|
... | ... | @@ -260,11 +260,18 @@ |
260 | 260 | #define TIMER_GET( timer ) ( timer )->total
|
261 | 261 | #define TIMER_RESET( timer ) ( timer )->total = 0
|
262 | 262 | |
263 | +#define CHUNK_SIZE 100
|
|
263 | 264 | |
265 | +int compare(const void* a, const void* b) {
|
|
266 | + if (*(double*)a > *(double*)b) return 1;
|
|
267 | + if (*(double*)a < *(double*)b) return -1;
|
|
268 | + return 0;
|
|
269 | +}
|
|
264 | 270 | /*
|
265 | 271 | * Bench code
|
266 | 272 | */
|
267 | 273 | |
274 | + |
|
268 | 275 | static void
|
269 | 276 | benchmark( FT_Face face,
|
270 | 277 | btest_t* test,
|
... | ... | @@ -276,6 +283,8 @@ |
276 | 283 | int n, done;
|
277 | 284 | btimer_t timer, elapsed;
|
278 | 285 |
|
286 | + int NUM_CHUNKS = max_iter / CHUNK_SIZE;
|
|
287 | + double results[NUM_CHUNKS];
|
|
279 | 288 | |
280 | 289 | if ( test->cache_first )
|
281 | 290 | {
|
... | ... | @@ -283,39 +292,59 @@ |
283 | 292 | test->bench( &timer, face, test->user_data );
|
284 | 293 | }
|
285 | 294 | |
295 | + // Initial warm-up
|
|
296 | + for (n = 0; n < warmup; n++) {
|
|
297 | + test->bench(&timer, face, test->user_data);
|
|
298 | + }
|
|
299 | + |
|
286 | 300 | printf( " %-25s ", test->title );
|
287 | 301 | fflush( stdout );
|
288 | 302 | |
289 | - TIMER_RESET( &timer );
|
|
290 | - TIMER_RESET( &elapsed );
|
|
291 | - |
|
292 | - int is_warmup = 1;
|
|
293 | 303 | |
294 | - for ( n = 0, done = 0; !max_iter || n < max_iter; n++ )
|
|
295 | - {
|
|
296 | - if ( is_warmup && n == warmup ){
|
|
297 | - is_warmup = 0;
|
|
304 | + for (int chunk = 0; chunk < NUM_CHUNKS; chunk++) {
|
|
298 | 305 | TIMER_RESET( &timer );
|
299 | 306 | TIMER_RESET( &elapsed );
|
300 | - }
|
|
301 | -
|
|
302 | 307 | |
303 | - TIMER_START( &elapsed );
|
|
308 | + // Execute a chunk of iterations
|
|
309 | + for (n = 0, done = 0; n < CHUNK_SIZE; n++) {
|
|
310 | + TIMER_START( &elapsed );
|
|
311 | + done += test->bench( &timer, face, test->user_data );
|
|
312 | + TIMER_STOP( &elapsed );
|
|
304 | 313 | |
305 | - done += test->bench( &timer, face, test->user_data );
|
|
314 | +
|
|
315 | + }
|
|
316 | + if (TIMER_GET( &elapsed ) > 1E6 * max_time) {
|
|
317 | + //break;
|
|
318 | + }
|
|
319 | + results[chunk] = TIMER_GET( &timer );
|
|
320 | + }
|
|
306 | 321 | |
307 | - TIMER_STOP( &elapsed );
|
|
322 | + // Sort results for IQR calculation
|
|
323 | + qsort(results, NUM_CHUNKS, sizeof(double), compare);
|
|
308 | 324 | |
309 | -
|
|
310 | - if (!is_warmup && TIMER_GET( &elapsed ) > 1E6 * max_time )
|
|
311 | - break;
|
|
325 | + double q1 = results[NUM_CHUNKS / 4];
|
|
326 | + double q3 = results[3 * NUM_CHUNKS / 4];
|
|
327 | + double iqr = q3 - q1;
|
|
328 | + double lower_bound = q1 - 1.5 * iqr;
|
|
329 | + double upper_bound = q3 + 1.5 * iqr;
|
|
330 | + |
|
331 | + double total_time = 0.0;
|
|
332 | + int valid_chunks = 0;
|
|
333 | + |
|
334 | + for (int chunk = 0; chunk < NUM_CHUNKS; chunk++) {
|
|
335 | + if (results[chunk] >= lower_bound && results[chunk] <= upper_bound) {
|
|
336 | + total_time += results[chunk];
|
|
337 | + valid_chunks++;
|
|
338 | + }
|
|
312 | 339 | }
|
313 | 340 | |
314 | - if ( done )
|
|
315 | - printf( "%10.1f microseconds %10d done\n",
|
|
316 | - TIMER_GET( &timer ), done );
|
|
317 | - else
|
|
318 | - printf( "no error-free calls\n" );
|
|
341 | + double average_time = total_time / valid_chunks;
|
|
342 | + |
|
343 | + |
|
344 | +
|
|
345 | + printf( "%10.1f microseconds %10d done\n",
|
|
346 | + average_time, done );
|
|
347 | +
|
|
319 | 348 | }
|
320 | 349 | |
321 | 350 |
... | ... | @@ -55,6 +55,9 @@ def main(): |
55 | 55 | |
56 | 56 | generate_info_table(html_file, baseline_info, benchmark_info)
|
57 | 57 | |
58 | + # Generate total results table
|
|
59 | + generate_total_results_table(html_file, BASELINE_DIR, BENCHMARK_DIR)
|
|
60 | +
|
|
58 | 61 | # Generate results tables
|
59 | 62 | for filename in os.listdir(BASELINE_DIR):
|
60 | 63 | if filename.endswith(".txt") and not filename == "info.txt":
|
... | ... | @@ -64,6 +67,8 @@ def main(): |
64 | 67 | generate_results_table(
|
65 | 68 | html_file, baseline_results, benchmark_results, filename
|
66 | 69 | )
|
70 | +
|
|
71 | +
|
|
67 | 72 | write_to_html(html_file, "<center>Freetype Benchmark</center>\n")
|
68 | 73 | write_to_html(html_file, "</body>\n</html>\n")
|
69 | 74 | |
... | ... | @@ -103,7 +108,95 @@ def generate_info_table(html_file, baseline_info, benchmark_info): |
103 | 108 | ),
|
104 | 109 | )
|
105 | 110 | write_to_html(html_file, "</table><br/>")
|
106 | - write_to_html(html_file, "* Cumulative time for iterations which is better in smaller values<br/>\n")
|
|
111 | + write_to_html(html_file, "<p>* Average time for all iterations. Smaller values are better.</p>")
|
|
112 | + write_to_html(html_file, "<p>** N count in (x | y) format is for showing baseline and benchmark N counts seperately when they differs.</p>")
|
|
113 | +
|
|
114 | + |
|
115 | +def generate_total_results_table(html_file, baseline_dir, benchmark_dir):
|
|
116 | + """Prepare total results table for html"""
|
|
117 | +
|
|
118 | + # This dictionary will store aggregated results.
|
|
119 | + test_results = {test: {"baseline": 0, "benchmark": 0, "n_baseline": 0, "n_benchmark": 0} for test in [
|
|
120 | + "Load", "Load_Advances (Normal)", "Load_Advances (Fast)", "Load_Advances (Unscaled)", "Render",
|
|
121 | + "Get_Glyph", "Get_Char_Index", "Iterate CMap", "New_Face", "Embolden", "Stroke", "Get_BBox",
|
|
122 | + "Get_CBox", "New_Face & load glyph(s)"
|
|
123 | + ]}
|
|
124 | +
|
|
125 | + for filename in os.listdir(baseline_dir):
|
|
126 | + if filename.endswith(".txt") and not filename == "info.txt":
|
|
127 | +
|
|
128 | + baseline_results = read_file(os.path.join(baseline_dir, filename))
|
|
129 | + benchmark_results = read_file(os.path.join(benchmark_dir, filename))
|
|
130 | +
|
|
131 | + for baseline_line, benchmark_line in zip(baseline_results, benchmark_results):
|
|
132 | + if baseline_line.startswith(" "):
|
|
133 | + baseline_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line)
|
|
134 | + benchmark_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", benchmark_line)
|
|
135 | +
|
|
136 | + if baseline_match and benchmark_match:
|
|
137 | + test = baseline_match.group(1).strip()
|
|
138 | + baseline_value = float(baseline_match.group(2))
|
|
139 | + benchmark_value = float(benchmark_match.group(2))
|
|
140 | + baseline_n = int(baseline_match.group(3))
|
|
141 | + benchmark_n = int(benchmark_match.group(3))
|
|
142 | +
|
|
143 | + # Aggregate the results
|
|
144 | + if test in test_results:
|
|
145 | + test_results[test]["baseline"] += baseline_value
|
|
146 | + test_results[test]["benchmark"] += benchmark_value
|
|
147 | + test_results[test]["n_baseline"] += baseline_n
|
|
148 | + test_results[test]["n_benchmark"] += benchmark_n
|
|
149 | +
|
|
150 | + # Writing to HTML
|
|
151 | + write_to_html(html_file, "<h2>Total Results</h2>\n")
|
|
152 | + write_to_html(html_file, '<table border="1">\n')
|
|
153 | + write_to_html(
|
|
154 | + html_file,
|
|
155 | + '<tr><th>Test</th><th>N</th><th>Baseline (µs)</th>\
|
|
156 | + <th>Benchmark (µs)</th><th>Difference (%)</th></tr>\n'
|
|
157 | + )
|
|
158 | + |
|
159 | + total_baseline = total_benchmark = total_diff = total_n_baseline = total_n_benchmark = 0
|
|
160 | +
|
|
161 | + for test, values in test_results.items():
|
|
162 | + baseline = values["baseline"]
|
|
163 | + benchmark = values["benchmark"]
|
|
164 | + n_baseline = values["n_baseline"]
|
|
165 | + n_benchmark = values["n_benchmark"]
|
|
166 | +
|
|
167 | + n_display = f"{n_baseline} | {n_benchmark}" if n_baseline != n_benchmark else str(n_baseline)
|
|
168 | +
|
|
169 | + diff = ((baseline - benchmark) / baseline) * 100
|
|
170 | + |
|
171 | + # Calculate for total row
|
|
172 | + total_baseline += baseline
|
|
173 | + total_benchmark += benchmark
|
|
174 | + total_n_baseline += n_baseline
|
|
175 | + total_n_benchmark += n_benchmark
|
|
176 | +
|
|
177 | + # Check which value is smaller for color highlighting
|
|
178 | + baseline_color = "highlight" if baseline <= benchmark else ""
|
|
179 | + benchmark_color = "highlight" if benchmark <= baseline else ""
|
|
180 | + |
|
181 | + write_to_html(
|
|
182 | + html_file,
|
|
183 | + f'<tr><td class="col1">{test}</td><td>{n_display}</td>\
|
|
184 | + <td class="{baseline_color}">{baseline:.0f}</td>\
|
|
185 | + <td class="{benchmark_color}">{benchmark:.0f}</td><td>{diff:.1f}</td></tr>\n'
|
|
186 | + )
|
|
187 | + |
|
188 | + total_diff = ((total_baseline - total_benchmark) / total_baseline) * 100
|
|
189 | + total_n_display = f"{total_n_baseline} | {total_n_benchmark}" if total_n_baseline != total_n_benchmark else str(total_n_baseline)
|
|
190 | +
|
|
191 | + write_to_html(
|
|
192 | + html_file,
|
|
193 | + f'<tr><td class="col1">TOTAL</td><td class="col1">{total_n_display}</td>\
|
|
194 | + <td class="col1">{total_baseline:.0f}</td><td class="col1">{total_benchmark:.0f}</td>\
|
|
195 | + <td class="col1">{total_diff:.1f}</td></tr>\n'
|
|
196 | + )
|
|
197 | +
|
|
198 | + write_to_html(html_file, "</table><br/>\n")
|
|
199 | + |
|
107 | 200 | |
108 | 201 | |
109 | 202 | def generate_results_table(html_file, baseline_results, benchmark_results, filename):
|
... | ... | @@ -114,7 +207,7 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen |
114 | 207 | if line.startswith("ftbench results for font")
|
115 | 208 | ][0]
|
116 | 209 | |
117 | - write_to_html(html_file, "<h2>Results for {}</h2>\n".format(fontname))
|
|
210 | + write_to_html(html_file, "<h3>Results for {}</h2>\n".format(fontname))
|
|
118 | 211 | write_to_html(html_file, '<table border="1">\n')
|
119 | 212 | write_to_html(
|
120 | 213 | html_file,
|
... | ... | @@ -185,7 +278,7 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen |
185 | 278 | percentage_diff,
|
186 | 279 | ),
|
187 | 280 | )
|
188 | - |
|
281 | +
|
|
189 | 282 | write_to_html(
|
190 | 283 | html_file,
|
191 | 284 | '<tr><td class="col1">TOTAL</td><td class="col1">{}</td>\
|