|
1
|
|
-"""This module converts the benchmark results into a HTML file."""
|
|
|
1
|
+"""This script generates a HTML file from the results of ftbench"""
|
|
|
2
|
+# Ahmet Goksu ahmet@goksu.in ahmet.goksu.in
|
|
|
3
|
+
|
|
2
|
4
|
import os
|
|
3
|
5
|
import re
|
|
4
|
6
|
import sys
|
|
5
|
7
|
|
|
6
|
|
-# Create the HTML file
|
|
7
|
8
|
PROJECT_ROOT = sys.argv[1]
|
|
8
|
9
|
BENCHMARK_HTML = os.path.join(PROJECT_ROOT, "benchmark.html")
|
|
9
|
|
-
|
|
10
|
|
-# GitLab URL
|
|
11
|
10
|
GITLAB_URL = "https://gitlab.freedesktop.org/freetype/freetype/-/commit/"
|
|
12
|
|
-
|
|
13
|
|
-# CSS style
|
|
14
|
11
|
CSS_STYLE = """
|
|
15
|
12
|
<style>
|
|
16
|
13
|
table {
|
| ... |
... |
@@ -36,157 +33,166 @@ CSS_STYLE = """ |
|
36
|
33
|
}
|
|
37
|
34
|
</style>
|
|
38
|
35
|
"""
|
|
39
|
|
-
|
|
40
|
|
-# Directories
|
|
41
|
36
|
BASELINE_DIR = os.path.join(PROJECT_ROOT, "baseline")
|
|
42
|
37
|
BENCHMARK_DIR = os.path.join(PROJECT_ROOT, "benchmark")
|
|
43
|
38
|
|
|
44
|
|
-# Open HTML file for writing and write the header
|
|
45
|
|
-with open(BENCHMARK_HTML, "w") as html_file:
|
|
46
|
|
- html_file.write(
|
|
47
|
|
- f"<html>\n\
|
|
48
|
|
- <head>\n\
|
|
49
|
|
- <title>Benchmark Results</title>\n\
|
|
50
|
|
- {CSS_STYLE}\
|
|
51
|
|
- </head>\n\
|
|
52
|
|
- <body>\n\
|
|
53
|
|
- <h1>Benchmark Results</h1>\n"
|
|
54
|
|
- )
|
|
|
39
|
+def main():
|
|
|
40
|
+ """Entry point for the script"""
|
|
|
41
|
+ with open(BENCHMARK_HTML, "w") as html_file:
|
|
|
42
|
+ write_to_html(html_file, "<html>\n<head>\n")
|
|
|
43
|
+ write_to_html(html_file, CSS_STYLE)
|
|
|
44
|
+ write_to_html(html_file, "</head>\n<body>\n")
|
|
55
|
45
|
|
|
56
|
|
- # If it's the info file, we want to handle it differently
|
|
57
|
|
- with open(os.path.join(BASELINE_DIR, "info.txt"), "r") as f:
|
|
58
|
|
- baseline_info = f.readlines()
|
|
59
|
|
- with open(os.path.join(BENCHMARK_DIR, "info.txt"), "r") as f:
|
|
60
|
|
- benchmark_info = f.readlines()
|
|
|
46
|
+ baseline_info = parse_info_file(os.path.join(BASELINE_DIR, "info.txt"))
|
|
|
47
|
+ benchmark_info = parse_info_file(os.path.join(BENCHMARK_DIR, "info.txt"))
|
|
61
|
48
|
|
|
62
|
|
- # Check if commit ids are the same
|
|
63
|
|
- if baseline_info[1].strip() == benchmark_info[1].strip():
|
|
64
|
|
- html_file.write(
|
|
65
|
|
- '<h2 class="warning">Warning: Baseline and Benchmark have the same commit ID</h2>\n'
|
|
66
|
|
- )
|
|
|
49
|
+ if baseline_info[1].strip() == benchmark_info[1].strip():
|
|
|
50
|
+ write_to_html(
|
|
|
51
|
+ html_file,
|
|
|
52
|
+ '<h2 class="warning">Warning: Baseline and Benchmark have the same commit ID</h2>\n',
|
|
|
53
|
+ )
|
|
67
|
54
|
|
|
68
|
|
- baseline_info[1] = '<a href="">{}</a>\n'.format(
|
|
69
|
|
- GITLAB_URL, baseline_info[1].strip(), baseline_info[1][:8]
|
|
70
|
|
- )
|
|
|
55
|
+ generate_info_table(html_file, baseline_info, benchmark_info)
|
|
71
|
56
|
|
|
72
|
|
- benchmark_info[1] = '<a href="">{}</a>\n'.format(
|
|
73
|
|
- GITLAB_URL, benchmark_info[1].strip(), benchmark_info[1][:8]
|
|
74
|
|
- )
|
|
|
57
|
+ # Generate results tables
|
|
|
58
|
+ for filename in os.listdir(BASELINE_DIR):
|
|
|
59
|
+ if filename.endswith(".txt") and not filename == "info.txt":
|
|
|
60
|
+ baseline_results = read_file(os.path.join(BASELINE_DIR, filename))
|
|
|
61
|
+ benchmark_results = read_file(os.path.join(BENCHMARK_DIR, filename))
|
|
|
62
|
+
|
|
|
63
|
+ generate_results_table(
|
|
|
64
|
+ html_file, baseline_results, benchmark_results, filename
|
|
|
65
|
+ )
|
|
|
66
|
+
|
|
|
67
|
+ write_to_html(html_file, "</body>\n</html>\n")
|
|
|
68
|
+
|
|
|
69
|
+def write_to_html(html_file, content):
|
|
|
70
|
+ """Write content to html file"""
|
|
|
71
|
+ html_file.write(content)
|
|
|
72
|
+
|
|
|
73
|
+
|
|
|
74
|
+def read_file(file_path):
|
|
|
75
|
+ """Read file and return list of lines"""
|
|
|
76
|
+ with open(file_path, "r") as f:
|
|
|
77
|
+ return f.readlines()
|
|
|
78
|
+
|
|
|
79
|
+
|
|
|
80
|
+def parse_info_file(info_file):
|
|
|
81
|
+ """Get info from info.txt file and return as list"""
|
|
|
82
|
+ info = read_file(info_file)
|
|
|
83
|
+ info[1] = '<a href="">{}</a>\n'.format(GITLAB_URL, info[1].strip(), info[1][:8])
|
|
|
84
|
+ return info
|
|
75
|
85
|
|
|
76
|
|
- # Write info table
|
|
77
|
|
- html_file.write("<h2>Info</h2>\n")
|
|
78
|
|
- html_file.write('<table border="1">\n')
|
|
79
|
|
- html_file.write("<tr><th>Info</th><th>Baseline</th><th>Benchmark</th></tr>\n")
|
|
|
86
|
+
|
|
|
87
|
+def generate_info_table(html_file, baseline_info, benchmark_info):
|
|
|
88
|
+ """Prepare info table for html"""
|
|
|
89
|
+ write_to_html(html_file, "<h2>Info</h2>\n")
|
|
|
90
|
+ write_to_html(html_file, '<table border="1">\n')
|
|
|
91
|
+ write_to_html(
|
|
|
92
|
+ html_file, "<tr><th>Info</th><th>Baseline</th><th>Benchmark</th></tr>\n"
|
|
|
93
|
+ )
|
|
80
|
94
|
info_list = ["Parameters", "Commit ID", "Commit Date", "Branch"]
|
|
81
|
95
|
for info, baseline_line, benchmark_line in zip(
|
|
82
|
96
|
info_list, baseline_info, benchmark_info
|
|
83
|
97
|
):
|
|
84
|
|
- html_file.write(
|
|
|
98
|
+ write_to_html(
|
|
|
99
|
+ html_file,
|
|
85
|
100
|
'<tr><td class="col1">{}</td><td>{}</td><td>{}</td></tr>\n'.format(
|
|
86
|
101
|
info, baseline_line.strip(), benchmark_line.strip()
|
|
87
|
|
- )
|
|
|
102
|
+ ),
|
|
88
|
103
|
)
|
|
89
|
|
- html_file.write("</table><br/>")
|
|
90
|
|
- html_file.write("*Smaller values mean faster operation<br/>\n")
|
|
91
|
|
- # Traverse through the 'baseline' directory
|
|
92
|
|
- for filename in os.listdir(BASELINE_DIR):
|
|
93
|
|
- if filename != "info.txt":
|
|
94
|
|
- with open(os.path.join(BASELINE_DIR, filename), "r") as f:
|
|
95
|
|
- baseline_results = f.readlines()
|
|
96
|
|
- with open(os.path.join(BENCHMARK_DIR, filename), "r") as f:
|
|
97
|
|
- benchmark_results = f.readlines()
|
|
98
|
|
-
|
|
99
|
|
- # Get font name from within the .txt file
|
|
100
|
|
- for line in baseline_results:
|
|
101
|
|
- if line.startswith("ftbench results for font"):
|
|
102
|
|
- fontname = line.split("/")[-1]\
|
|
103
|
|
- .strip("'")[:-2]
|
|
104
|
|
-
|
|
105
|
|
- # Write test column headers
|
|
106
|
|
- html_file.write("<h2>Results for {}</h2>\n".format(fontname))
|
|
107
|
|
- html_file.write('<table border="1">\n')
|
|
108
|
|
- html_file.write(
|
|
109
|
|
- '<tr>\
|
|
110
|
|
- <th>Test</th>\
|
|
111
|
|
- <th>N</th>\
|
|
112
|
|
- <th><a href="">Baseline</a> (ms)</th>\
|
|
113
|
|
- <th><a href="">Benchmark</a> (ms)</th>\
|
|
114
|
|
- <th>Difference (%)</th>\
|
|
115
|
|
- </tr>\n'.format(
|
|
116
|
|
- os.path.join(BASELINE_DIR, fontname[:-4]),
|
|
117
|
|
- os.path.join(BENCHMARK_DIR, fontname[:-4]),
|
|
118
|
|
- )
|
|
|
104
|
+ write_to_html(html_file, "</table><br/>")
|
|
|
105
|
+ write_to_html(html_file, "*Smaller values mean faster operation<br/>\n")
|
|
|
106
|
+
|
|
|
107
|
+
|
|
|
108
|
+def generate_results_table(html_file, baseline_results, benchmark_results, filename):
|
|
|
109
|
+ """Prepare results table for html"""
|
|
|
110
|
+ fontname = [
|
|
|
111
|
+ line.split("/")[-1].strip("'")[:-2]
|
|
|
112
|
+ for line in baseline_results
|
|
|
113
|
+ if line.startswith("ftbench results for font")
|
|
|
114
|
+ ][0]
|
|
|
115
|
+
|
|
|
116
|
+ write_to_html(html_file, "<h2>Results for {}</h2>\n".format(fontname))
|
|
|
117
|
+ write_to_html(html_file, '<table border="1">\n')
|
|
|
118
|
+ write_to_html(
|
|
|
119
|
+ html_file,
|
|
|
120
|
+ '<tr><th>Test</th><th>N</th>\
|
|
|
121
|
+ <th><a href="">Baseline</a> (ms)</th>\
|
|
|
122
|
+ <th><a href="">Benchmark</a> (ms)</th>\
|
|
|
123
|
+ <th>Difference (%)</th></tr>\n'.format(
|
|
|
124
|
+ os.path.join(BASELINE_DIR, filename[:-4]),
|
|
|
125
|
+ os.path.join(BENCHMARK_DIR, filename[:-4]),
|
|
|
126
|
+ ),
|
|
|
127
|
+ )
|
|
|
128
|
+
|
|
|
129
|
+ total_n = total_time_baseline = total_time_benchmark = total_difference = 0
|
|
|
130
|
+
|
|
|
131
|
+ for baseline_line, benchmark_line in zip(baseline_results, benchmark_results):
|
|
|
132
|
+ if baseline_line.startswith(" "):
|
|
|
133
|
+ baseline_match = re.match(
|
|
|
134
|
+ r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", baseline_line
|
|
|
135
|
+ )
|
|
|
136
|
+ benchmark_match = re.match(
|
|
|
137
|
+ r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", benchmark_line
|
|
119
|
138
|
)
|
|
120
|
139
|
|
|
121
|
|
- # Write test results
|
|
122
|
|
- for baseline_line, benchmark_line in zip(
|
|
123
|
|
- baseline_results, benchmark_results
|
|
124
|
|
- ):
|
|
125
|
|
- # Check if line is a test result
|
|
126
|
|
- # Get results by Regex
|
|
127
|
|
- if baseline_line.startswith(" "):
|
|
128
|
|
- baseline_match = re.match(
|
|
129
|
|
- r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", baseline_line
|
|
130
|
|
- )
|
|
|
140
|
+ if baseline_match and benchmark_match:
|
|
|
141
|
+ baseline_value = float(baseline_match.group(2))
|
|
|
142
|
+ benchmark_value = float(benchmark_match.group(2))
|
|
|
143
|
+
|
|
|
144
|
+ percentage_diff = (
|
|
|
145
|
+ (baseline_value - benchmark_value) / baseline_value
|
|
|
146
|
+ ) * 100
|
|
|
147
|
+
|
|
|
148
|
+ baseline_n = baseline_match.group(3)
|
|
|
149
|
+ benchmark_n = benchmark_match.group(3)
|
|
|
150
|
+
|
|
|
151
|
+ n = (
|
|
|
152
|
+ baseline_n
|
|
|
153
|
+ if baseline_n == benchmark_n
|
|
|
154
|
+ else baseline_n + " / " + benchmark_n
|
|
|
155
|
+ )
|
|
131
|
156
|
|
|
132
|
|
- benchmark_match = re.match(
|
|
133
|
|
- r"\s+(.*?)\s+(\d+\.\d+)\s+ms\s+(\d+)\s", benchmark_line
|
|
|
157
|
+ total_n += int(baseline_n)
|
|
|
158
|
+ total_n += int(benchmark_n)
|
|
|
159
|
+ total_time_baseline += baseline_value
|
|
|
160
|
+ total_time_benchmark += benchmark_value
|
|
|
161
|
+ total_difference += percentage_diff
|
|
|
162
|
+
|
|
|
163
|
+ if baseline_value > benchmark_value:
|
|
|
164
|
+ write_to_html(
|
|
|
165
|
+ html_file,
|
|
|
166
|
+ '<tr><td class="col1">{}</td><td>{}</td>\
|
|
|
167
|
+ <td class="lowlight">{:.3f}</td><td class="highlight">{:.3f}</td><td>{:.1f}</td></tr>\n'.format(
|
|
|
168
|
+ baseline_match.group(1),
|
|
|
169
|
+ n,
|
|
|
170
|
+ baseline_value,
|
|
|
171
|
+ benchmark_value,
|
|
|
172
|
+ percentage_diff,
|
|
|
173
|
+ ),
|
|
|
174
|
+ )
|
|
|
175
|
+ else:
|
|
|
176
|
+ write_to_html(
|
|
|
177
|
+ html_file,
|
|
|
178
|
+ '<tr><td class="col1">{}</td><td>{}</td>\
|
|
|
179
|
+ <td class="highlight">{:.3f}</td><td class="lowlight">{:.3f}</td><td>{:.1f}</td></tr>\n'.format(
|
|
|
180
|
+ baseline_match.group(1),
|
|
|
181
|
+ n,
|
|
|
182
|
+ baseline_value,
|
|
|
183
|
+ benchmark_value,
|
|
|
184
|
+ percentage_diff,
|
|
|
185
|
+ ),
|
|
134
|
186
|
)
|
|
135
|
187
|
|
|
136
|
|
- if baseline_match and benchmark_match:
|
|
137
|
|
- baseline_value = float(baseline_match.group(2))
|
|
138
|
|
- benchmark_value = float(benchmark_match.group(2))
|
|
139
|
|
-
|
|
140
|
|
- # Calculate the percentage difference
|
|
141
|
|
- percentage_diff = (
|
|
142
|
|
- (baseline_value - benchmark_value) / baseline_value
|
|
143
|
|
- ) * 100
|
|
144
|
|
-
|
|
145
|
|
- # Get itaration number
|
|
146
|
|
- baseline_n = baseline_match.group(3)
|
|
147
|
|
- benchmark_n = benchmark_match.group(3)
|
|
148
|
|
-
|
|
149
|
|
- # Check if iteration number is the same
|
|
150
|
|
- if baseline_n == benchmark_n:
|
|
151
|
|
- total_n = baseline_n
|
|
152
|
|
- else:
|
|
153
|
|
- total_n = baseline_n +\
|
|
154
|
|
- " / " + benchmark_n
|
|
155
|
|
-
|
|
156
|
|
- # Highlight the faster value
|
|
157
|
|
- if baseline_value > benchmark_value:
|
|
158
|
|
- html_file.write(
|
|
159
|
|
- '<tr>\
|
|
160
|
|
- <td class="col1">{}</td>\
|
|
161
|
|
- <td>{}</td>\
|
|
162
|
|
- <td class="lowlight">{:.3f}</td>\
|
|
163
|
|
- <td class="highlight">{:.3f}</td>\
|
|
164
|
|
- <td>{:.1f}</td>\
|
|
165
|
|
- </tr>\n'.format(
|
|
166
|
|
- baseline_match.group(1),
|
|
167
|
|
- total_n,
|
|
168
|
|
- baseline_value,
|
|
169
|
|
- benchmark_value,
|
|
170
|
|
- percentage_diff,
|
|
171
|
|
- )
|
|
172
|
|
- )
|
|
173
|
|
- else:
|
|
174
|
|
- html_file.write(
|
|
175
|
|
- '<tr>\
|
|
176
|
|
- <td class="col1">{}</td>\
|
|
177
|
|
- <td>{}</td>\
|
|
178
|
|
- <td class="highlight">{:.3f}</td>\
|
|
179
|
|
- <td class="lowlight">{:.3f}</td>\
|
|
180
|
|
- <td>{:.1f}</td>\
|
|
181
|
|
- </tr>\n'.format(
|
|
182
|
|
- baseline_match.group(1),
|
|
183
|
|
- total_n,
|
|
184
|
|
- baseline_value,
|
|
185
|
|
- benchmark_value,
|
|
186
|
|
- percentage_diff,
|
|
187
|
|
- )
|
|
188
|
|
- )
|
|
189
|
|
-
|
|
190
|
|
- html_file.write("</table><br/>\n")
|
|
191
|
|
-
|
|
192
|
|
- html_file.write("<center>Freetype Benchmark</center></body>\n</html>\n") |
|
|
188
|
+ write_to_html(
|
|
|
189
|
+ html_file,
|
|
|
190
|
+ '<tr><td class="col1">TOTAL</td><td class="col1">{}</td>\
|
|
|
191
|
+ <td class="col1">{:.3f}</td><td class="col1">{:.3f}</td><td class="col1">{:.1f}</td></tr>\n'.format(
|
|
|
192
|
+ total_n, total_time_baseline, total_time_benchmark, total_difference / 14
|
|
|
193
|
+ ),
|
|
|
194
|
+ )
|
|
|
195
|
+ write_to_html(html_file, "</table><br/>\n")
|
|
|
196
|
+
|
|
|
197
|
+if __name__ == "__main__":
|
|
|
198
|
+ main() |