Skip to content

Commit 2b202bc

Browse files
baltzellbaltzell
authored
Add script for scaling testing (#945)
* make names better for plotting * add plotter * limit fit range, add another total metric * cleanup * more flexible thread configuration * use real filenamt * columning * better service choices * fix header name * remove duplicate * plot it too * add test data * cleanup * Delete libexec/scaling.txt
1 parent 4e8227e commit 2b202bc

2 files changed

Lines changed: 145 additions & 26 deletions

File tree

‎libexec/scaling‎

Lines changed: 94 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,27 @@
11
#!/usr/bin/env python3
22

33
def cli():
4-
import os,argparse
4+
import os,sys,argparse
55
cli = argparse.ArgumentParser(description='CLARA scaling test')
6-
cli.add_argument('-y',help='YAML file',required=True)
6+
cli.add_argument('-P',help='plot only',action='store_true')
7+
cli.add_argument('-y',help='YAML file',default=None)
78
cli.add_argument('-c',help='CLARA_HOME path',default=os.getenv('CLARA_HOME',None))
8-
cli.add_argument('-t',help='threads',default=[2,4],type=int,action='append')
9+
cli.add_argument('-t',help='threads (default=4,8)',default='4,8')
910
cli.add_argument('-e',help='events per thread',default=100,type=int)
10-
cli.add_argument('input',help='input data file')
11+
cli.add_argument('-i',help='input data file',default=None)
1112
cfg = cli.parse_args()
12-
import sys
13-
if cfg.c is None: sys.exit('-c or $CLARA_HOME is required')
13+
cfg.t = cfg.t.split(',')
14+
if cfg.P:
15+
cfg.i = 'scaling.txt'
16+
else:
17+
if cfg.y is None: sys.exit('-y YAML is required w/o -P')
18+
if cfg.c is None: sys.exit('-c or $CLARA_HOME is required w/o -P')
19+
if cfg.i is None: sys.exit('-i is required')
1420
return cfg
1521

1622
def run(cmd):
1723
import subprocess
18-
print('scaling >>> '+' '.join(cmd))
24+
print('run >>> '+' '.join(cmd))
1925
p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,universal_newlines=True,encoding='latin-1')
2026
for line in iter(p.stdout.readline, ''):
2127
line = line.strip()
@@ -47,38 +53,38 @@ def benchmark(cfg, threads):
4753
exiting = False
4854
elif len(cols) > 20:
4955
if line.find('Processed') >= 0:
50-
benchmarks['event'] = float(cols[12])
56+
benchmarks['Event'] = float(cols[12])
5157
elif exiting:
5258
# catch-all for services:
5359
if len(cols) > 14:
54-
if 'services' not in benchmarks:
55-
benchmarks['services'] = collections.OrderedDict()
56-
benchmarks['services'][cols[2]] = float(cols[14])
60+
if 'Services' not in benchmarks:
61+
benchmarks['Services'] = collections.OrderedDict()
62+
benchmarks['Services'][cols[2]] = float(cols[14])
5763
# FIXME: what are these, why don't they add up?
5864
elif line.find('Average processing time') >= 0:
59-
benchmarks['avg'] = float(cols[6])
65+
benchmarks['Avg'] = float(cols[6])
6066
elif line.find('Total processing time') >= 0:
61-
benchmarks['total'] = float(cols[6])
67+
benchmarks['Total'] = float(cols[6])
6268
elif line.find('Total orchestrator time') >= 0:
63-
benchmarks['orch'] = float(cols[6])
69+
benchmarks['Orch'] = float(cols[6])
6470
except ValueError:
6571
pass
6672
return benchmarks
6773

6874
def table(benchmarks):
6975
table = []
70-
header = [ 'threads' ]
76+
header = [ 'Threads' ]
7177
b = benchmarks[0][1]
72-
header.extend([x for x in b if x != 'services'])
73-
if 'services' in b:
74-
header.extend(b['services'].keys())
78+
header.extend([x for x in b if x != 'Services'])
79+
if 'Services' in b:
80+
header.extend(b['Services'].keys())
7581
table.append(header)
7682
for b in benchmarks:
7783
threads,benchmark = b[0],b[1]
7884
row = [threads]
79-
for k in ['event','avg','total','orch','services']:
85+
for k in ['Event','Avg','Total','Orch','Services']:
8086
if k in benchmark:
81-
if k == 'services':
87+
if k == 'Services':
8288
row.extend(benchmark[k].values())
8389
else:
8490
row.append(benchmark[k])
@@ -92,13 +98,75 @@ def show(benchmarks):
9298
def save(benchmarks):
9399
with open('scaling.txt','w') as f:
94100
for row in table(benchmarks):
95-
f.write(' '.join([str(x) for x in row]))
101+
f.write(' '.join([str(x) for x in row])+'\n')
102+
103+
gnuplot='''
104+
set terminal qt size 1000,700
105+
106+
set multiplot layout 2,2
107+
set datafile columnheaders
108+
set title font 'sans,14'
109+
set key box
110+
set xlabel 'Threads'
111+
set ylabel 'Event Time [ms]'
112+
set yrange [0:]
113+
114+
set title 'I/O'
115+
set key outside right width 2
116+
plot 'scaling.txt' \
117+
using 1:(($2*$1)) pt 7 with points title columnhead(2) ,\
118+
'' using 1:((column($#))) pt 7 with points title 'Total' ,\
119+
'' using 1:($6) pt 7 with points title 'Reader' ,\
120+
'' using 1:((column($#-1))) pt 7 with points title 'Writer' ,\
121+
'' using 1:(($6*$1)) pt 7 with points title 'IOver' ,\
122+
123+
set title 'Engines'
124+
125+
# data-ai-uber.yml:
126+
#plot 'scaling.txt' \
127+
# using 1:($7) pt 7 with points title columnhead(7) ,\
128+
# '' using 1:($8) pt 7 with points title columnhead(8) ,\
129+
# '' using 1:($9) pt 7 with points title columnhead(9) ,\
130+
# '' using 1:($10) pt 7 with points title columnhead(10) ,\
131+
# '' using 1:($11) pt 7 with points title columnhead(11) ,\
132+
# '' using 1:($12) pt 7 with points title columnhead(12) ,\
133+
134+
# rgd-clarode.yml:
135+
plot 'scaling.txt' \
136+
using 1:($7) pt 7 with points title columnhead(7) ,\
137+
'' using 1:($14) pt 7 with points title columnhead(14) ,\
138+
'' using 1:($15) pt 7 with points title columnhead(15) ,\
139+
'' using 1:($16) pt 7 with points title columnhead(16) ,\
140+
'' using 1:($26) pt 9 with points title columnhead(26) ,\
141+
'' using 1:($19) pt 7 with points title columnhead(19) ,\
142+
'' using 1:($28) pt 9 with points title columnhead(28) ,\
143+
'' using 1:($33) pt 9 with points title columnhead(33) ,\
144+
145+
set title 'Throughput'
146+
set ylabel 'Rate [Hz]'
147+
set size 0.7,0.5
148+
set origin 0.12,0
149+
unset key
150+
f(x) = m*x
151+
fit [0:24] f(x) 'scaling.txt' using 1:(1/$2*1e3) via m
152+
plot 'scaling.txt' using 1:(1/$2*1e3) pt 7, f(x)
153+
'''
154+
155+
def plot():
156+
import tempfile
157+
with tempfile.NamedTemporaryFile(mode='w') as f:
158+
f.write(gnuplot)
159+
f.flush()
160+
list(run(['gnuplot','-p',f.name]))
161+
input()
96162

97163
if __name__ == '__main__':
98164
cfg = cli()
99-
benchmarks = []
100-
for threads in cfg.t:
101-
benchmarks.append([threads, benchmark(cfg, threads)])
102-
show(benchmarks)
103-
save(benchmarks)
165+
if not cfg.P:
166+
benchmarks = []
167+
for threads in cfg.t:
168+
benchmarks.append([threads, benchmark(cfg, threads)])
169+
show(benchmarks)
170+
save(benchmarks)
171+
plot()
104172

‎libexec/scaling.gpl‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
2+
set terminal qt size 1000,700
3+
4+
set multiplot layout 2,2
5+
set datafile columnheaders
6+
set title font 'sans,14'
7+
set key box
8+
set xlabel 'Threads'
9+
set ylabel 'Event Time [ms]'
10+
set yrange [0:]
11+
12+
set title 'I/O'
13+
set key outside right width 2
14+
plot 'scaling.txt' \
15+
using 1:(($2*$1)) pt 7 with points title columnhead(2) ,\
16+
'' using 1:((column($#))) pt 7 with points title 'Total' ,\
17+
'' using 1:($6) pt 7 with points title 'Reader' ,\
18+
'' using 1:((column($#-1))) pt 7 with points title 'Writer' ,\
19+
'' using 1:(($6*$1)) pt 7 with points title 'IOver' ,\
20+
21+
set title 'Engines'
22+
23+
# data-ai-uber.yml:
24+
#plot 'scaling.txt' \
25+
# using 1:($7) pt 7 with points title columnhead(7) ,\
26+
# '' using 1:($8) pt 7 with points title columnhead(8) ,\
27+
# '' using 1:($9) pt 7 with points title columnhead(9) ,\
28+
# '' using 1:($10) pt 7 with points title columnhead(10) ,\
29+
# '' using 1:($11) pt 7 with points title columnhead(11) ,\
30+
# '' using 1:($12) pt 7 with points title columnhead(12) ,\
31+
32+
# rgd-clarode.yml:
33+
plot 'scaling.txt' \
34+
using 1:($7) pt 7 with points title columnhead(7) ,\
35+
'' using 1:($14) pt 7 with points title columnhead(14) ,\
36+
'' using 1:($15) pt 7 with points title columnhead(15) ,\
37+
'' using 1:($16) pt 7 with points title columnhead(16) ,\
38+
'' using 1:($26) pt 9 with points title columnhead(26) ,\
39+
'' using 1:($19) pt 7 with points title columnhead(19) ,\
40+
'' using 1:($28) pt 9 with points title columnhead(28) ,\
41+
'' using 1:($33) pt 9 with points title columnhead(33) ,\
42+
43+
set title 'Throughput'
44+
set ylabel 'Rate [Hz]'
45+
set size 0.7,0.5
46+
set origin 0.12,0
47+
unset key
48+
f(x) = m*x
49+
fit [0:24] f(x) 'scaling.txt' using 1:(1/$2*1e3) via m
50+
plot 'scaling.txt' using 1:(1/$2*1e3) pt 7, f(x)
51+

0 commit comments

Comments
 (0)