[Public] Serverless Benchmarks

	A	B	C	D	E	F	G	H	I	J	K	L	M	N	O	P	Q	U
1	file	num_requests	duration	qps	slowest	fastest	average	lat(.10)	lat(.25)	lat(.50)	lat(.75)	lat(.90)	lat(.95)	lat(.99)	time for first request	time to reach steady state	git

2
3	Distillbert
4	2 CORES warm verta/distilbert/1/output-amd.csv	539	540	1	1.454	0.7526	0.8044	0.7738	0.7784	0.7978	0.7978	0.7978	0.7978	0.7978
5	2 CORES warm verta/distilbert/1/output-amd.csv (second try)	540	539	1	1.1007	0.7596	0.8051	0.7731	0.7782	0.7944	0.7944	0.7944	0.7944	0.7944
6	2 CORES warm verta/distilbert/1/output-amd.csv (MKL_DEBUG_CPU_TYPE=5)	541	540	1	0.4704	0.2747	0.2883	0.2779	0.2807	0.2859	0.2859	0.2859	0.2859	0.2859
7	warm verta/distilbert/1/output-intel.csv	540	539	1	0.3043	0.2531	0.2635	0.2559	0.2575	0.2605	0.2664	0.2766	0.2809	0.2935
8	cold verta/distilbert/100														.7s	315s
9	cold verta/distilbert/100, overprovisioned nodes															105s
10	warm verta/distilbert/100/output.csv	53953	540	99.91	1.4051	0.1184	0.3812	0.1954	0.2419	0.3721	0.4217	0.5983	0.6613	0.803
11	warm verta/distilbert/100/output-2cores.csv 2 CPU	53983	539	100.15	0.8403	0.1201	0.2586	0.1635	0.1772	0.2153	0.319	0.4117	0.4594	0.574
12
13	warm : lambda/distilbert/1/output.csv	540	539	1	0.6009	0.465	0.4941	0.4734	0.4814	0.4885	0.5015	0.5199	0.5341	0.5738			link
14	cold: lambda/distilbert-2/100/output.csv	55928	559	100.05	12.9502	0.4395	0.4852	0.4653	0.4704	0.4786	0.4909	0.508	0.5253	0.5938	41.0138	79	link	Tweakable in Verta
15
16	knative/results/warm_cloudrun_distilbert_1cpu_1qps.csv	299	299	1	2.4596	0.374	0.4161	0.3799	0.3855	0.3937	0.4056	0.4546	0.4678	0.4878
17	knative/results/warm_cloudrun_distilbert_1cpu_1qps.csv (cut)	271	270	1	0.4933	0.3857	0.4167	0.3989	0.4051	0.4165	0.4165	0.4165	0.4165	0.4165
18	knative/results/warm_cloudrun_distilbert_2cpu_1qps.csv (cut)	541	540	1	0.5826	0.3667	0.3983	0.3725	0.3765	0.3848	0.416	0.4401	0.4574	0.4971
19	knative/results/cold_cloudrun_distilbert_1cpu_100qps.csv	47969	599	80.08	19.5553	0.3679	0.7725	0.4087	0.4432	0.529	0.6366	0.7068	0.7883	9.7496	10.8298	34*
20	knative/results/cold_cloudrun_distilbert_1cpu_envvars_100qps.csv	48136	598.97	80.36	18.5845	0.3552	0.826	0.412	0.4473	0.534	0.6577	0.712	0.7761	9.3343	7.8	36*
21	knative/results/cold_cloudrun_distilbert_2cpu_100qps.csv	51083	599	85.28	18.2494	0.3531	0.6881	0.3938	0.4213	0.4782	0.5559	0.6205	0.6536	9.2433	14	52*
22	knative/results/cold_cloudrun_distilbert_2cpu_envvars_100qps.csv	50607	599	84.49	18.7104	0.3618	0.7274	0.4005	0.4325	0.5075	0.6173	0.6697	0.7053	8.7376	8.2	33*
23	knative/results/warm_cloudrun_distilbert_1cpu_100qps.csv (cut)	45716	539	84.82	19.126	0.3607	0.7601	0.4164	0.4681	0.5737	0.6602	0.711	0.7495	9.027
24	knative/results/warm_cloudrun_distilbert_1cpu_envvars_100qps.csv (cut)	44591	539	82.73	19.9768	0.3684	0.8076	0.42	0.4771	0.629	0.6821	0.723	0.7603	8.5836
25	knative/results/warm_cloudrun_distilbert_2cpu_100qps.csv (cut)	45542	543	83.87	17.7925	0.3653	0.7479	0.4095	0.4495	0.5513	0.6485	0.7064	0.7434	8.9639
26	knative/results/warm_cloudrun_distilbert_2cpu_envvars_100qps.csv (cut)	45355	547	82.92	19.659	0.3658	0.7665	0.4423	0.5276	0.6201	0.6684	0.7288	0.889	9.0606
27
28
29
30	Bert
31	warm verta/bert/1/output-amd.csv	444	539.24	0.82	1.7539	1.1631	1.2172	1.1764	1.1823	1.2118	1.2118	1.2118	1.2118	1.2118
32	warm verta/bert/1/output-intel.csv	541	540	1	0.8179	0.3265	0.3875	0.3354	0.3608	0.3748	0.4021	0.4377	0.4887	0.5416
33	2 CORES warm verta/bert/1/output-intel.csv	541	540	1	1.0765	0.3163	0.3976	0.3307	0.36	0.3793	0.4278	0.4802	0.5032	0.5991
34	warm verta/bert/100/output.csv	53992	540	99.99	2.6741	0.2379	0.6622	0.3627	0.4853	0.6522	0.7491	0.9763	1.1161	1.3553
35	warm verta/bert/100/output-2cores.csv 2 CPU	53923	539	100.04	1.9441	0.2134	0.5969	0.349	0.4977	0.5625	0.678	0.8705	0.9376	1.2219
36
37	warm : lambda/bert/1/output.csv	540	539	1	1.2789	0.8892	0.9235	0.9032	0.9078	0.9184	0.9289	0.9445	0.9647	1.058			link	- add more cpu
38	cold: lambda/bert-2/100/output.csv	55925	558.91	100.06	4.2187	0.8434	0.9366	0.9078	0.917	0.9293	0.9455	0.9663	0.9878	1.0735	41.1018	74	link	- number of parallel requests
39
40
41
42	knative/results/warm_cloudrun_bert_2cpu_envvars_1qps.csv (cut)	541	540	1	1.0999	0.7073	0.7373	0.7222	0.7279	0.7336	0.7424	0.7533	0.7616	0.7857
43	knative/results/warm_cloudrun_bert_1cpu_envvars_1qps.csv (cut)	531	530	1	0.8437	0.7179	0.7616	0.7352	0.7435	0.7634	0.7634	0.7634	0.7634	0.7634
44	knative/results/cold_cloudrun_bert_2cpu_envvars_100qps.csv	41125	599	68.66	19.9609	0.6906	1.3653	0.7471	0.7763	0.8554	1.0583	1.1851	1.2784	14.309	15.8	32*
45	knative/results/warm_cloudrun_bert_1cpu_envvars_100qps.csv (cut)	42940	526.42	81.57	19.9451	0.6997	1.1845	0.7836	0.838	0.9565	1.1281	1.2269	1.2742	12.2201
46	knative/results/warm_cloudrun_bert_1cpu_100qps.csv (cut)	40578	524.2	77.41	19.9925	0.7166	1.2181	0.8066	0.8977	1.1145	1.2442	1.3144	1.3714	10.5467
47	knative/results/warm_cloudrun_bert_2cpu_envvars_100qps.csv (cut)	42237	534.49	79.02	19.9982	0.7069	1.2315	0.7946	0.9398	1.157	1.2575	1.3356	1.429	10.1936
48	knative/results/warm_cloudrun_bert_2cpu_100qps.csv (cut)	43326	538.91	80.4	19.8257	0.6985	1.2152	0.7865	0.8845	1.0528	1.1792	1.255	1.2976	12.0589
49	knative/results/warm_cloudrun_bert_2cpu_envvars_100qps.csv (cut)	39491	536.96	73.54	19.9493	0.6982	1.2281	0.7899	0.858	0.9844	1.1756	1.3606	1.5821	12.3521
50
51
52
53
54	ResNet
55	warm verta/resnet/1/output.csv	541	540	1	0.0859	0.059	0.0616	0.0602	0.0606	0.0614	0.0622	0.0631	0.0643	0.0665
56	warm verta/resnet/100/steady-output.csv (first 80% removed)	9024	90.02	100.24	0.4457	0.0612	0.1512	0.088	0.108	0.1384	0.1869	0.2351	0.2556	0.3207
57	warm verta/resnet/100/output-2cores.csv 2 CPU	53979	540.02	99.96	0.7005	0.0627	0.1269	0.0824	0.0909	0.1201	0.1505	0.1856	0.2088	0.2524
58
59	warm lambda/resnet/cold_run_1qps.csv	526	525	1	0.5207	0.1999	0.2306	0.21	0.2148	0.2239	0.2378	0.2564	0.2716	0.3564
60	cold lambda/resnet/cold_run_1qps.csv	583	599	0.97	18.6899	0.1999	0.2627	0.2103	0.2155	0.2243	0.2384	0.2573	0.2735	0.3603
61	warm lambda/resnet/cold_run_100qps.csv	52634	526.01	100.06	1.3035	0.1872	0.2225	0.2022	0.207	0.2145	0.227	0.2467	0.27	0.3476
62	cold lambda/resnet/cold_run_100qps.csv	58481	599.02	97.63	17.7517	0.1872	0.2506	0.2023	0.2072	0.2147	0.2274	0.2475	0.2715	0.3553
63
64	knative/results/warm_cloudrun_resnet_1cpu_100qps.csv (cut)	45978	533	86.26	19.9433	0.1369	0.5049	0.2508	0.2992	0.3413	0.3922	0.4399	0.4766	11.0069
65	knative/results/warm_cloudrun_resnet_2cpu_envvars_1qps.csv (cut)	541	540	1	0.2136	0.1397	0.1573	0.1465	0.1507	0.1558	0.1623	0.1689	0.173	0.1885
66	knative/results/warm_cloudrun_resnet_2cpu_envvars_100qps.csv (cut)	44629	497	89.8	19.8694	0.1421	0.4756	0.2535	0.3024	0.3627	0.4139	0.4594	0.4926	8.4587
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100