MMLU - Alternative Prompts

	A	B	C	D	E	F	G	H	I	J	K	L	M	N	P	Q	R	S	T	U	V	W	X	Y	Z	AA	AB	AC
1	Example Input Prompt
2
3	Input Prompt		Format
4	01		{{question.strip}}
5	02		Q: {{question.strip}}\nA:
6	03		Question: {{question.strip}}\nAnswer:
7
8			Llama-2-7b-hf			Mistral-7B-v0.1			falcon-7b			pythia-6.9b			Choice		Llama-2-7b-hf			Mistral-7B-v0.1			falcon-7b			pythia-6.9b
9			acc (↑)	acc_norm (↑)	brier_score (↓)	acc (↑)	acc_norm (↑)	brier_score (↓)	acc (↑)	acc_norm (↑)	brier_score (↓)	acc (↑)	acc_norm (↑)	brier_score (↓)	Choice		acc (↑)	acc_norm (↑)	brier_score (↓)	acc (↑)	acc_norm (↑)	brier_score (↓)	acc (↑)	acc_norm (↑)	brier_score (↓)	acc (↑)	acc_norm (↑)	brier_score (↓)

10	0-Shot														0-Shot
11	humanities	01	0.3126	0.3537	1.2310	0.3286	0.3558	1.2050	0.2993	0.3367	1.2636	0.2844	0.3245	1.2862	humanities	01	0.3473	0.4050	1.1312	0.3746	0.4208	1.0891	0.3305	0.3753	1.1745	0.3032	0.3537	1.2119
12		02	0.3252	0.3656	1.2210	0.3396	0.3709	1.1847	0.3180	0.3473	1.2434	0.3063	0.3286	1.2653		02	0.3702	0.4293	1.1015	0.3926	0.4406	1.0652	0.3587	0.4013	1.1390	0.3332	0.3689	1.1817
13		03	0.3339	0.3781	1.2084	0.3566	0.3926	1.1705	0.3165	0.3513	1.2425	0.3035	0.3301	1.2667		03	0.3833	0.4484	1.0821	0.4209	0.4724	1.0341	0.3580	0.4043	1.1371	0.3276	0.3693	1.1873
14	formal_logic	01	0.3175	0.3413	1.0522	0.3889	0.4127	1.0258	0.3492	0.3333	1.1133	0.3254	0.3651	1.1124	others	01	0.3859	0.3905	0.9487	0.4256	0.4332	0.8927	0.3702	0.3870	0.9776	0.3341	0.3478	1.0503
15		02	0.3175	0.3413	1.0639	0.4206	0.3889	0.9585	0.3254	0.3254	1.1319	0.3254	0.3571	1.0984		02	0.4476	0.4764	0.8760	0.4826	0.5077	0.8207	0.4338	0.4689	0.8838	0.3533	0.3921	1.0054
16		03	0.3175	0.3492	1.0436	0.3810	0.3571	0.9980	0.3254	0.3333	1.1500	0.3413	0.3254	1.0792		03	0.4575	0.4837	0.8583	0.5081	0.5341	0.7827	0.4512	0.4824	0.8697	0.3491	0.3876	1.0053
17	high_school_european_history	01	0.3818	0.4848	1.0775	0.3576	0.4485	1.1324	0.3455	0.4364	1.1811	0.2909	0.4242	1.2472	social_sciences	01	0.3888	0.3931	1.0255	0.4196	0.4109	0.9756	0.3872	0.3857	1.0264	0.3460	0.3467	1.1005
18		02	0.3758	0.5273	1.0810	0.3030	0.4424	1.1984	0.3576	0.4545	1.1410	0.3091	0.4545	1.2177		02	0.4045	0.4196	0.9913	0.4300	0.4528	0.9529	0.4061	0.4114	1.0060	0.3479	0.3495	1.0956
19		03	0.4000	0.5818	1.0573	0.3879	0.5273	1.1181	0.3697	0.4727	1.1251	0.2727	0.4485	1.2849		03	0.4157	0.4310	0.9736	0.4591	0.4839	0.8993	0.4173	0.4283	0.9909	0.3560	0.3556	1.0938
20	high_school_us_history	01	0.3922	0.4265	1.0274	0.3824	0.4412	1.0167	0.3873	0.3824	1.0508	0.3284	0.3627	1.1224	stem	01	0.3121	0.3272	1.0703	0.3386	0.3514	1.0251	0.2936	0.3144	1.0977	0.2810	0.3028	1.1261
21		02	0.4118	0.4510	1.0123	0.3824	0.4657	1.0396	0.3676	0.3873	1.0804	0.3676	0.3873	1.1201		02	0.3426	0.3651	1.0144	0.3777	0.4111	0.9360	0.3207	0.3447	1.0501	0.2924	0.3063	1.0879
22		03	0.4314	0.5049	0.9689	0.4510	0.5196	0.9308	0.3578	0.4118	1.0904	0.3431	0.3873	1.1250		03	0.3506	0.3664	0.9938	0.3973	0.4351	0.9075	0.3187	0.3351	1.0480	0.2918	0.3021	1.0906
23	high_school_world_history	01	0.3713	0.3924	1.1315	0.3797	0.3797	1.1062	0.3502	0.3460	1.1879	0.3207	0.3460	1.1993	1-Shot
24		02	0.4008	0.4304	1.0755	0.3966	0.3882	1.0985	0.3755	0.3882	1.1405	0.3502	0.3882	1.1642	humanities	01	0.3473	0.4050	1.1312	0.3746	0.4208	1.0891	0.3305	0.3753	1.1745	0.3032	0.3537	1.2119
25		03	0.4177	0.4684	1.0554	0.4388	0.4684	1.0342	0.3713	0.3671	1.1267	0.3713	0.3586	1.1561		02	0.3702	0.4293	1.1015	0.3926	0.4406	1.0652	0.3587	0.4013	1.1390	0.3332	0.3689	1.1817
26	international_law	01	0.2314	0.4628	1.4351	0.2975	0.5041	1.3138	0.2314	0.4380	1.4238	0.2314	0.3554	1.4482		03	0.3833	0.4484	1.0821	0.4209	0.4724	1.0341	0.3580	0.4043	1.1371	0.3276	0.3693	1.1873
27		02	0.2479	0.4793	1.4025	0.2975	0.5041	1.2991	0.2645	0.4298	1.3861	0.2231	0.3388	1.4367	others	01	0.3859	0.3905	0.9487	0.4256	0.4332	0.8927	0.3702	0.3870	0.9776	0.3341	0.3478	1.0503
28		03	0.2562	0.4793	1.3813	0.3140	0.5207	1.2800	0.2645	0.4050	1.3914	0.2314	0.3471	1.4269		02	0.4476	0.4764	0.8760	0.4826	0.5077	0.8207	0.4338	0.4689	0.8838	0.3533	0.3921	1.0054
29	jurisprudence	01	0.2963	0.4537	1.2034	0.3426	0.4444	1.1262	0.3333	0.3796	1.2006	0.2315	0.3241	1.3076		03	0.4575	0.4837	0.8583	0.5081	0.5341	0.7827	0.4512	0.4824	0.8697	0.3491	0.3876	1.0053
30		02	0.3333	0.4537	1.1953	0.3611	0.4630	1.1185	0.3148	0.4167	1.2317	0.2685	0.3796	1.2969	social_sciences	01	0.3888	0.3931	1.0255	0.4196	0.4109	0.9756	0.3872	0.3857	1.0264	0.3460	0.3467	1.1005
31		03	0.3426	0.4444	1.1822	0.4167	0.5093	1.0713	0.3148	0.4074	1.2198	0.2593	0.3889	1.2897		02	0.4045	0.4196	0.9913	0.4300	0.4528	0.9529	0.4061	0.4114	1.0060	0.3479	0.3495	1.0956
32	logical_fallacies	01	0.3804	0.4110	1.0475	0.3865	0.4356	1.0136	0.3190	0.4110	1.1171	0.3374	0.4110	1.1340		03	0.4157	0.4310	0.9736	0.4591	0.4839	0.8993	0.4173	0.4283	0.9909	0.3560	0.3556	1.0938
33		02	0.4049	0.4294	1.0152	0.3865	0.4908	0.9977	0.3558	0.4356	1.0767	0.3620	0.3681	1.1106	stem	01	0.3121	0.3272	1.0703	0.3386	0.3514	1.0251	0.2936	0.3144	1.0977	0.2810	0.3028	1.1261
34		03	0.3988	0.4479	1.0007	0.4356	0.4908	0.9604	0.3313	0.4417	1.0885	0.3374	0.3804	1.1246		02	0.3426	0.3651	1.0144	0.3777	0.4111	0.9360	0.3207	0.3447	1.0501	0.2924	0.3063	1.0879
35	moral_disputes	01	0.3497	0.3468	1.1441	0.3786	0.3526	1.1112	0.3266	0.3353	1.1602	0.3237	0.2803	1.1918		03	0.3506	0.3664	0.9938	0.3973	0.4351	0.9075	0.3187	0.3351	1.0480	0.2918	0.3021	1.0906
36		02	0.3468	0.3439	1.1605	0.3410	0.3642	1.1633	0.3410	0.3439	1.1898	0.3266	0.2717	1.2224	4-Shot
37		03	0.3468	0.3584	1.1573	0.3815	0.3988	1.1338	0.3353	0.3526	1.1901	0.3035	0.2803	1.2486	humanities	01	0.4005	0.4487	1.0136	0.4266	0.4748	0.9739	0.3522	0.3869	1.1191	0.3286	0.3746	1.1203
38	moral_scenarios	01	0.2380	0.2726	1.4696	0.2380	0.2737	1.4613	0.2380	0.2894	1.5096	0.2391	0.2726	1.5141		02	0.4139	0.4570	1.0021	0.4363	0.4834	0.9670	0.3523	0.3959	1.1023	0.3428	0.3799	1.1014
39		02	0.2380	0.2939	1.5194	0.2380	0.2737	1.4495	0.2380	0.2771	1.5176	0.2380	0.2726	1.5096		03	0.4167	0.4635	0.9949	0.4568	0.5057	0.9319	0.3601	0.4040	1.0974	0.3436	0.3822	1.0977
40		03	0.2380	0.2827	1.5178	0.2380	0.2737	1.4902	0.2380	0.2771	1.5174	0.2380	0.2726	1.5089	others	01	0.4578	0.4774	0.8526	0.5006	0.5244	0.7937	0.4396	0.4627	0.9015	0.3731	0.3875	0.9846
41	philosophy	01	0.3762	0.4180	1.0500	0.3826	0.4469	1.0297	0.2894	0.3730	1.1548	0.3055	0.3698	1.1620		02	0.4763	0.5000	0.8330	0.5087	0.5414	0.7774	0.4487	0.4749	0.8848	0.3842	0.4128	0.9615
42		02	0.3794	0.4212	1.0490	0.4373	0.4598	0.9733	0.3601	0.3794	1.0914	0.3344	0.3569	1.1570		03	0.4808	0.5081	0.8243	0.5265	0.5625	0.7503	0.4599	0.4814	0.8696	0.3882	0.4221	0.9561
43		03	0.4084	0.4502	1.0095	0.4630	0.5177	0.9158	0.3762	0.3923	1.0718	0.3376	0.3698	1.1539	social_sciences	01	0.4228	0.4299	0.9690	0.4583	0.4735	0.9094	0.4106	0.4165	1.0052	0.3585	0.3716	1.0722
44	prehistory	01	0.4383	0.3704	0.9545	0.4599	0.3858	0.9176	0.4074	0.3272	1.0096	0.3796	0.3241	1.0429		02	0.4267	0.4377	0.9570	0.4654	0.4794	0.9012	0.4034	0.4254	0.9968	0.3642	0.3687	1.0695
45		02	0.4907	0.4321	0.8714	0.5216	0.4506	0.8226	0.4475	0.4105	0.9208	0.4475	0.3735	0.9629		03	0.4267	0.4377	0.9570	0.4654	0.4794	0.9012	0.4034	0.4254	0.9968	0.3642	0.3687	1.0695
46		03	0.5031	0.4475	0.8513	0.5463	0.5062	0.7852	0.4599	0.4290	0.9166	0.4352	0.3673	0.9778	stem	01	0.3333	0.3562	1.0079	0.3839	0.4169	0.9197	0.2998	0.3155	1.0681	0.2986	0.3142	1.0712
47	professional_law	01	0.2627	0.3110	1.3359	0.2731	0.2901	1.3225	0.2627	0.3012	1.3370	0.2542	0.3070	1.3467		02	0.3565	0.3692	0.9815	0.4121	0.4405	0.8930	0.3215	0.3394	1.0325	0.3039	0.3221	1.0554
48		02	0.2634	0.2927	1.3326	0.2751	0.2992	1.3042	0.2666	0.2901	1.3354	0.2699	0.2920	1.3320		03	0.3626	0.3741	0.9728	0.4261	0.4582	0.8715	0.3279	0.3473	1.0281	0.3051	0.3181	1.0481
49		03	0.2673	0.3012	1.3281	0.2757	0.3025	1.3020	0.2608	0.2940	1.3384	0.2731	0.2960	1.3227
50	world_religions	01	0.4795	0.5731	0.7774	0.6023	0.6550	0.5813	0.4561	0.5263	0.8222	0.3743	0.4561	0.9261
51		02	0.6023	0.6842	0.5407	0.7427	0.7368	0.4245	0.6491	0.6784	0.5632	0.5088	0.5556	0.7332
52		03	0.6550	0.7135	0.5141	0.7427	0.7485	0.4235	0.6491	0.6725	0.5564	0.5146	0.5789	0.7368
53	other	01	0.4055	0.3985	0.9259	0.4487	0.4483	0.8654	0.3914	0.3952	0.9535	0.3463	0.3505	1.0329
54		02	0.4686	0.4915	0.8454	0.5069	0.5301	0.7868	0.4590	0.4857	0.8595	0.3817	0.4081	0.9752
55		03	0.4808	0.5060	0.8260	0.5301	0.5533	0.7508	0.4699	0.4992	0.8495	0.3743	0.4010	0.9768
56	business_ethics	01	0.6200	0.5200	0.6094	0.6400	0.5900	0.5763	0.5200	0.4500	0.7288	0.4500	0.3800	0.9002
57		02	0.6500	0.5900	0.5815	0.6700	0.6400	0.5123	0.5700	0.5600	0.6099	0.4600	0.4700	0.8346
58		03	0.6500	0.5800	0.5682	0.7100	0.6600	0.4882	0.6200	0.5800	0.6019	0.4300	0.4400	0.8316
59	clinical_knowledge	01	0.3472	0.4226	1.0838	0.3698	0.4755	1.0146	0.3094	0.4189	1.0990	0.2755	0.3660	1.1618
60		02	0.3698	0.4679	1.0362	0.3925	0.4755	1.0021	0.3887	0.4830	1.0204	0.3396	0.4264	1.1065
61		03	0.3925	0.4717	1.0178	0.4151	0.5245	0.9629	0.3849	0.4943	1.0056	0.3472	0.4113	1.0933
62	college_medicine	01	0.2948	0.2659	1.0722	0.3179	0.3468	1.0283	0.2775	0.3121	1.1058	0.3410	0.3064	1.0768
63		02	0.3757	0.3642	1.0001	0.3931	0.4104	0.9855	0.3584	0.3757	1.0322	0.3237	0.3353	1.1035
64		03	0.3931	0.3757	0.9940	0.4104	0.4277	0.9431	0.3699	0.3757	1.0227	0.3121	0.3353	1.1017
65	global_facts	01	0.3400	0.3300	0.7958	0.4600	0.4300	0.6877	0.3100	0.2800	0.8531	0.2400	0.2300	0.8982
66		02	0.5300	0.5400	0.6337	0.5600	0.5700	0.6173	0.4900	0.4500	0.6508	0.2600	0.2600	0.8196
67		03	0.5200	0.5300	0.6293	0.5800	0.5900	0.6008	0.4900	0.4800	0.6292	0.2500	0.2400	0.8292
68	human_aging	01	0.4529	0.3587	0.8968	0.4529	0.3632	0.8641	0.4395	0.3677	0.9369	0.3991	0.3632	0.9664
69		02	0.4484	0.4484	0.8564	0.4753	0.4529	0.8112	0.4395	0.4350	0.8608	0.4126	0.3991	0.9205
70		03	0.4843	0.4619	0.8232	0.5426	0.5112	0.7475	0.4709	0.4484	0.8436	0.3812	0.3946	0.9494
71	management	01	0.3204	0.4272	0.9744	0.4078	0.5049	0.8450	0.3883	0.4466	0.8782	0.3107	0.3981	1.0994
72		02	0.4466	0.5631	0.8406	0.5049	0.6019	0.7381	0.4854	0.6019	0.7387	0.3495	0.4466	1.0113
73		03	0.4660	0.5631	0.8133	0.5146	0.6311	0.7185	0.5243	0.5631	0.7225	0.3301	0.4466	1.0125
74	marketing	01	0.5812	0.5342	0.6523	0.6239	0.5940	0.6066	0.5641	0.5171	0.7106	0.4701	0.4402	0.8389
75		02	0.6239	0.6368	0.5757	0.6923	0.6496	0.5238	0.6197	0.6026	0.6094	0.4658	0.5000	0.8103
76		03	0.6410	0.6453	0.5571	0.7265	0.6795	0.4688	0.6368	0.6496	0.5846	0.4957	0.4957	0.7998
77	medical_genetics	01	0.3200	0.4100	1.0028	0.4000	0.4100	0.9284	0.3500	0.4500	0.9915	0.3400	0.4200	1.0033
78		02	0.4100	0.4700	0.9180	0.4500	0.4700	0.8816	0.4000	0.4900	0.9197	0.3500	0.4900	0.9713
79		03	0.4100	0.4800	0.8920	0.4800	0.5300	0.8129	0.4300	0.5100	0.8718	0.3700	0.4900	0.9737
80	miscellaneous	01	0.5121	0.4534	0.7130	0.6156	0.5632	0.5886	0.5185	0.4508	0.7128	0.4112	0.3691	0.8643
81		02	0.6411	0.6284	0.5533	0.6935	0.7050	0.4798	0.6347	0.6284	0.5739	0.5121	0.5070	0.7304
82		03	0.6552	0.6667	0.5254	0.7075	0.7152	0.4542	0.6284	0.6347	0.5707	0.4917	0.4866	0.7351
83	nutrition	01	0.3007	0.3791	1.1605	0.2908	0.3725	1.1886	0.2876	0.3856	1.2103	0.2778	0.3595	1.2272
84		02	0.3203	0.3824	1.1603	0.3562	0.4314	1.1240	0.3399	0.4052	1.1462	0.2908	0.3660	1.2216
85		03	0.3235	0.3987	1.1410	0.3660	0.4542	1.0899	0.3464	0.4314	1.1388	0.2941	0.3856	1.2142
86	professional_accounting	01	0.3014	0.3121	1.1205	0.2766	0.3191	1.1294	0.2695	0.2518	1.1565	0.2730	0.2695	1.2062
87		02	0.3050	0.3227	1.0832	0.3227	0.3085	1.0183	0.2695	0.2730	1.1292	0.2872	0.2447	1.1762
88		03	0.3156	0.3262	1.0792	0.3333	0.3191	1.0139	0.2979	0.2908	1.1355	0.2624	0.2411	1.1886
89	professional_medicine	01	0.3493	0.3382	1.0120	0.3824	0.3493	0.9568	0.3015	0.3566	1.0999	0.3199	0.3125	1.1049
90		02	0.3787	0.4118	0.9710	0.4265	0.4632	0.8784	0.3603	0.3934	1.0328	0.3309	0.3456	1.0793
91		03	0.3897	0.4154	0.9601	0.4632	0.4853	0.8255	0.3713	0.3971	1.0286	0.3088	0.3162	1.1072
92	virology	01	0.2771	0.3253	1.2395	0.2952	0.3133	1.1910	0.2771	0.3434	1.2248	0.2349	0.3072	1.3064
93		02	0.3193	0.3675	1.1778	0.3373	0.4217	1.0968	0.2831	0.3976	1.1648	0.2108	0.3072	1.2845
94		03	0.3072	0.3735	1.1572	0.3554	0.4157	1.0487	0.2952	0.4157	1.1508	0.2651	0.3554	1.2330
95	social_sciences	01	0.3997	0.4082	1.0040	0.4303	0.4283	0.9577	0.3923	0.3955	1.0140	0.3484	0.3568	1.0875
96		02	0.4189	0.4358	0.9713	0.4381	0.4631	0.9419	0.4118	0.4215	0.9916	0.3513	0.3659	1.0832
97		03	0.4293	0.4452	0.9540	0.4712	0.4989	0.8836	0.4231	0.4348	0.9769	0.3588	0.3679	1.0847
98	econometrics	01	0.2544	0.2456	1.2122	0.3158	0.2719	1.1216	0.2632	0.2368	1.1694	0.2193	0.2544	1.2622
99		02	0.2281	0.2368	1.1743	0.3158	0.3333	1.0567	0.2807	0.2719	1.1257	0.2281	0.2368	1.2705
100		03	0.2544	0.2281	1.1666	0.3246	0.3246	1.0447	0.2895	0.2632	1.1460	0.2368	0.2193	1.2474