NTCIR-15 SHINRA2020-ML_EvaluationReport

	A	B	C	D	E	F	G	H	I	J	K	L	M	N	O	P	Q	R	S	T	U	V	W	X	Y	Z	AA	AB	AC	AD	AE	AF	AG
1	Group ID	Method	Late Submission	ar	bg	ca	cs	da	de	el	en	es	fa	fi	fr	he	hi	hu	id	it	ko	nl	no	pl	pt	ro	ru	sv	th	tr	uk	vi	zh
2				Arabic	Bulgarian	Catalan, Valencian	Czech	Danish	German	Greek, Modern (1453-)	English	Spanish, Castilian	Persian	Finnish	French	Hebrew	Hindi	Hungarian	Indonesian	Italian	Korean	Dutch, Flemish	Norwegian	Polish	Portuguese	Romanian, Moldavian, Moldovan	Russian	Swedish	Thai	Turkish	Ukrainian	Vietnamese	Chinese
3	FPTAI	BERT		73.24602432	83.77213978	52.55126371	84.46969697	82.29710489	22.62311068	84.39849624	82.23495702	80.60263653	81.69685415	83.62271234	21.5925745	83.78887842	76.43192488	85.45541249	81.93336462	26.54953636	83.66762178	83.28557784	80.53181387	84.53410183	83.23098725	84.60445287	84.08007626	83.18246784	81.25880695	86.50453028	83.12056738	80.34351145	81.24704771
4	LIAT	ML-BERT		63.15789474	75.19920319	76.27538385	79.46428571	74.8015873	79.48844073	72.43083004	78.55731225	77.72839506	75.42120912	79.12741696	76.88492063	79.10669975	16.49382716	78.92910263	72.44796829	81.35759961	80.37754595	79.86213688	76.50273224	80.59701493	78.49355798	76.16683217	79.09270217	71.63437655	49.57941613	77.18951014	78.71287129	75.23573201	77.96944308
5	PribL	BERTGRU		76.26613704	-	-	-	-	80.23656974	-	81.27173373	80.2970297	-	-	77.93240557	-	-	-	-	81.91699605	81.51469856	80.95002474	-	82.72772524	82.36462991	-	82.5979177	-	-	84.35643564	-	-	78.3810464
6	PribL	BERTLINCONCAT		75.45195052	-	-	81.191067	-	79.83074753	-	80.11753183	80.71532141	-	-	78.52216749	-	-	-	-	81.89360664	81.03616813	81.25613346	78.390462	83.46303502	81.88440991	-	83.07392996	-	-	83.22800194	-	-	79.37438905
7	RH312	RnnGnnXlmr		-	82.13228036	-	-	-	-	-	-	-	-	-	80.31418753	-	71.69995129	-	77.55501222	-	-	-	-	-	-	-	-	-	76.76669894	83.28445748	-	-	-
8	ousia	RoBERTa+wiki2vec+wikidata		70.51909892	-	-	-	-	81.8627451	-	-	80.94302554	-	-	81.00890208	-	69.74708171	-	-	81.21330724	-	-	-	-	81.3976378	-	-	-	76.36363636	-	-	-	79.76424361
9	uomfj	jointrep		64.54816286	83.06772908	79.8215171	81.29032258	80.55555556	81.02513553	-	82.73170732	81.38613861	80.37661051	80.91224591	78.20811946	81.09181141	66.66666667	85.01984127	78.51116625	82.01581028	82.51121076	81.64275111	78.78787879	84.51966152	80.87431694	80.83416087	82.89538919	-	65.01731816	84.85148515	81.60634606	77.06057597	78.57847976
10	uomfj	jointrepPostprocess		64.54816286	83.06772908	79.8215171	81.29032258	80.55555556	81.02513553	-	82.57059396	81.38613861	80.37661051	80.91224591	78.20811946	81.09181141	66.66666667	85.01984127	78.51116625	82.01581028	82.51121076	81.64275111	78.78787879	84.51966152	80.87431694	80.83416087	82.89538919	-	65.01731816	84.85148515	81.60634606	77.06057597	78.57847976
11	uomfj	jointrepUnionPostprocess		64.54816286	83.06772908	79.8215171	81.29032258	80.55555556	81.02513553	-	82.67564966	81.38613861	80.37661051	80.91224591	78.20811946	81.09181141	66.66666667	85.01984127	78.51116625	82.01581028	82.51121076	81.64275111	78.78787879	84.51966152	80.87431694	80.83416087	82.89538919	-	65.01731816	84.85148515	81.60634606	77.06057597	78.57847976
12	FPTAI	BERT	Y	73.2498841	83.27806727	81.10383536	83.7406015	81.73666824	81.26452813	84.1025641	81.95631529	80.60122123	81.52019002	83.36448598	80.67860509	84.21052632	75.65298507	84.7826087	81.65350673	82.80734809	83.76963351	83.17490494	80.16997167	84.07037565	82.70042194	84.59726802	83.43558282	83.43616516	81.15671642	86.02558029	82.60665729	80.41825095	80.59841047
13	HUKB	AB	Y	30.98311817	60.85657371	42.34010907	52.60545906	49.00793651	53.72104485	7.509881423	45.10680576	49.20792079	45.58969277	53.14823996	43.83697813	59.95037221	39.7037037	69.1468254	44.06947891	45.55335968	63.67713004	42.35526967	34.5752608	62.71777003	42.32488823	57.59682224	42.04263758	50.32322228	39.98020782	61.88118812	60.28755578	60.37735849	21.22408687
14	HUKB	ABC	Y	30.98311817	61.05577689	42.5384234	52.60545906	49.00793651	53.81961557	7.509881423	45.10680576	49.10891089	45.58969277	53.44571145	43.73757455	60.04962779	39.50617284	69.44444444	44.46650124	45.55335968	63.97608371	42.45423058	34.5752608	63.51418616	42.62295082	57.69612711	42.24095191	50.6215813	40.37605146	62.47524752	60.18839861	60.47666336	21.42152024
15	HUKB	AC	Y	13.50546177	28.0876494	16.26177491	18.85856079	13.98809524	26.81123706	7.509881423	11.92250373	19.5049505	15.65906838	17.05503223	11.23260437	15.78163772	22.02469136	26.09126984	16.27791563	12.05533597	13.9511709	17.12023751	11.32637854	32.55350921	16.09538003	28.50049652	11.30391671	21.97911487	24.04750124	16.73267327	22.50867625	22.1449851	17.5715696
16	LIAT	ML-BERT	Y	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	79.58271237	-	-	-	-	-
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97