SoLaR | Code Inspections Appendix

	A	B	C	D	E	F	H	I	J	K
1	Repo	Category	Reviewer 1	Reviewer 2	Reviewer 3	KEY: scoring	Repo	Fleiss' Kappa	Krippendorff's Alpha	Light's Kappa

2	autogen_studio	action	2	2	2	0 = low	autogen_studio	0.3182	0.3636363636	0.2803030303
3	autogen_studio	environment	2	2	2	1 = middle
4	autogen_studio	orchestration	2	1	1	2 = high
5	autogen_studio	human-in-the-loop	2	2	2
6	autogen_studio	observability	1	1	2
7	composio	action	2	2	2		composio	1	1	1
8	composio	environment	2	2	2
9	composio	orchestration	0	0	0
10	composio	human-in-the-loop	2	2	2
11	composio	observability	0	0	0
12	sibyl_system	action	2	2	0		sibyl_system	0.4444	0.3248484848	0.4715007215
13	sibyl_system	environment	2	2	2
14	sibyl_system	orchestration	1	1	1
15	sibyl_system	human-in-the-loop	2	2	2
16	sibyl_system	observability	2	1	2
17	dream_team	action	2	2	2		dream_team	0.7581	0.9142857143	0.7777777778
18	dream_team	environment	1	1	0
19	dream_team	orchestration	1	1	1
20	dream_team	human-in-the-loop	2	2	2
21	dream_team	observability	1	1	1
22	graphrag_ollama	action	1	1	2		graphrag_ollama	0.1549	0.3466666667	0.3333333333
23	graphrag_ollama	environment	0	0	0
24	graphrag_ollama	orchestration	0	0	1
25	graphrag_ollama	human-in-the-loop	0	0	2
26	graphrag_ollama	observability	1	1	2
27	auto_tx	action	1	1	2		auto_tx	0.2373	0.4533333333	0.4444444444
28	auto_tx	environment	1	1	0
29	auto_tx	orchestration	1	1	1
30	auto_tx	human-in-the-loop	2	2	2
31	auto_tx	observability	1	1	2
32	letta	action	2	2	2		letta	1	1	1
33	letta	environment	2	2	2
34	letta	orchestration	2	2	2
35	letta	human-in-the-loop	2	2	2
36	letta	observability	0	0	0
37	h2opgt	action	2	2	2		h2opgt	0.7887	0.86	0.7916666667
38	h2opgt	environment	2	2	2
39	h2opgt	orchestration	1	1	1
40	h2opgt	human-in-the-loop	1	1	2
41	h2opgt	observability	0	0	0
42	langflow	action	2	2	2		langflow	0.6809	0.8	0.7222222222
43	langflow	environment	1	2	2
44	langflow	orchestration	2	2	2
45	langflow	human-in-the-loop	2	2	2
46	langflow	observability	0	0	0
47	gpt_academic	action	2	2	2		gpt_academic	0.5588	0.8282828283	0.5807072829
48	gpt_academic	environment	2	2	2
49	gpt_academic	orchestration	2	1	2
50	gpt_academic	human-in-the-loop	0	0	0
51	gpt_academic	observability	1	1	0
52							overall agreement	0.6422	0.7429428553	0.6432121355
53
54							specific taxonomy categories
55							action	0.3023	0.2723076923	0.2857142857
56							environment	0.6	0.8724	0.6139334522
57							orchestration	0.6654	0.777600907	0.6692588326
58							human-in-the-loop	0.6491	0.6746870748	0.619047619
59							observability	0.4662	0.7749498911	0.5252529525
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100