run_id	vendor	model	spec_pin	weighted_total	l1	grammar_pass_rate	exec_pass_rate	judge_jcs	judge_schema	judge_mode_acc	judge_vector_score	judge_mae	judge_boundary_acc	grammar_pass_count	exec_pass_count	exec_fail_violation	exec_fail_missing	error_count	degraded_count
relay-gemini-3.8-flash-20260919-093409	relay-gemini-3.8-flash	gemini-3.8-flash	127ba56	0.8417	below_L1	0.95	0.7	0.8806	1.0	0.9	0.8532	0.0367	0.75	114	70	25	5	0	0
relay-gpt-6-astra-20260920-000551	relay-gpt-6-astra	gpt-6-astra	127ba56	0.7733	below_L1	0.925	0.56	0.8451	1.0	0.86	0.8053	0.0487	0.7	111	56	44	0	0	0
relay-qwen3.8-max-20260920-053102	relay-qwen3.8-max	qwen3.8-max	127ba56	0.7614	below_L1	0.9	0.5	0.9048	1.0	0.9	0.8239	0.044	0.9	108	50	15	0	35	0
orcarouter-glm-free-20260918-053227	orcarouter-glm-free	z-ai/glm-5.3-flash-free	127ba56	0.7537	below_L1	0.9167	0.54	0.8129	0.98	0.83	0.7745	0.0564	0.65	110	54	43	3	0	0
orca-anthropic-claude-fable-5.1-20260920-000704	orca-anthropic-claude-fable-5.1	anthropic/claude-fable-5.1	127ba56	0.73	below_L1	0.9333	0.5	0.7613	0.86	0.75	0.6963	0.0759	0.75	112	50	38	12	0	0
relay-kimi-k3-20260920-002353	relay-kimi-k3	kimi-k3	127ba56	0.7165	below_L1	0.8917	0.46	0.8115	1.0	0.83	0.7975	0.0506	0.6	107	46	54	0	0	0
relay-glm-5.3-flash-20260919-110013	relay-glm-5.3-flash	glm-5.3-flash	127ba56	0.6812	below_L1	0.9167	0.46	0.6646	0.86	0.74	0.7331	0.0667	0.25	110	46	36	18	0	0
relay-gpt-5.5-instant-20260919-141252	relay-gpt-5.5-instant	gpt-5.5-instant	127ba56	0.677	below_L1	0.8417	0.38	0.8313	1.0	0.84	0.7767	0.0558	0.7	101	38	62	0	0	0
relay-gpt-5.6-terra-20260919-235421	relay-gpt-5.6-terra	gpt-5.6-terra	127ba56	0.6733	below_L1	0.9083	0.34	0.788	1.0	0.8	0.79	0.0525	0.55	109	34	66	0	0	0
relay-gemini-3.1-pro-20260919-084805	relay-gemini-3.1-pro	gemini-3.1-pro	127ba56	0.656	below_L1	0.9333	0.54	0.4679	0.63	0.54	0.5294	0.1176	0.1	112	54	46	0	36	0
relay-gpt-5.6-sol-20260919-234520	relay-gpt-5.6-sol	gpt-5.6-sol	127ba56	0.653	below_L1	0.8833	0.29	0.8079	1.0	0.83	0.7797	0.0551	0.6	106	29	71	0	0	0
relay-qwen3.8-flash-20260920-040159	relay-qwen3.8-flash	qwen3.8-flash	127ba56	0.6521	below_L1	0.8583	0.25	0.8807	1.0	0.89	0.8237	0.0441	0.8	103	25	57	0	19	0
relay-gpt-5.5-20260919-140721	relay-gpt-5.5	gpt-5.5	127ba56	0.6374	below_L1	0.9333	0.19	0.8141	1.0	0.83	0.8103	0.0474	0.6	112	19	81	0	0	0
relay-gpt-5.6-luna-20260919-142153	relay-gpt-5.6-luna	gpt-5.6-luna	127ba56	0.6104	below_L1	0.8167	0.21	0.837	1.0	0.85	0.7849	0.0538	0.7	98	21	79	0	0	0
relay-glm-5.2-20260919-101211	relay-glm-5.2	glm-5.2	127ba56	0.6011	below_L1	0.9	0.22	0.6969	0.84	0.72	0.7043	0.0739	0.5	108	22	53	25	0	0
relay-glm-5.1-20260919-093739	relay-glm-5.1	glm-5.1	127ba56	0.5865	below_L1	0.9417	0.08	0.7631	0.96	0.79	0.7756	0.0561	0.5	113	8	92	0	3	0
relay-gemini-3.6-flash-20260919-093238	relay-gemini-3.6-flash	gemini-3.6-flash	127ba56	0.5826	below_L1	0.9417	0.03	0.8083	0.93	0.8	0.8115	0.0471	0.7	113	3	97	0	7	0
orcarouter-deepseek-free-20260918-050901	orcarouter-deepseek-free	deepseek/deepseek-v4-flash-free	127ba56	0.5825	below_L1	0.8417	0.1	0.8429	1.0	0.86	0.7947	0.0513	0.7	101	10	90	0	0	0
relay-gemini-3-flash-20260919-083635	relay-gemini-3-flash	gemini-3-flash	127ba56	0.5802	below_L1	0.8417	0.12	0.8119	0.95	0.82	0.7193	0.0702	0.75	101	12	88	0	0	0
relay-deepseek-v4-pro-20260919-080833	relay-deepseek-v4-pro	deepseek-v4-pro	127ba56	0.5692	below_L1	0.7083	0.28	0.7444	0.9	0.77	0.7822	0.0544	0.5	85	28	68	4	0	0
relay-claude-sonnet-5-20260919-080833	relay-claude-sonnet-5	claude-sonnet-5	127ba56	0.5652	below_L1	0.8833	0.07	0.7717	0.96	0.8	0.7986	0.0504	0.5	106	7	92	1	0	0
relay-gemini-3.5-flash-20260919-090636	relay-gemini-3.5-flash	gemini-3.5-flash	127ba56	0.5628	below_L1	0.8583	0.02	0.8512	0.99	0.86	0.7462	0.0634	0.8	103	2	98	0	0	0
relay-gpt-5-mini-20260919-113644	relay-gpt-5-mini	gpt-5-mini	127ba56	0.5628	below_L1	0.5417	0.45	0.719	0.93	0.76	0.745	0.0638	0.4	65	45	55	0	0	0
relay-deepseek-v4.1-flash-20260919-082534	relay-deepseek-v4.1-flash	deepseek-v4.1-flash	127ba56	0.529	below_L1	0.7	0.19	0.725	0.87	0.76	0.7351	0.0662	0.5	84	19	75	6	0	0
relay-hy3-20260920-000622	relay-hy3	hy3	127ba56	0.5233	below_L1	0.7333	0.06	0.8188	1.0	0.83	0.784	0.054	0.65	88	6	94	0	0	0
relay-claude-opus-4.5-20260918-225457	relay-claude-opus-4.5	claude-opus-4.5	127ba56	0.5095	below_L1	0.85	0.05	0.6483	0.92	0.54	0.7913	0.0522	0.45	102	5	95	0	0	0
relay-mimo-v2.5-pro-20260920-012356	relay-mimo-v2.5-pro	mimo-v2.5-pro	127ba56	0.4996	below_L1	0.725	0.04	0.7728	0.98	0.77	0.7938	0.0515	0.55	87	4	96	0	0	0
relay-gpt-5.4-20260919-130318	relay-gpt-5.4	gpt-5.4	127ba56	0.4951	below_L1	0.7	0.08	0.7402	1.0	0.68	0.7909	0.0523	0.55	84	8	92	0	0	0
relay-qwen3.8-27b-20260920-035028	relay-qwen3.8-27b	qwen3.8-27b	127ba56	0.4773	below_L1	0.7833	0.0	0.6772	0.98	0.62	0.7658	0.0585	0.4	94	0	100	0	0	0
relay-gpt-5.2-20260919-125317	relay-gpt-5.2	gpt-5.2	127ba56	0.4484	below_L1	0.5667	0.08	0.7403	0.96	0.67	0.8016	0.0496	0.6	68	8	92	0	0	0
relay-gemini-3.5-flash-lite-20260919-092638	relay-gemini-3.5-flash-lite	gemini-3.5-flash-lite	127ba56	0.4415	below_L1	0.7	0.0	0.6551	0.94	0.57	0.7956	0.0511	0.4	84	0	100	0	0	0
relay-mimo-v2.5-20260920-010155	relay-mimo-v2.5	mimo-v2.5	127ba56	0.4313	below_L1	0.675	0.0	0.65	0.96	0.56	0.8198	0.045	0.35	81	0	100	0	0	0
relay-claude-sonnet-4.5-20260919-003613	relay-claude-sonnet-4.5	claude-sonnet-4.5	127ba56	0.4046	below_L1	0.6417	0.0	0.5999	0.92	0.43	0.7695	0.0576	0.45	77	0	100	0	0	0
relay-minimax-m2.7-20260920-031356	relay-minimax-m2.7	minimax-m2.7	127ba56	0.403	below_L1	0.4833	0.07	0.6978	0.96	0.63	0.7689	0.0578	0.5	58	7	93	0	0	0
relay-gpt-5.4-mini-20260919-131318	relay-gpt-5.4-mini	gpt-5.4-mini	127ba56	0.3617	below_L1	0.5333	0.0	0.5836	0.88	0.46	0.668	0.083	0.45	64	0	100	0	0	0
relay-minimax-m3-20260920-034727	relay-minimax-m3	minimax-m3	127ba56	0.316	below_L1	0.7167	0.0	0.2172	0.25	0.25	0.3359	0.166	0.0	86	0	100	0	75	0
relay-gpt-5.4-pro-20260919-133020	relay-gpt-5.4-pro	gpt-5.4-pro	127ba56	0.3043	below_L1	0.8083	0.05	0.013	0.0	0.0	0.0649	0.2338	0.0	97	5	7	0	198	0
relay-gpt-5.4-nano-20260919-132119	relay-gpt-5.4-nano	gpt-5.4-nano	127ba56	0.2306	below_L1	0.1833	0.0	0.5547	0.82	0.49	0.6233	0.0942	0.35	22	0	100	0	0	0
relay-gpt-5-nano-20260919-121446	relay-gpt-5-nano	gpt-5-nano	127ba56	0.2074	below_L1	0.0833	0.0	0.5943	0.91	0.58	0.6514	0.0871	0.25	10	0	100	0	0	0
relay-claude-opus-4.8-20260918-233028	relay-claude-opus-4.8	claude-opus-4.8	127ba56	0.0622	below_L1	0.0167	0.15	0.013	0.0	0.0	0.0649	0.2338	0.0	2	15	13	0	194	0
relay-claude-opus-4.6-20260918-231057	relay-claude-opus-4.6	claude-opus-4.6	127ba56	0.0576	below_L1	0.0333	0.12	0.013	0.0	0.0	0.0649	0.2338	0.0	4	12	67	0	97	0
relay-claude-sonnet-4.6-20260919-011415	relay-claude-sonnet-4.6	claude-sonnet-4.6	127ba56	0.0284	below_L1	0.05	0.02	0.013	0.0	0.0	0.0649	0.2338	0.0	6	2	98	0	0	0
relay-claude-opus-4.7-20260918-231258	relay-claude-opus-4.7	claude-opus-4.7	127ba56	0.0278	below_L1	0.0583	0.01	0.013	0.0	0.0	0.0649	0.2338	0.0	7	1	99	0	0	0
relay-claude-opus-5-20260918-235100	relay-claude-opus-5	claude-opus-5	127ba56	0.0156	below_L1	0.0333	0.0	0.013	0.0	0.0	0.0649	0.2338	0.0	4	0	0	0	269	0
relay-claude-haiku-4.5-20260918-225457	relay-claude-haiku-4.5	claude-haiku-4.5	127ba56	0.0039	below_L1	0.0	0.0	0.013	0.0	0.0	0.0649	0.2338	0.0	0	0	0	100	0	0