Benchmark Run Details

Model	llama3.2:3b:Q4_K_M
Benchmark	0051_pinyin_letters
Normed Score	36
Run Timestamp	2025-04-01 19:07:42

Question-Level Details

Question ID	Score	Evaluation Time (ms)	Debug Info
0051_pinyin_letters:0	100	2428	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 我喜欢学习中文", "expected_count": 2, "model_count": 2, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 我喜欢学习中文", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:1	0	450	{ "question": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 北京是中国的首都", "expected_count": 3, "model_count": 2, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 北京是中国的首都", "answer_type": "numeric", "correct_answer": 3, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:10	0	419	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你好吗？", "expected_count": 1, "model_count": 2, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你好吗？", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:11	100	430	{ "question": "Count how many times the letter 'C' appears in the Pinyin representation of the following Chinese sentence: 我喜欢吃饭。", "expected_count": 1, "model_count": 1, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'C' appears in the Pinyin representation of the following Chinese sentence: 我喜欢吃饭。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:12	0	461	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 天气很好。", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 天气很好。", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:13	100	454	{ "question": "Count how many times the letter 'M' appears in the Pinyin representation of the following Chinese sentence: 我叫李明。", "expected_count": 1, "model_count": 1, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'M' appears in the Pinyin representation of the following Chinese sentence: 我叫李明。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:14	0	425	{ "question": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 今天很忙。", "expected_count": 1, "model_count": 2, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 今天很忙。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:15	0	438	{ "question": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 多少钱？", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 多少钱？", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:16	0	456	{ "question": "Count how many times the letter 'T' appears in the Pinyin representation of the following Chinese sentence: 他去学校了。", "expected_count": 1, "model_count": 2, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'T' appears in the Pinyin representation of the following Chinese sentence: 他去学校了。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:17	100	438	{ "question": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 你想喝水吗？", "expected_count": 1, "model_count": 1, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 你想喝水吗？", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:18	0	450	{ "question": "Count how many times the letter 'N' appears in the Pinyin representation of the following Chinese sentence: 再见，朋友。", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'N' appears in the Pinyin representation of the following Chinese sentence: 再见，朋友。", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:2	0	460	{ "question": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 今天天气很好", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 今天天气很好", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:3	0	454	{ "question": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 他的哥哥八岁了", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 他的哥哥八岁了", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:4	0	426	{ "question": "Count how many times the letter 'A' appears in the Pinyin representation of the following Chinese sentence: 我们一起去公园吧", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'A' appears in the Pinyin representation of the following Chinese sentence: 我们一起去公园吧", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:5	0	433	{ "question": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 这本书非常有趣", "expected_count": 3, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 这本书非常有趣", "answer_type": "numeric", "correct_answer": 3, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:6	0	434	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你明天有什么计划", "expected_count": 4, "model_count": 2, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你明天有什么计划", "answer_type": "numeric", "correct_answer": 4, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:7	100	461	{ "question": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 我的猫喜欢睡觉", "expected_count": 1, "model_count": 1, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 我的猫喜欢睡觉", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:8	100	443	{ "question": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 中国有很长的历史", "expected_count": 2, "model_count": 2, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 中国有很长的历史", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:9	100	432	{ "question": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 昨天我去了图书馆", "expected_count": 1, "model_count": 1, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 昨天我去了图书馆", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }

Run Summary

Question-Level Details