Benchmark Run Details

Model	qwen2.5:7b:Q4_K_M
Benchmark	0051_pinyin_letters
Normed Score	15
Run Timestamp	2025-04-01 19:08:03

Question-Level Details

Question ID	Score	Evaluation Time (ms)	Debug Info
0051_pinyin_letters:0	0	5196	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 我喜欢学习中文", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 我喜欢学习中文", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:1	0	836	{ "question": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 北京是中国的首都", "expected_count": 3, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 北京是中国的首都", "answer_type": "numeric", "correct_answer": 3, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:10	100	871	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你好吗？", "expected_count": 1, "model_count": 1, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你好吗？", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:11	0	875	{ "question": "Count how many times the letter 'C' appears in the Pinyin representation of the following Chinese sentence: 我喜欢吃饭。", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'C' appears in the Pinyin representation of the following Chinese sentence: 我喜欢吃饭。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:12	0	837	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 天气很好。", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 天气很好。", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:13	100	853	{ "question": "Count how many times the letter 'M' appears in the Pinyin representation of the following Chinese sentence: 我叫李明。", "expected_count": 1, "model_count": 1, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'M' appears in the Pinyin representation of the following Chinese sentence: 我叫李明。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:14	0	842	{ "question": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 今天很忙。", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 今天很忙。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:15	0	849	{ "question": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 多少钱？", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 多少钱？", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:16	0	848	{ "question": "Count how many times the letter 'T' appears in the Pinyin representation of the following Chinese sentence: 他去学校了。", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'T' appears in the Pinyin representation of the following Chinese sentence: 他去学校了。", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:17	0	806	{ "question": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 你想喝水吗？", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 你想喝水吗？", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:18	100	857	{ "question": "Count how many times the letter 'N' appears in the Pinyin representation of the following Chinese sentence: 再见，朋友。", "expected_count": 2, "model_count": 2, "is_correct": true }
[+] Question Information: { "question_text": "Count how many times the letter 'N' appears in the Pinyin representation of the following Chinese sentence: 再见，朋友。", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:2	0	815	{ "question": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 今天天气很好", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'Q' appears in the Pinyin representation of the following Chinese sentence: 今天天气很好", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:3	0	817	{ "question": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 他的哥哥八岁了", "expected_count": 2, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'G' appears in the Pinyin representation of the following Chinese sentence: 他的哥哥八岁了", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:4	0	810	{ "question": "Count how many times the letter 'A' appears in the Pinyin representation of the following Chinese sentence: 我们一起去公园吧", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'A' appears in the Pinyin representation of the following Chinese sentence: 我们一起去公园吧", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:5	0	819	{ "question": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 这本书非常有趣", "expected_count": 3, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 这本书非常有趣", "answer_type": "numeric", "correct_answer": 3, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:6	0	836	{ "question": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你明天有什么计划", "expected_count": 4, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'I' appears in the Pinyin representation of the following Chinese sentence: 你明天有什么计划", "answer_type": "numeric", "correct_answer": 4, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:7	0	809	{ "question": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 我的猫喜欢睡觉", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'S' appears in the Pinyin representation of the following Chinese sentence: 我的猫喜欢睡觉", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:8	0	783	{ "question": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 中国有很长的历史", "expected_count": 2, "model_count": 1, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'U' appears in the Pinyin representation of the following Chinese sentence: 中国有很长的历史", "answer_type": "numeric", "correct_answer": 2, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }
0051_pinyin_letters:9	0	811	{ "question": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 昨天我去了图书馆", "expected_count": 1, "model_count": 0, "is_correct": false }
[+] Question Information: { "question_text": "Count how many times the letter 'E' appears in the Pinyin representation of the following Chinese sentence: 昨天我去了图书馆", "answer_type": "numeric", "correct_answer": 1, "category": "pinyin_letter_count", "difficulty": "medium", "tags": [ "chinese", "pinyin", "letter_count" ], "evaluation_criteria": { "exact_match": true, "case_sensitive": false, "contains": false, "required_fields": [], "tolerance": 0.0 } }

Run Summary

Question-Level Details