{
  "schema": "s1-mini-task-gate/2",
  "subject": "s1_mini_decode_int8lin",
  "hf_id": "superwhisper/s1-mini",
  "revision": null,
  "verdict_reference": "/Users/majimadaisuke/code/coreai/coreai-models-community/_smoke/s1_mini_task_ref.json",
  "protocol": {
    "greedy": true,
    "max_new_tokens": 192,
    "enable_thinking": false,
    "engine": "coreai-pipelined, warmup off, COREAI_CHUNK_THRESHOLD=1"
  },
  "passed": 13,
  "of": 14,
  "card_agreement": 9,
  "cases": [
    {
      "case": "ex.report",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "so um i need to like send the the report by uh friday no wait make that thursday",
      "got": "So I need to send the report by Thursday.",
      "card_expected": "I need to send the report by Thursday.",
      "matches_card": false,
      "matches_reference": true,
      "prompt_tokens": 97
    },
    {
      "case": "ex.number",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "i think the answer is forty two no sorry forty three",
      "got": "I think the answer is 43.",
      "card_expected": "I think the answer is 43.",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 89
    },
    {
      "case": "ex.time",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "let's meet at half past two tomorrow uh actually make it three fifteen p m",
      "got": "Let's meet at 3:15pm tomorrow.",
      "card_expected": "Let's meet at 3:15pm tomorrow.",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 94
    },
    {
      "case": "ex.money",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "the invoice came to twenty three thousand four hundred and fifty dollars and it's due on march third twenty twenty six",
      "got": "The invoice came to $23,450 and it's due on March 3, 2026.",
      "card_expected": "The invoice came to $23,450, and it's due on March 3, 2026.",
      "matches_card": false,
      "matches_reference": false,
      "prompt_tokens": 100
    },
    {
      "case": "ex.email-addr",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "send it to support at superwhisper dot com",
      "got": "Send it to support@superwhisper.com.",
      "card_expected": "Send it to support@superwhisper.com.",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 88
    },
    {
      "case": "ex.filler",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "um",
      "got": "",
      "card_expected": "",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 79
    },
    {
      "case": "style.casual",
      "styling": "casual",
      "structure": "prose",
      "context": "general",
      "input": "hmm im gonna be late theres a cute dog outside i cant just walk past him",
      "got": "hmm im gonna be late. theres a cute dog outside. i cant just walk past him",
      "card_expected": "hmm im gonna be late. theres a cute dog outside. i cant just walk past him",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 93
    },
    {
      "case": "style.semi-casual",
      "styling": "semi-casual",
      "structure": "prose",
      "context": "general",
      "input": "hmm im gonna be late theres a cute dog outside i cant just walk past him",
      "got": "hmm, I'm gonna be late. there's a cute dog outside. I can't just walk past him",
      "card_expected": "hmm, I'm gonna be late. there's a cute dog outside. I can't just walk past him",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 96
    },
    {
      "case": "style.semi-formal",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "hmm im gonna be late theres a cute dog outside i cant just walk past him",
      "got": "Hmm, I'm going to be late. There's a cute dog outside. I can't just walk past him.",
      "card_expected": "I'm going to be late. There's a cute dog outside. I can't just walk past him.",
      "matches_card": false,
      "matches_reference": true,
      "prompt_tokens": 95
    },
    {
      "case": "style.formal",
      "styling": "formal",
      "structure": "prose",
      "context": "general",
      "input": "hmm im gonna be late theres a cute dog outside i cant just walk past him",
      "got": "Hmm, I am going to be late. There is a cute dog outside. I cannot just walk past him.",
      "card_expected": "I am going to be late. There is a cute dog outside. I cannot just walk past him.",
      "matches_card": false,
      "matches_reference": true,
      "prompt_tokens": 93
    },
    {
      "case": "struct.prose",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "so for the trip we need to pack sunscreen and then also a first aid kit and um chargers for everything",
      "got": "So for the trip, we need to pack sunscreen and then also a first aid kit and chargers for everything.",
      "card_expected": "So for the trip, we need to pack sunscreen and then also a first aid kit and chargers for everything.",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 100
    },
    {
      "case": "struct.lists",
      "styling": "semi-formal",
      "structure": "lists",
      "context": "general",
      "input": "so for the trip we need to pack sunscreen and then also a first aid kit and um chargers for everything",
      "got": "So for the trip, we need to pack sunscreen and then also a first aid kit and chargers for everything.",
      "card_expected": "So for the trip, we need to pack:\n- Sunscreen\n- A first aid kit\n- Chargers for everything",
      "matches_card": false,
      "matches_reference": true,
      "prompt_tokens": 100
    },
    {
      "case": "ctx.general",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "general",
      "input": "hey sarah just wanted to follow up on the proposal can you send the numbers by end of week thanks john",
      "got": "Hey Sarah, just wanted to follow up on the proposal. Can you send the numbers by end of week? Thanks, John.",
      "card_expected": "Hey Sarah, just wanted to follow up on the proposal. Can you send the numbers by end of week? Thanks, John.",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 100
    },
    {
      "case": "ctx.email",
      "styling": "semi-formal",
      "structure": "prose",
      "context": "email",
      "input": "hey sarah just wanted to follow up on the proposal can you send the numbers by end of week thanks john",
      "got": "Hey Sarah,\n\nJust wanted to follow up on the proposal. Can you send the numbers by end of week?\n\nThanks,\nJohn",
      "card_expected": "Hey Sarah,\n\nJust wanted to follow up on the proposal. Can you send the numbers by end of week?\n\nThanks,\nJohn",
      "matches_card": true,
      "matches_reference": true,
      "prompt_tokens": 100
    }
  ]
}