[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let count = $state(0);\n\n    function increment() {\n        count++;\n    }\n\n    function decrement() {\n        count--;\n    }\n</script>\n\n<div class=\"counter\">\n    <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n    <span data-testid=\"count-value\">{count}</span>\n    <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let count = $state(0);\n</script>\n\n<style>\n    .counter {\n        display: flex;\n        align-items: center;\n        gap: 10px;\n    }\n    .counter button {\n        padding: 5px 10px;\n        font-size: 1.2em;\n    }\n</style>\n\n<div class=\"counter\">\n    <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n    <span data-testid=\"count-value\">{count}</span>\n    <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n\nfunction increment() {\n    count += 1;\n}\n\nfunction decrement() {\n    count -= 1;\n}\n</script>\n\n<style>\n.counter {\n  display: flex;\n  align-items: center;\n  gap: 10px;\n}\nbutton {\n  padding: 5px 10px;\n  font-size: 1.2em;\n}\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n\n  function increment() {\n    count++;\n  }\n\n  function decrement() {\n    count--;\n  }\n</script>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 10px;\n    font-family: sans-serif;\n  }\n\n  button {\n    width: 30px;\n    height: 30px;\n    font-size: 18px;\n    cursor: pointer;\n    padding: 5px;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n  }\n\n  span {\n    font-size: 20px;\n    font-weight: bold;\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\nconst increment = () => count++;\nconst decrement = () => count--;\n</script>\n\n<div class=\"counter\">\n    <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n    <span data-testid=\"count-value\">{count}</span>\n    <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n\n  function decrement() {\n    count -= 1;\n  }\n\n  function increment() {\n    count += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 0.5rem;\n  }\n  button {\n    padding: 0.25rem 0.5rem;\n    font-size: 1.2rem;\n    cursor: pointer;\n  }\n  span {\n    font-size: 1.2rem;\n    font-weight: bold;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 10px;\n    font-family: sans-serif;\n  }\n  button {\n    width: 30px;\n    height: 30px;\n    font-size: 18px;\n    cursor: pointer;\n  }\n  span {\n    font-size: 20px;\n    font-weight: bold;\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let count = $state(0);\n</script>\n\n<div class=\"counter\">\n    <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n    <span data-testid=\"count-value\">{count}</span>\n    <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>\n\n<style>\n    .counter {\n        display: flex;\n        gap: 0.5rem;\n        align-items: center;\n    }\n    button {\n        width: 2rem;\n        height: 2rem;\n        font-size: 1.2rem;\n        cursor: pointer;\n    }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" onclick={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" onclick={() => count++}>+</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 8,
    "pass1": 0.7999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let number = $state(5);\n  const doubled = $derived(number * 2);\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n\n  function handleIncrement() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={handleIncrement}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const number = $state(5);\n  const doubled = $derived(number * 2);\n  \n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:8:4 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  const doubled = $derived(number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  const doubled = $derived(number * 2);\n  function handleIncrement() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={handleIncrement}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let number = $state(5);\n    let doubled = $derived(number * 2);\n\n    function increment() {\n        number += 1;\n    }\n</script>\n\n<div>\n    <p data-testid=\"number-value\">Number: {number}</p>\n    <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n    <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  const doubled = $derived(number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  const doubled = $derived(number * 2);\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nconst doubled = $derived(number * 2);\n\nfunction increment() {\n    number += 1;\n}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(() => number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 10\u001b[39m\nReceived:\n\u001b[31m  Doubled: () => get(number) * 2\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 12\u001b[39m\nReceived:\n\u001b[31m  Doubled: () => get(number) * 2\u001b[39m"
        ]
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 6,
    "pass1": 0.5999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const { wordCount, charCount, isLong } = $derived.by(() => {\n    const words = text.split(' ').filter(word => word.length > 0);\n    const wordCount = words.length;\n    const charCount = text.length;\n    const isLong = wordCount > 15;\n    return { wordCount, charCount, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\" on:click={() => text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const { words, characters, isLong } = $derived.by(() => {\n    const words = text.split(/\\s+/).filter(word => word.length > 0).length;\n    const characters = text.length;\n    const isLong = words > 15;\n    return { words, characters, isLong };\n  });\n\n  function clearText() {\n    text = '';\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$words}</p>\n    <p data-testid=\"char-count\">Characters: {$characters}</p>\n    <p data-testid=\"length-indicator\">Status: {$isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "store_invalid_shape\n`words` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "store_invalid_shape\n`words` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "store_invalid_shape\n`words` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const [derived, _cleanup] = $derived.by(() => {\n    const words = text.split(' ').filter(Boolean).length;\n    const chars = text.length;\n    const isLong = text.split(' ').filter(Boolean).length > 15;\n    return { wordCount: words, charCount: chars, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" bind:value={text} type=\"text\" />\n  <button data-testid=\"clear-button\" on:click={() => text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'wordCount')\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'wordCount')\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'wordCount')\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let input = $state('');\n\n  const derivedValues = $derived.by(() => {\n    const words = input.split(/\\s+/).filter(word => word.length > 0).length;\n    const chars = input.length;\n    const isLong = words > 15;\n    const status = isLong ? 'Long text' : 'Short text';\n    \n    return {\n      words,\n      chars,\n      status\n    };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={input} />\n  <button data-testid=\"clear-button\" onclick={() => input = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derivedValues.words}</p>\n    <p data-testid=\"char-count\">Characters: {derivedValues.chars}</p>\n    <p data-testid=\"length-indicator\">Status: {derivedValues.status}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet text = $state(\"\");\n\nconst derivedStats = $derived.by(() => {\n  const wordCount = text ? text.split(/\\s+/).filter(Boolean).length : 0;\n  const charCount = text.length;\n  const isLong = wordCount > 15;\n  return { wordCount, charCount, isLong };\n});\n\nfunction clearText() {\n  text = \"\";\n}\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derivedStats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derivedStats.charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {derivedStats.isLong ? \"Long text\" : \"Short text\"}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"\");\n  const derived = $derived.by(() => {\n    const words = text.split(/\\s+/).filter(Boolean);\n    const wordCount = words.length;\n    const charCount = text.length;\n    const isLong = wordCount > 15;\n    return { wordCount, charCount, isLong };\n  });\n\n  function clear() {\n    text = \"\";\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" bind:value={text} type=\"text\" />\n  <button data-testid=\"clear-button\" on:click={clear}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? \"Long text\" : \"Short text\"}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let text = $state('');\n\n    const stats = $derived.by(() => {\n        const words = text.trim().split(/\\s+/).filter(Boolean).length;\n        const chars = text.length;\n        const isLong = words > 15;\n        return { words, chars, isLong };\n    });\n\n    function clear() {\n        text = '';\n    }\n</script>\n\n<div>\n    <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n    <button data-testid=\"clear-button\" on:click={clear}>Clear</button>\n    <div>\n        <p data-testid=\"word-count\">Words: {stats.words}</p>\n        <p data-testid=\"char-count\">Characters: {stats.chars}</p>\n        <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n    </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const derived = $derived.by(() => ({\n    wordCount: text.split(/\\s+/).filter(w => w).length,\n    charCount: text.length,\n    isLong: text.split(/\\s+/).filter(w => w).length > 15,\n  }));\n\n  function clear() {\n    text = '';\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" bind:value={text} type=\"text\" />\n  <button data-testid=\"clear-button\" on:click={clear}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let text = $state(\"\");\n\n    const $textInfo = $derived.by(text, text => {\n        const words = text.split(/\\s+/).filter(Boolean);\n        return {\n            wordCount: words.length,\n            charCount: text.length,\n            isLong: words.length > 15\n        };\n    });\n\n    function clear() {\n        text = \"\";\n    }\n</script>\n\n<div>\n    <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n    <button data-testid=\"clear-button\" on:click={clear}>Clear</button>\n    <div>\n        <p data-testid=\"word-count\">Words: {$textInfo.wordCount}</p>\n        <p data-testid=\"char-count\">Characters: {$textInfo.charCount}</p>\n        <p data-testid=\"length-indicator\">Status: {$textInfo.isLong ? \"Long text\" : \"Short text\"}</p>\n    </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:10 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const derived = $derived.by({\n    wordCount: (text) => text.trim().split(/\\s+/).filter(Boolean).length,\n    charCount: (text) => text.length,\n    isLong: (text) => text.trim().split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\" on:click={() => text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "fn is not a function\n\n\tin <unknown>\n",
          "fn is not a function\n\n\tin <unknown>\n",
          "fn is not a function\n\n\tin <unknown>\n"
        ]
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n    function addGeorge() {\n        characters = [...characters, \"George\"];\n    }\n</script>\n\n<div>\n    <ul data-testid=\"characters-list\">\n        {#each characters as character}\n            <li data-testid=\"character\">{character}</li>\n        {/each}\n    </ul>\n    <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as name}\n      <li data-testid=\"character\">{name}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n    function addGeorge() {\n        characters.push(\"George\");\n    }\n</script>\n\n<div>\n    <ul data-testid=\"characters-list\">\n        {#each characters as character}\n            <li data-testid=\"character\">{character}</li>\n        {/each}\n    </ul>\n    <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<ul data-testid=\"characters-list\">\n  {#each characters as character}\n    <li data-testid=\"character\">{character}</li>\n  {/each}\n</ul>\n\n<button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n  function addGeorge() {\n    characters = [...characters, \"George\"];\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n    function addGeorge() {\n        characters.push(\"George\");\n    }\n</script>\n\n<div>\n    <ul data-testid=\"characters-list\">\n        {#each characters as character}\n            <li data-testid=\"character\">{character}</li>\n        {/each}\n    </ul>\n    <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n  \n  const addGeorge = () => {\n    characters = [...characters, \"George\"];\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n   let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n   function addGeorge() {\n       characters = [...characters, \"George\"];\n   }\n</script>\n\n<div>\n   <ul data-testid=\"characters-list\">\n       {#each characters as character}\n           <li data-testid=\"character\">{character}</li>\n       {/each}\n   </ul>\n   <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n    <ul data-testid=\"characters-list\">\n        {#each characters as character (character)}\n            <li data-testid=\"character\">{character}</li>\n        {/each}\n    </ul>\n    <button data-testid=\"add-george-button\" on:click={() => characters.push(\"George\")}>\n        Add George\n    </button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 6,
    "pass1": 0.5999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $state(0);\n\n$effect(() => {\n  doubled = number * 2;\n});\n\nfunction increment() {\n  number += 1;\n}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  const [number, doubled] = $state({\n    number: 5,\n    doubled: 0,\n  });\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number++;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:10:4 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(() => number * 2);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\nconst number = $state(5);\nlet doubled = $state(0);\n\n$effect(() => {\n    doubled = number * 2;\n});\n\nfunction increment() {\n    number++;\n}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:12:4 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  const [number, setNumber] = $state(5);\n  const doubled = $derived.by(() => {\n    if (number === 5) return 10;\n    return number * 2;\n  });\n  const increment = () => setNumber(number + 1);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "Cannot use 'in' operator to search for 'Symbol(Symbol.iterator)' in 5\n\n\tin <unknown>\n",
          "Cannot use 'in' operator to search for 'Symbol(Symbol.iterator)' in 5\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<textarea cols=\"40\" rows=\"20\">\n<script>\n  let number = $state(5);\n  let doubled = $derived.by(() => {\n    console.log('Calculating doubled value');\n    return number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>\n</textarea>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:44 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    margin: 1rem auto;\n    text-align: center;\n  }\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<style>\n    .greeting {\n        color: blue;\n        text-align: center;\n        margin: 10px auto;\n    }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n    .greeting {\n        color: blue;\n        text-align: center;\n        margin: 0.5rem auto;\n    }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px auto;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n</script>\n\n<style>\n    .greeting {\n        color: blue;\n        margin: 20px;\n        text-align: center;\n    }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n    Hello, World!\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n.greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px auto;\n}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<style>\n    .greeting {\n        color: blue;\n        margin: 10px auto;\n        text-align: center;\n    }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<style>\n.greeting {\n  color: blue;\n  text-align: center;\n  margin: 0.5rem;\n}\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n.greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n}\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const text = $state('Hello world');\n\n  // Requirement 2: Use basic $inspect to log the input value\n  $inspect(text).log();\n\n  // Requirement 3: Implement $inspect(...).with to track updates with a custom callback\n  $inspect(text).with((val) => {\n    console.log('Custom callback:', val);\n  });\n\n  // Requirement 4: Implement $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    const cleanup = $inspect(text).trace();\n    return () => cleanup();\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:22:46 Cannot bind to constant\nhttps://svelte.dev/e/constant_binding"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const text = $state('Hello world');\n  $inspect(text);\n  $inspect(text).with(() => console.log('Input updated:', text));\n  $effect(() => {\n    $inspect(text).trace();\n  });\n  const charCount = $derived(() => text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:14:46 Cannot bind to constant\nhttps://svelte.dev/e/constant_binding"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  $inspect(text);\n  $inspect(text).with(() => console.log('Input updated:', text));\n  const inspected = $inspect(text).trace();\n  $effect(() => {\n    console.log('Traced input:', inspected);\n  });\n  const charCount = $derived(() => text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'trace')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  $inspect(text);\n  $inspect(text).with(() => console.log('Text changed to:', text));\n  const charCount = $derived(() => text.length);\n  $effect(() => $inspect(text).trace());\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const text = $state(\"Hello world\");\n  \n  $inspect(text);\n  \n  $inspect(text).with((value) => console.log(\"Custom callback:\", value));\n  \n  const charCount = $derived(text.length);\n  \n  $effect(() => {\n    $inspect(text).trace();\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:18:46 Cannot bind to constant\nhttps://svelte.dev/e/constant_binding"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\nconst text = $state(\"Hello world\");\nconst charCount = $derived(text.length);\n\n$inspect(\"Input value\", text);\n\n$inspect(text).with(() => {\n    console.log(\"Text updated:\", text);\n});\n\n$effect(() => {\n    $inspect(text).trace();\n});\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:19:46 Cannot bind to constant\nhttps://svelte.dev/e/constant_binding"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n    // 1. Use Svelte 5's `$state` for a text input starting with \"Hello world\"\n    let text = $state(\"Hello world\");\n\n    // 2. Use basic `$inspect` to log the input value\n    // This makes the value inspectable in development tools\n    const basicInspection = $inspect(text);\n\n    // 3. Implement `$inspect(...).with` to track updates to the input with a custom callback\n    $effect(() => {\n        $inspect(text).with((newValue) => {\n            console.log(\"With callback tracking update:\", newValue);\n        });\n    });\n\n    // 4. Implement `$inspect.trace()` inside an effect that runs when the input changes\n    $effect(() => {\n        $inspect(text).trace();\n    });\n</script>\n\n<div>\n    <!-- 6. Include an input field with `data-testid=\"text-input\"` -->\n    <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n    \n    <!-- 7. Display the input value with `data-testid=\"text-value\"` -->\n    <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n    \n    <!-- 5. Display the character count of the input text (to demonstrate a derived value that depends on the input) -->\n    <!-- 8. Display the character count with `data-testid=\"char-count\"` -->\n    <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  // Requirement 2: Use basic $inspect to log the input value\n  console.log($inspect(text).get());\n\n  // Requirement 3: Implement $inspect(...).with to track updates with a custom callback\n  $inspect(text).with((val) => {\n    console.log(\"Update captured by .with:\", val);\n  });\n\n  // Requirement 4: Implement $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    $inspect(text).trace();\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'get')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'get')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'get')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'get')\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const text = $state(\"Hello world\");\n\n  // Basic $inspect to log the input value\n  $inspect(text);\n\n  // $inspect with custom callback to track updates\n  $inspect(text).with((value) => {\n    console.log(\"Custom callback value:\", value);\n  });\n\n  // $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    $inspect(text).trace();\n  });\n\n  // Derived value to demonstrate a derived state\n  const charCount = $derived(text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:24:46 Cannot bind to constant\nhttps://svelte.dev/e/constant_binding"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  // Requirement 2: Use basic $inspect to log the input value\n  $inspect(text);\n\n  // Requirement 3: Implement $inspect(...).with to track updates\n  $inspect(text).with((val) => console.log(\"Custom callback value:\", val));\n\n  // Requirement 4: Implement $inspect.trace() inside an effect\n  $effect(() => {\n    $inspect(text).trace();\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n"
        ]
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 6,
    "pass1": 0.5999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nconst { name = \"World\", count = 0, showDetails = false } = $props();\nlet localCount = $state(count);\n\nfunction increment() {\n    localCount++;\n}\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {localCount}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {localCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const { name = \"World\", count = 0, showDetails = false } = $props();\n\n    let currentCount = $state(count);\n</script>\n\n<div>\n    <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n    <p data-testid=\"count-display\">Count: {currentCount}</p>\n    <button data-testid=\"increment-button\" on:click={() => currentCount++}>Increment</button>\n\n    {#if showDetails}\n        <div data-testid=\"details\">\n            <p>Name is {name}</p>\n            <p>Count is {currentCount}</p>\n            <p>ShowDetails is true</p>\n        </div>\n    {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n\timport { afterUpdate } from 'svelte';\n\timport { onMount } from 'svelte';\n\n\tconst props = $props<{\n\t\tname?: string;\n\t\tcount?: number;\n\t\tshowDetails?: boolean;\n\t}>();\n\n\tconst state = $state({\n\t\tname: props.name ?? 'World',\n\t\tcount: props.count ?? 0,\n\t\tshowDetails: props.showDetails ?? false,\n\t});\n\n\tconst increment = () => {\n\t\tstate.count++;\n\t};\n\n\tonMount(() => {\n\t\tconsole.log('PropsDemo mounted with:', state);\n\t});\n\n\tafterUpdate(() => {\n\t\tconsole.log('PropsDemo updated with:', state);\n\t});\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {state.name}!</h1>\n\t<p data-testid=\"count-display\">Count: {state.count}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if state.showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {state.name}</p>\n\t\t\t<p>Count is {state.count}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:10 afterUpdate cannot be used in runes mode\nhttps://svelte.dev/e/runes_mode_invalid_import"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"svelte\">\n    const { name = \"World\", count = 0, showDetails = false } = $props();\n    let localCount = $state(count);\n\n    function increment() {\n        localCount += 1;\n    }\n</script>\n\n<div>\n    <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n    <p data-testid=\"count-display\">Count: {localCount}</p>\n    <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n    {#if showDetails}\n        <div data-testid=\"details\">\n            <p>Name is {name}</p>\n            <p>Count is {localCount}</p>\n            <p>ShowDetails is true</p>\n        </div>\n    {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\nconst { name = 'World', count = 0, showDetails = false } = $props();\nlet countState = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {countState}</p>\n  <button data-testid=\"increment-button\" on:click={() => countState++}>Increment</button>\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {countState}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n\timport { onMount } from 'svelte';\n\t\n\tconst props = $props<{\n\t\tname?: string;\n\t\tcount?: number;\n\t\tshowDetails?: boolean;\n\t}>();\n\n\tconst name = $derived(props.name ?? 'World');\n\tconst count = $state(props.count ?? 0);\n\tconst showDetails = $state(props.showDetails ?? false);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:17:2 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const { name, count, showDetails } = $props();\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  {#if $state}\n    <p data-testid=\"count-display\">Count: {count + 1}</p>\n  {:else}\n    <p data-testid=\"count-display\">Count: {count}</p>\n  {/if}\n  <button data-testid=\"increment-button\" onclick={() => count++}>\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:9:7 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  const props = $props<{\n    name?: string\n    count?: number\n    showDetails?: boolean\n  }>()\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {props.count}</p>\n  <button data-testid=\"increment-button\" onclick={() => props.count++}>Increment</button>\n\n  {#if props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {props.name}</p>\n      <p>Count is {props.count}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "Cannot convert object to primitive value"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nconst { name = \"World\", count: initialCount = 0, showDetails = false } = $props();\nlet count = $state(initialCount);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={() => count++}>\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\nconst { name = \"World\", count = 0, showDetails = false } = $props();\nlet myCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {myCount}</p>\n  <button data-testid=\"increment-button\" on:click={() => myCount++}>Increment</button>\n\n  {#if showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {name}</p>\n    <p>Count is {myCount}</p>\n    <p>ShowDetails is true</p>\n  </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "north-mini-code-1.0:q4_K_M",
    "numSamples": 10,
    "numCorrect": 4,
    "pass1": 0.4,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(bookTitle)}\n    <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n    {#each books as book}\n        <li data-testid=\"book-item\">\n            {@render title(book)}\n        </li>\n    {/each}\n</ul>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nconst books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(book)}\n    <span data-testid=\"book-title\">{book}</span>\n{/snippet}\n\n<ul>\n    {#each books as book}\n        <li data-testid=\"book-item\">\n            {@render title(book)}\n        </li>\n    {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  const title = (title: string) => {\n    return `<span data-testid=\"book-title\">${title}</span>`;\n  };\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{@render title(book)}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const books = [\n        \"The Lord of the Rings\",\n        \"To Kill a Mockingbird\",\n        \"1984\"\n    ];\n\n    const title = (bookTitle) => {\n        return <span data-testid=\"book-title\">{bookTitle}</span>;\n    };\n</script>\n\n<ul>\n    {#each books as book}\n        <li data-testid=\"book-item\">{@render title(book)}</li>\n    {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:11:15 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(bookTitle)}\n    <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n    {#each books as book}\n        <li data-testid=\"book-item\">{@render title(book)}</li>\n    {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{@render title(book)}</li>\n  {/each}\n</ul>\n\n{#snippet title(bookTitle)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { snippet } from 'svelte';\n\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  const title = snippet((bookTitle) => {\n    return `<span data-testid=\"book-title\">${bookTitle}</span>`;\n  });\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{@render title(book)}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "snippet is not a function\n\n\tin <unknown>\n",
          "snippet is not a function\n\n\tin <unknown>\n",
          "snippet is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet books = $state([\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]);\n\nexport const title = (book) => () => `<span data-testid=\"book-title\">${book}</span>`;\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  const title = (bookTitle: string) => {\n    return `<li data-testid=\"book-item\"><span data-testid=\"book-title\">${bookTitle}</span></li>`;\n  };\n</script>\n\n<ul>\n  {@render title(books[0])}\n  {@render title(books[1])}\n  {@render title(books[2])}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-item\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[0m \u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[0m \u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-item\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[0m \u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  const title = snippet((bookTitle) => {\n    return {\n      render() {\n        return `<span data-testid=\"book-title\">${bookTitle}</span>`;\n      }\n    };\n  });\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{@render title(book)}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "snippet is not defined\n\n\tin <unknown>\n",
          "snippet is not defined\n\n\tin <unknown>\n",
          "snippet is not defined\n\n\tin <unknown>\n"
        ]
      }
    ],
    "timestamp": "2026-09-14T21:37:20.922Z",
    "tps": 28,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 722,
      "evalDurationNs": 25788840000,
      "promptEvalCount": 285,
      "promptEvalDurationNs": 1588147000,
      "promptTps": 179.45,
      "loadDurationNs": 1492735,
      "measuredAt": "2026-09-16T18:35:00.869Z"
    }
  }
]