[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 3,
    "pass1": 0.30000000000000004,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 12px;\n    font-size: 1.25rem;\n    font-family: monospace;\n  }\n\n  .counter button {\n    cursor: pointer;\n    padding: 4px 12px;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n    background: #f5f5f5;\n    font-size: 1.25rem;\n    line-height: 1;\n  }\n\n  .counter button:hover {\n    background: #e0e0e0;\n  }\n</style>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={() => (count -= 1)}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={() => (count += 1)}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n    font-family: sans-serif;\n    font-size: 1.25rem;\n  }\n\n  .counter button {\n    cursor: pointer;\n    padding: 0.25rem 0.5rem;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n    background: #f5f5f5;\n  }\n\n  .counter button:hover {\n    background: #e0e0e0;\n  }\n</style>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 12px;\n    font-family: monospace;\n    font-size: 24px;\n  }\n\n  .counter button {\n    cursor: pointer;\n    padding: 6px 14px;\n    font-size: 24px;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n    background: #f5f5f5;\n  }\n\n  .counter button:hover {\n    background: #e0e0e0;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 12px;\n    font-size: 1.25rem;\n    font-family: monospace;\n  }\n\n  .counter button {\n    cursor: pointer;\n    padding: 0.25rem 0.75rem;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n    background: #f5f5f5;\n  }\n\n  .counter button:hover {\n    background: #e0e0e0;\n  }\n</style>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n    font-family: sans-serif;\n    font-size: 1.25rem;\n  }\n\n  .counter button {\n    cursor: pointer;\n    font-size: 1.5rem;\n    padding: 0.25rem 0.75rem;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n    background: #f7f7f7;\n  }\n\n  .counter button:hover {\n    background: #eaeaea;\n  }\n</style>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 0.5rem;\n    font-size: 1.5rem;\n    font-family: monospace;\n  }\n\n  .counter button {\n    cursor: pointer;\n    font-size: 1.5rem;\n    padding: 0.25rem 0.5rem;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n    background: #f5f5f5;\n  }\n</style>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 0.5rem;\n    font-size: 1.5rem;\n  }\n\n  .counter button {\n    font-size: 1.5rem;\n    padding: 0.25rem 0.5rem;\n    cursor: pointer;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n    background: #f5f5f5;\n  }\n\n  .counter button:hover {\n    background: #e0e0e0;\n  }\n</style>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n    font-size: 1.5rem;\n  }\n\n  button {\n    font-size: 1.5rem;\n    padding: 0.25rem 0.75rem;\n    cursor: pointer;\n  }\n\n  button:hover {\n    opacity: 0.8;\n  }\n</style>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 12px;\n    font-size: 18px;\n    font-family: sans-serif;\n  }\n\n  .counter button {\n    padding: 6px 14px;\n    font-size: 18px;\n    border: 1px solid #ccc;\n    border-radius: 6px;\n    background-color: #f5f5f5;\n    cursor: pointer;\n  }\n\n  .counter button:hover {\n    background-color: #e0e0e0;\n  }\n\n  .counter[data-testid=\"count-value\"] {\n    min-width: 24px;\n    text-align: center;\n  }\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<button data-testid=\"increment-button\" onclick={increment}>Increment</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:14:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n{svelte.onMount(() => {\n  const button = document.querySelector('[data-testid=\"increment-button\"]');\n  button.addEventListener('click', () => {\n    number += 1;\n  });\n});}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:19:2 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n  function increment() {\n    number++;\n  }\n</script>\n\n<button data-testid=\"increment-button\" on:click={increment}>Increment</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<button\n\tdata-testid=\"increment-button\"\n\tonclick={increment}>\n\tIncrement\n</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $derived(number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<button data-testid=\"increment-button\" on:click={increment}>\n  Increment\n</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:14:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $derived(number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n  function increment() {\n    number += 1;\n  }\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:19:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<button\n  data-testid=\"increment-button\"\n  onclick={increment}\n>\n  Increment\n</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<button data-testid=\"increment-button\" onclick={increment}>Increment</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { onMount } = $props();\n\n  let text = $state('');\n\n  const wordCount = $derived.by(() => {\n    return text.trim() === '' ? 0 : text.trim().split(/\\s+/).length;\n  });\n\n  const charCount = $derived.by(() => text.length);\n\n  const isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:this={inputEl}\n    bind:value={text}\n    oninput={(e) => (text = e.currentTarget.value)}\n  />\n  <button data-testid=\"clear-button\" onclick={() => (text = '')}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "inputEl is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "inputEl is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "inputEl is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const wordCount = $derived.by(() => {\n    return text.trim() === '' ? 0 : text.trim().split(/\\s+/).length;\n  });\n\n  const charCount = $derived.by(() => text.length);\n\n  const isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:this={undefined}\n    value={text}\n    oninput={(e) => (text = e.target.value)}\n  />\n  <button data-testid=\"clear-button\" onclick={() => (text = '')}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "Cannot assign to read only property 'undefined' of object '#<Object>'\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "Cannot assign to read only property 'undefined' of object '#<Object>'\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "Cannot assign to read only property 'undefined' of object '#<Object>'\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { text = '' } = $props();\n\n  $state({ text: '' });\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:this={inputEl}\n    oninput={(e) => (text.value = e.target.value)}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount.value}</p>\n    <p data-testid=\"char-count\">Characters: {charCount.value}</p>\n    <p data-testid=\"length-indicator\">Status: {longText.value ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:2 `$state(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const wordCount = $derived.by(() => {\n    const trimmed = text.trim();\n    if (trimmed === '') return 0;\n    return trimmed.split(/\\s+/).length;\n  });\n\n  const charCount = $derived.by(() => text.length);\n\n  const isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:this={text}\n    oninput={(e) => (text = e.target.value)}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "get(...).trim is not a function\n\n\tin <unknown>\n",
          "get(...).trim is not a function\n\n\tin <unknown>\n",
          "get(...).trim is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const wordCount = $derived.by(() => {\n    if (!text.trim()) return 0;\n    return text.trim().split(/\\s+/).length;\n  });\n\n  const charCount = $derived.by(() => text.length);\n\n  const isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:this={text}\n    oninput={(e) => (text = e.currentTarget.value)}\n  />\n  <button data-testid=\"clear-button\" onclick={() => (text = '')}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "get(...).trim is not a function\n\n\tin <unknown>\n",
          "get(...).trim is not a function\n\n\tin <unknown>\n",
          "get(...).trim is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst wordCount = $derived.by(() => {\n\t\treturn text.trim().length === 0 ? 0 : text.trim().split(/\\s+/).length;\n\t});\n\n\tconst charCount = $derived.by(() => {\n\t\treturn text.length;\n\t});\n\n\tconst isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tvalue={text}\n\t\toninput={(e) => (text = e.target.value)}\n\t/>\n\t<button data-testid=\"clear-button\">Clear</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Words: 0\u001b[39m\nReceived:\n\u001b[31m  Words: 2\u001b[39m"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const wordCount = $derived.by(() => {\n    return text.trim().length === 0\n      ? 0\n      : text.trim().split(/\\s+/).length;\n  });\n\n  const charCount = $derived.by(() => text.length);\n\n  const isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>\n\n<script>\n  function clearText() {\n    text = '';\n  }\n</script>\n\n<button data-testid=\"clear-button\" on:click={clearText}>Clear</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:32:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const wordCount = $derived.by(() =>\n    text.trim().length === 0 ? 0 : text.trim().split(/\\s+/).length\n  );\n\n  const charCount = $derived.by(() => text.length);\n\n  const isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:this={{ value: text }}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:19:4 Can only bind to an Identifier or MemberExpression or a `{get, set}` pair\nhttps://svelte.dev/e/bind_invalid_expression"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { onMount, text, setText, words, chars, isLong, clear } = $derived.by(() => ({\n    onMount: null,\n    text: $state(''),\n    words: $derived.by(() => text.trim().length === 0 ? 0 : text.trim().split(/\\s+/).length),\n    chars: $derived.by(() => text.length),\n    isLong: $derived.by(() => words > 15),\n    clear: () => setText('')\n  }));\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" on:click={() => clear()}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {words}</p>\n    <p data-testid=\"char-count\">Characters: {chars}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:10 `$state(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { children = null } = $props();\n\n  let text = $state('');\n\n  let wordCount = $derived.by(() =>\n    text.trim().length === 0 ? 0 : text.trim().split(/\\s+/).length\n  );\n\n  let charCount = $derived.by(() => text.length);\n\n  let isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n  />\n\n  <button data-testid=\"clear-button\">Clear</button>\n\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Words: 0\u001b[39m\nReceived:\n\u001b[31m  Words: 2\u001b[39m"
        ]
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 4,
    "pass1": 0.4,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>\n\n{svelte.onMount(() => {\n  const button = document.querySelector('[data-testid=\"add-george-button\"]');\n  button.addEventListener(\"click\", () => {\n    characters.push(\"George\");\n  });\n});}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:21:2 Expected token }\nhttps://svelte.dev/e/expected_token"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>\n\n{svelte:on}\n  {#if $dirty}\n    <script context=\"module\">\n      import { onMount } from 'svelte';\n    </script>\n  {/if}\n{/svelte:on}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:16:7 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>\n\n{svelte.onMount((node) => {\n  const button = node.querySelector('[data-testid=\"add-george-button\"]');\n  button?.addEventListener(\"click\", () => {\n    characters = [...characters, \"George\"];\n  });\n});}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:21:2 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>\n\n<svelte:window\n  on:click={(event) => {\n    if (event.target?.dataset?.testid !== \"add-george-button\") return;\n    characters.push(\"George\");\n  }}\n/>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\n    \"Jerry\",\n    \"Elaine\",\n    \"Kramer\",\n  ]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>\n\n{svelte:on}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:20:7 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<button data-testid=\"increment-button\" onclick={increment}>Increment</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:19:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n\tlet doubled = $state(0);\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n{#snippet increment()}\n  number += 1;\n{/snippet}",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<script>\n\tlet doubled = $state(0);\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\timport { $effect, $state } from 'svelte';\n\n\tlet number = $state(5);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<style>\n\tdiv {\n\t\tfont-family: sans-serif;\n\t\tfont-size: 18px;\n\t}\n\tbutton {\n\t\tmargin-top: 10px;\n\t\tpadding: 6px 12px;\n\t\tcursor: pointer;\n\t}\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:10 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n{svelte:action(...)}\n\nWait, I included an accidental extra block. Here is the clean implementation:\n\n<script>\n  let number = $state(5);\n\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:19:7 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 8,
    "pass1": 0.7999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { children = 'Hello, World!' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">{children}</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\texport default function HelloWorld() {}\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n\tgreeting {\n\t\tcolor: blue;\n\t\tfont-size: 2rem;\n\t\tmargin: 1rem;\n\t\ttext-align: center;\n\t}\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:1 A component cannot have a default export\nhttps://svelte.dev/e/module_illegal_default_export"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const greeting = \"Hello, World!\";\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  {greeting}\n</div>\n\n<style>\n  :global(body) {\n    margin: 0;\n    font-family: sans-serif;\n  }\n\n  .greeting {\n    color: blue;\n    margin: 1rem;\n    text-align: center;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { children = 'Hello, World!' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">{children}</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  export let message = \"Hello, World!\";\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">{message}</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n\tlet message = \"Hello, World!\";\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\t{message}\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px;\n\t\tfont-size: 2rem;\n\t\tfont-family: sans-serif;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { name = 'World' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  Hello, {name}!\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let { name = 'World' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, {name}!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let { name = 'World' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, {name}!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n\n  html,\n  body {\n    margin: 0;\n    padding: 0;\n    height: 100%;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let { name = \"World\" } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, {name}!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 2,
    "pass1": 0.20000000000000007,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n\n  // Basic $inspect to log the input value\n  $inspect(text);\n\n  // $inspect(...).with to track updates with a custom callback\n  $inspect(text).with((value, prev) => {\n    console.log('Text updated:', value, 'from:', prev);\n  });\n\n  // $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    const trace = $inspect.trace();\n    trace.set('text', text);\n    console.log('Input changed:', trace.get('text'));\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:18 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect } from 'svelte';\n\n  const text = $state('Hello world');\n\n  // 1. Basic $inspect to log the input value\n  $inspect(text);\n\n  // 2. $inspect(...).with to track updates with a custom callback\n  $inspect(text).with((value) => {\n    console.log('Input updated to:', value);\n  });\n\n  // 3. $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    const trace = $inspect.trace(text);\n    console.log('Reactive trace for text:', trace.value);\n    return trace;\n  });\n\n  const charCount = $derived(text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n\n  $inspect(text);\n\n  $inspect(text).with((value) => {\n    console.log('text updated to:', value);\n  });\n\n  $effect(() => {\n    $inspect.trace('input changed:', text);\n    console.log('Character count:', text.length);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:13:4 `$inspect.trace` must be called with zero or one arguments\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  // Requirement 2: basic $inspect to log the input value\n  $inspect(() => {\n    console.log(\"text:\", text);\n  });\n\n  // Requirement 3: $inspect(...).with to track updates with a custom callback\n  $inspect(text).with((value) => {\n    console.log(\"text updated to:\", value);\n  });\n\n  // Derived value that depends on the input\n  let charCount = $derived(text.length);\n\n  // Requirement 4: $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    $inspect.trace(text, { label: \"text\" });\n    console.log(\"effect ran:\", text, charCount);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:21:4 `$inspect.trace` must be called with zero or one arguments\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  $inspect(text);\n\n  $inspect(text).with((value) => {\n    console.log(\"Input updated:\", value);\n  });\n\n  $effect(() => {\n    $inspect.trace(\"Input changed to:\", text);\n  });\n\n  const charCount = $derived(text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:13:4 `$inspect.trace` must be called with zero or one arguments\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { value = \"Hello world\" } = $props();\n\n  let text = $state(value);\n  let characterCount = $derived(text.length);\n\n  // Basic $inspect: logs the input value whenever it changes\n  $inspect(text);\n\n  // $inspect(...).with: tracks updates with a custom callback\n  $inspect(text).with((newValue, meta) => {\n    console.log(`[${meta.direction}] text updated to \"${newValue}\" at ${meta.time}`);\n  });\n\n  // $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    const trace = $inspect.trace(text);\n    console.log(\"input changed:\", trace.last);\n    return () => {\n      trace.stop();\n    };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:19:18 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  // Basic $inspect to log the input value\n  $inspect(text);\n\n  // Track updates to the input with a custom callback\n  $inspect(text).with((previous, next) => {\n    console.log(`Text updated from \"${previous}\" to \"${next}\"`);\n  });\n\n  // $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    $inspect.trace(text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $inspect } from 'svelte/inspector';\n\n  let text = $state('Hello world');\n\n  // Basic $inspect to log the input value\n  $inspect(text);\n\n  // $inspect(...).with to track updates with a custom callback\n  $inspect(text).with((prev, next) => {\n    console.log(`Input changed: \"${prev}\" -> \"${next}\"`);\n  });\n\n  // $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    const trace = $inspect.trace();\n    console.log(`Character count: ${trace.text.length}`);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  // Basic $inspect to log the input value\n  $inspect(text);\n\n  // Track updates to the input with a custom callback\n  $inspect(text).with((old, next) => {\n    console.log(\"Text updated:\", old, \"->\", next);\n  });\n\n  // $inspect.trace() inside an effect that runs when the input changes\n  $effect(() => {\n    $inspect.trace(() => {\n      console.log(\"Traced text change:\", text);\n    });\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  $inspect(...).with(() => {\n    console.log(\"text updated\", text);\n  });\n\n  $effect(() => {\n    $inspect.trace({ text });\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:14 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const {\n    name = \"World\",\n    count = 0,\n    showDetails = false\n  } = $props();\n\n  const countState = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {countState}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {countState}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "expected 'Count: 10' to match /11/"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const {\n    name = \"World\",\n    count = 0,\n    showDetails = false\n  } = $props();\n\n  const stateCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {stateCount}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {stateCount}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>\n\n{svelte:window on:click={() => {\n  // keep the button reference accessible if needed\n}}}\n\n<button data-testid=\"increment-button\" onclick={() => (stateCount += 1)}>\n  Increment\n</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:27:7 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet {\n\t\tname = \"World\",\n\t\tcount = 0,\n\t\tshowDetails = false\n\t} = $props();\n\n\tlet incrementCount = () => {\n\t\tcount = count + 1;\n\t};\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "expected 'Count: 10' to match /11/"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const {\n    name = \"World\",\n    count = 0,\n    showDetails = false\n  } = $props();\n\n  let countState = $state(count);\n\n  function increment() {\n    countState++;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {countState}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {countState}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>\n\n<style>\n  h1 {\n    font-size: 2rem;\n  }\n</style>",
        "success": false,
        "errors": [
          "expected 'Count: 10' to match /11/"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\timport { $props, $state } from 'svelte';\n\n\tconst props = $props({\n\t\tname: 'World',\n\t\tcount: 0,\n\t\tshowDetails: false\n\t});\n\n\tconst count = $state(props.count);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {props.name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button\n\t\tdata-testid=\"increment-button\"\n\t\tonclick={() => (count = count + 1)}\n\t>\n\t\tIncrement\n\t</button>\n\n\t{#if props.showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {props.name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:10 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const {\n    name = 'World',\n    count = 0,\n    showDetails = false\n  } = $props();\n\n  let countState = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {countState}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {countState}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>\n\n<script>\n  function increment() {\n    countState += 1;\n  }\n</script>\n\n{#key incrementButton}\n  <button data-testid=\"increment-button\" onclick={increment}>\n    Increment\n  </button>\n{/key}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:27:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet {\n\t\tname = \"World\",\n\t\tcount = 0,\n\t\tshowDetails = false\n\t} = $props();\n\n\tlet localCount = $state(count);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {localCount}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {localCount}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>\n\n<script>\n\tfunction increment() {\n\t\tlocalCount += 1;\n\t}\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:27:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const props = $props({\n    name: \"World\",\n    count: 0,\n    showDetails: false\n  });\n\n  let count = $state(props.count);\n\n  function increment() {\n    count += 1;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {props.name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:16 `$props` cannot be called with arguments\nhttps://svelte.dev/e/rune_invalid_arguments"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet {\n\t\tname = \"World\",\n\t\tcount = 0,\n\t\tshowDetails = false\n\t} = $props();\n\n\tlet countState = $state(count);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countState}</p>\n\t<button data-testid=\"increment-button\">Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>\n\n{svelte:window\n\tonclick={() => {\n\t\tcountState++;\n\t}}\n}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:27:7 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const { name = \"World\", showDetails = false } = $props();\n\n  const count = $state(0);\n\n  function increment() {\n    count += 1;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:9:4 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "ornith-1.5:9b-q4_K_M",
    "numSamples": 10,
    "numCorrect": 5,
    "pass1": 0.5,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>\n\n{#snippet title(book)}\n  <span data-testid=\"book-title\">{book}</span>\n{/snippet}",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  let { title } = $props();\n</script>\n\n{@snippet title (title: string)}\n  <span data-testid=\"book-title\">{title}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:9:2 Expected 'html', 'render', 'attach', 'const', or 'debug'\nhttps://svelte.dev/e/expected_tag"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>\n\n{#snippet title(book)}\n  <span data-testid=\"book-title\">{book}</span>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>\n\nsnippet title(book: string) {\n  <span data-testid=\"book-title\">{book}</span>\n}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:20:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let title = (bookTitle) => {\n    return (\n      <span data-testid=\"book-title\">\n        {bookTitle}\n      </span>\n    );\n  };\n\n  const books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\",\n  ]);\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    {@render title(book)}\n  {/each}\n</ul>\n\n{#snippet title(book)}\n  <li data-testid=\"book-item\">\n    <span data-testid=\"book-title\">{book}</span>\n  </li>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>\n\n{#snippet title(book)}\n  <span data-testid=\"book-title\">{book}</span>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  let title = snippet((book: string) => {\n    return <span data-testid=\"book-title\">{book}</span>;\n  });\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:27 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    {@render title(book)}\n  {/each}\n</ul>\n\n#title={ (book) => (\n  <li data-testid=\"book-item\">\n    <span data-testid=\"book-title\">{book}</span>\n  </li>\n)}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:18:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ];\n</script>\n\n{#each books as book}\n  <li data-testid=\"book-item\">\n    {@render title(book)}\n  </li>\n{/each}\n\n{#snippet title(book)}\n  <span data-testid=\"book-title\">{book}</span>\n{/snippet}",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-11-01T17:06:04.964Z",
    "tps": 12.98,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 273,
      "evalDurationNs": 21032970000,
      "promptEvalCount": 290,
      "promptEvalDurationNs": 166308000,
      "promptTps": 1743.75,
      "loadDurationNs": 699373,
      "measuredAt": "2026-11-01T17:06:27.173Z"
    }
  }
]