[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = 0;\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:35:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:52:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:66:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const countStore = $state(0);\n\n  function increment() {\n    countStore.set(countStore.get() + 1);\n  }\n\n  function decrement() {\n    countStore.set(countStore.get() - 1);\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{countStore.get()}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "TypeError: countStore.get is not a function\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:47\n    at Array.map (<anonymous>)",
          "TypeError: countStore.get is not a function\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:47\n    at Array.map (<anonymous>)",
          "TypeError: countStore.get is not a function\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:47\n    at Array.map (<anonymous>)",
          "TypeError: countStore.get is not a function\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:47\n    at Array.map (<anonymous>)"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state.set(0);\n\n  function increment() {\n    $state.set($state.get() + 1);\n  }\n\n  function decrement() {\n    $state.set($state.get() - 1);\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">${$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$state.set` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => $count = $count - 1}>-</button>\n  <span data-testid=\"count-value\">{$count}</span>\n  <button data-testid=\"increment-button\" on:click={() => $count = $count + 1}>+</button>\n</div>",
        "success": false,
        "errors": [
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:9:36",
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:9:36",
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:9:36",
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:9:36"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => $state = $state - 1}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={() => $state = $state + 1}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:35:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:52:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:66:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = 0;\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n</script>\n\n<style>\n  .counter {\n    display: inline-flex;\n    align-items: center;\n    gap: 0.5rem;\n    padding: 0.5rem 1rem;\n    border: 1px solid #ccc;\n    border-radius: 4px;\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:35:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:52:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:66:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"svelte-5\">\n  const count = $state(0);\n  function inc() {\n    count.update(n => n + 1);\n  }\n  function dec() {\n    count.update(n => n - 1);\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={dec}>-</button>\n  <span data-testid=\"count-value\">{$count}</span>\n  <button data-testid=\"increment-button\" on:click={inc}>+</button>\n</div>",
        "success": false,
        "errors": [
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:36",
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:36",
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:36",
          "Svelte error: store_invalid_shape\n`count` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n    at $count (/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:36"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { writable } from 'svelte/store';\n  const state = writable(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => state.update(x => x - 1)}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={() => state.update(x => x + 1)}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = 0;\n\tfunction increment() {\n\t\tcount += 1;\n\t}\n\tfunction decrement() {\n\t\tcount -= 1;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:35:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:52:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/counter.test.ts:66:26\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state(5, number => {\n    $derived(number, doubled => doubled = number * 2);\n  });\n\n  function increment() {\n    $state(number + 1);\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$state(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst number = $state(5);\n\tconst doubled = $derived(() => number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number = number + 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:11:56 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const number = $state(5);\n  const doubled = $derived(number, (n) => n * 2);\n\n  function increment() {\n    number = number + 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:18 `$derived` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $let number = $state(5);\n  $let doubled = $derived(() => number * 2, [number]);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $update(number, n => n + 1)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:7 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const num = $state(5);\n  const doubled = $derived(() => num * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {num}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => num += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:11:57 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state.number = 5;\n  $derived($state.number, n => n * 2, doubled);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$state.number` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  $let number = $state(5);\n  $let doubled = $derived(() => number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $assign(number, number + 1)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:7 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(() => number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number = $state(number + 1)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:11:66 `$state(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let number = $state(5);\n  let doubled;\n  $derived(() => number * 2, doubled);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onClick={() => $set(number, number + 1)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:56 `$set` is an illegal variable name. To reference a global variable called `$set`, use `globalThis.$set`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const number = $state(5);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived(() => number * 2)}</p>\n  <button data-testid=\"increment-button\" on:click={() => $set(number, number + 1)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:57 `$set` is an illegal variable name. To reference a global variable called `$set`, use `globalThis.$set`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state.text = '';\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.set({ text: '' })}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.by($state.text, (text) => text.trim().split(/\\s+/).length)}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.by($state.text, (text) => text.length)}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.by($state.text, (text) => {\n      const words = text.trim().split(/\\s+/).length;\n      return words > 15 ? 'Long text' : 'Short text';\n    })}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$state.text` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const text = $state('');\n\n    const derived = $derived.by(text, () => {\n        const words = text.trim().split(/\\s+/).filter(Boolean);\n        return {\n            wordCount: words.length,\n            charCount: text.length,\n            isLong: words.length > 15\n        };\n    });\n</script>\n\n<div>\n    <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n    <button data-testid=\"clear-button\" on:click={() => $text = ''}>Clear</button>\n    <div>\n        <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n        <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n        <p data-testid=\"length-indicator\">\n            {derived.isLong ? 'Status: Long text' : 'Status: Short text'}\n        </p>\n    </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:20 `$derived.by` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$state text = '';\n\t$derived.by text = (text) => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean);\n\t\tconst wordCount = words.length;\n\t\tconst charCount = text.length;\n\t\tconst isLong = wordCount > 15;\n\t\treturn { wordCount, charCount, isLong };\n\t};\n\t\n\tfunction clear() {\n\t\t$text = '';\n\t}\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<button data-testid=\"clear-button\" on:click={clear}>Clear</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = '';\n\tconst derivedState = $derived.by(\n\t\t() => [text],\n\t\t() => {\n\t\t\tconst words = text.split(/\\s+/).filter(Boolean).length;\n\t\t\tconst chars = text.length;\n\t\t\tconst isLong = words > 15;\n\t\t\treturn { words, chars, isLong };\n\t\t}\n\t);\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<button data-testid=\"clear-button\" on:click={() => (text = '')}>Clear</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {derivedState.words}</p>\n\t\t<p data-testid=\"char-count\">Characters: {derivedState.chars}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {derivedState.isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:22 `$derived.by` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$state text = \"\";\n\n\tconst wordCount = $derived.by($text, text => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean);\n\t\treturn words.length;\n\t});\n\n\tconst charCount = $derived.by($text, text => text.length);\n\n\tconst isLong = $derived.by($text, text => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean);\n\t\treturn words.length > 15;\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<button data-testid=\"clear-button\" on:click={() => (text = \"\")}>Clear</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {isLong ? \"Long text\" : \"Short text\"}</p>\n\t</div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state(\"\");\n  const derived = $derived.by($state, (text) => {\n    const words = text.trim().split(/\\s+/).filter(Boolean).length;\n    const chars = text.length;\n    const isLong = words > 15;\n    return { words, chars, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={value} on:input={(e) => $state(e.target.value)} />\n  <button data-testid=\"clear-button\" on:click={() => $state('')}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {derived.chars}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$state(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state.text = '';\n  $derived.by(() => {\n    const text = $text;\n    const words = text.trim().split(/\\s+/).length;\n    const chars = text.length;\n    const isLong = words > 15;\n    return { words, chars, isLong };\n  }, 'textDerivation');\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$text} />\n  <button data-testid=\"clear-button\" on:click={() => $text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.textDerivation.words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.textDerivation.chars}</p>\n    <p data-testid=\"length-indicator\">Status: {($derived.textDerivation.isLong ? 'Long text' : 'Short text')}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:17 Cannot subscribe to stores that are not declared at the top level of the component\nhttps://svelte.dev/e/store_invalid_scoped_subscription"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$state(text = '');\n\tconst derived = $derived.by(text, () => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.length;\n\t\tconst isLong = words > 15;\n\t\treturn { words, chars, isLong };\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<button data-testid=\"clear-button\" on:click={() => { text = ''; }}>Clear</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {derived.words}</p>\n\t\t<p data-testid=\"char-count\">Characters: {derived.chars}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:1 `$state(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$state text = '';\n\tconst metrics = $derived.by(text, text => {\n\t\tconst words = text.split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.length;\n\t\tconst isLong = words > 15;\n\t\treturn { words, chars, isLong };\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" bind:value={text} type=\"text\" />\n\t<button data-testid=\"clear-button\" on:click={() => (text = '')}>Clear</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {metrics.words}</p>\n\t\t<p data-testid=\"char-count\">Characters: {metrics.chars}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {metrics.isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const text = $state(\"\");\n\n  $derived.by('words', () => {\n    return text.trim().split(/\\s+/).filter(Boolean).length;\n  });\n\n  $derived.by('chars', () => {\n    return text.length;\n  });\n\n  $derived.by('isLong', () => {\n    return text.trim().split(/\\s+/).filter(Boolean).length > 15;\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\" on:click={() => (text = \"\")}>Clear</button>\n\n  <div>\n    <p data-testid=\"word-count\">Words: {$words}</p>\n    <p data-testid=\"char-count\">Characters: {$chars}</p>\n    <p data-testid=\"length-indicator\">Status: {$isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:24:40 `$words` is an illegal variable name. To reference a global variable called `$words`, use `globalThis.$words`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  $: count = characters.length;\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{{ character }}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={() => characters.push('George')}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    characters = [...characters, 'George'];\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{{ char }}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Jerry\u001b[39m\nReceived:\n\u001b[31m  [object Object]\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:12:34\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20",
          "AssertionError: expected 3 to be 4 // Object.is equality\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:32:38\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  \n  function addGeorge() {\n    characters = [...characters, 'George'];\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">\n        {character}\n      </li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "AssertionError: expected 3 to be 4 // Object.is equality\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:32:38\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{{ character }}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={() => characters.push('George')}>\n    Add George\n  </button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Jerry\u001b[39m\nReceived:\n\u001b[31m  [object Object]\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:12:34\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20",
          "AssertionError: expected 3 to be 4 // Object.is equality\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:32:38\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{{character}}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={() => $characters = $characters.concat('George')}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:7 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"js\">\n  let characters = [\"Jerry\", \"Elaine\", \"Kramer\"];\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{{ char }}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Jerry\u001b[39m\nReceived:\n\u001b[31m  [object Object]\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:12:34\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20",
          "AssertionError: expected 3 to be 4 // Object.is equality\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:32:38\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$let chars = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each chars as char}\n\t\t\t<li data-testid=\"character\">{char}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" on:click={() => chars.push('George')}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let chars = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    chars = [...chars, 'George'];\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each chars as char}\n      <li data-testid=\"character\">{{ char }}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Jerry\u001b[39m\nReceived:\n\u001b[31m  [object Object]\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:12:34\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20",
          "AssertionError: expected 3 to be 4 // Object.is equality\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:32:38\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, 'George'];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">\n\t\t\t\t{character}\n\t\t\t</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{{ character }}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Jerry\u001b[39m\nReceived:\n\u001b[31m  [object Object]\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:12:34\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20",
          "AssertionError: expected 3 to be 4 // Object.is equality\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/each.test.ts:32:38\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n$state number = 5;\n\nlet doubled;\n\n$effect(() => {\n\tdoubled = number * 2;\n});\n\nfunction increment() {\n\tnumber = number + 1;\n}\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:8:21 Expected token }\nhttps://svelte.dev/e/expected_token"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const number = $state(5);\n  const doubled = $effect(() => number * 2);\n\n  function increment() {\n    number = number + 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:18 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$let number = 5;\n\t$let doubled = 0;\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber = number + 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n  let doubled;\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 10\u001b[39m\nReceived:\n\u001b[31m  Doubled:\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/effect.test.ts:11:49\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/effect.test.ts:20:48\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$let number = 5;\n\t$let doubled;\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { set } from 'svelte';\n\n  $let number = 5;\n  $let doubled = 0;\n\n  $effect(() => {\n    doubled = number * 2;\n  }, [number]);\n\n  function increment() {\n    set(number, number + 1);\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:7 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"svelte-5\">\n\t$state<number> number = 5;\n\t$state<number> doubled = 0;\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number = number + 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:1 Assigning to rvalue\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const number = $state(5);\n  let doubled = 0;\n  $effect(() => {\n    doubled = number * 2;\n    return null;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:15:57 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const { number, setNumber } = $state(5);\n  let doubled = 0;\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => setNumber(number + 1)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 10\u001b[39m\nReceived:\n\u001b[31m  Doubled: 0\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/effect.test.ts:11:49\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20",
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/effect.test.ts:20:48\n    at async file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:20"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$state let number = 5;\n\tlet doubled;\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n.greeting {\n\tcolor: blue;\n\tmargin: 10px;\n\tdisplay: flex;\n\tjustify-content: center;\n\talign-items: center;\n\theight: 100vh;\n}\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n.greeting {\n  color: blue;\n  text-align: center;\n  margin: 10px auto;\n  max-width: 80%;\n}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div class=\"container\">\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</div>\n\n<style>\n  .container {\n    display: flex;\n    justify-content: center;\n    align-items: center;\n    height: 100vh;\n  }\n  .greeting {\n    color: blue;\n    margin: 0.5rem 0;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n.greeting {\n\tcolor: blue;\n\tdisplay: flex;\n\tjustify-content: center;\n\talign-items: center;\n\tmargin: 1rem;\n}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    display: flex;\n    align-items: center;\n    justify-content: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<style>\n.container {\n    display: flex;\n    justify-content: center;\n    align-items: center;\n    min-height: 100vh;\n}\n.greeting {\n    color: blue;\n    margin: 10px;\n}\n</style>\n\n<div class=\"container\">\n    <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n.greeting {\n  display: flex;\n  justify-content: center;\n  align-items: center;\n  height: 100vh;\n  padding: 10px;\n  color: blue;\n}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<style>\n\t:local(.greeting) {\n\t\tcolor: blue;\n\t\tmargin: 0.5rem;\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tjustify-content: center;\n\t}\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script></script>\n<style>\n  .greeting-wrapper {\n    display: flex;\n    justify-content: center;\n    align-items: center;\n    min-height: 100vh;\n  }\n  .greeting {\n    color: blue;\n    margin: 10px;\n  }\n</style>\n<div class=\"greeting-wrapper\">\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tdisplay: flex;\n\t\tjustify-content: center;\n\t\talign-items: center;\n\t\tmargin: 1rem;\n\t\tcolor: blue;\n\t}\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 8,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"svelte-5\">\n\t$let text = 'Hello world';\n\n\t$: textEffect = () => {\n\t\t$inspect.trace(() => {\n\t\t\tconsole.log('Effect tracing for text change');\n\t\t});\n\t};\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t$inspect(text)\n\t$inspect(text).with(() => {\n\t\tconsole.log('Custom update callback for text');\n\t})\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 'Hello world';\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={ $state } />\n  <p data-testid=\"text-value\">Current text: \"{ $state }\"</p>\n  <p data-testid=\"char-count\">Character count: { $state.length }</p>\n  <p $inspect($state)>Logging value via $inspect</p>\n  <p $inspect($state).with((old, new) => {\n    console.log('Custom callback: changed from', old, 'to', new);\n  })>With callback</p>\n  $effect(() => {\n    $inspect.trace($state);\n  }, [$state]);\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:39 Expected attribute value\nhttps://svelte.dev/e/expected_attribute_value"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = 'Hello world';\n\n\teffect(() => {\n\t\t$inspect.trace(text, (old, new) => {\n\t\t\tconsole.log(`Trace: changed from \"${old}\" to \"${new}\"`);\n\t\t});\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t$inspect(text)\n\t$inspect(text).with(() => console.log('Updated via with:', text))\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:32 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state(text = 'Hello world');\n</script>\n\n<div>\n  <div $inspect(text)>Basic inspect</div>\n  <div $effect={() => { $inspect.trace(text); }}>Trace effect</div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} $inspect(text).with(() => console.log('custom callback:', text)) />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:88 Expected attribute value\nhttps://svelte.dev/e/expected_attribute_value"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n    const text = $state('Hello world');\n    $effect(() => {\n        $inspect.trace();\n    }, [text]);\n</script>\n\n<div>\n    <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n    $inspect(text)\n    $inspect(text).with(() => console.log('Custom callback for text update', text))\n    <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n    <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:4 `$effect` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$let text = 'Hello world';\n\t$effect(() => {\n\t\t$inspect.trace();\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n\t$inspect(text)\n\t$inspect(text).with(() => {\n\t\tconsole.log('Custom callback: Text updated to', text);\n\t})\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  $state let text = \"Hello world\";\n\n  $inspect(text);\n  $inspect(text).with(() => {\n    console.log(\"input changed\", text);\n  });\n\n  $effect(() => {\n    $inspect.trace(() => {\n      console.log(\"effect triggered\", text);\n    });\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { state, effect } from 'svelte';\n  let text = $state('Hello world');\n\n  $effect(() => {\n    $inspect.trace(text);\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} $inspect(text) $inspect(text).with(value => `Custom update: ${value}`) />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:106 Expected attribute value\nhttps://svelte.dev/e/expected_attribute_value"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 7,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$props { name = \"World\", count = 0, showDetails = false }\n\n\t$let currentCount = count\n\n\tfunction increment() {\n\t\t$set({ currentCount: currentCount + 1 })\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {currentCount}</p>\n\t<button data-testid=\"increment-button\" type=\"button\">Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {currentCount}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const $count = $state($props.count ?? 0);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name ?? \"World\"}!</h1>\n  <p data-testid=\"count-display\">Count: {$count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $count.set($count + 1)}>Increment</button>\n\n  {#if $props.showDetails ?? false}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name ?? \"World\"}</p>\n      <p>Count is {$count}</p>\n      <p>ShowDetails is {$props.showDetails ?? false}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:8 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = \"World\";\n  export let count = 0;\n  export let showDetails = false;\n\n  $state = count;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$state}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props { name, count, showDetails } = {\n    name: 'World',\n    count: 0,\n    showDetails: false\n  };\n\n  $state count = count;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $set({ count: $count + 1 })}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails ? 'true' : 'false'}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\texport let name = 'World';\n\texport let count = 0;\n\texport let showDetails = false;\n\n\t$state({ count: $props.count });\n\n\tfunction increment() {\n\t\t$state.update((state) => {\n\t\t\tstate.count += 1;\n\t\t});\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {$state.count}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n\t{#if $props.showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {$state.count}</p>\n\t\t\t<p>ShowDetails is {showDetails ? 'true' : 'false'}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:1 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\texport let name = \"World\";\n\texport let count = 0;\n\texport let showDetails = false;\n\n\tlet stateCount = $props.count;\n\n\tfunction increment() {\n\t\tstateCount += 1;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n\t<p data-testid=\"count-display\">Count: {stateCount}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\t{#if $props.showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {$props.name}</p>\n\t\t\t<p>Count is {stateCount}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:1 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props;\n  const props = $props();\n  const name = props.name ?? \"World\";\n  const initialCount = props.count ?? 0;\n  const showDetails = props.showDetails ?? false;\n\n  $state count = initialCount;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={() => count = count + 1}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "twil-8b:q8_0-verbose",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(title: string)}\n  <span data-testid=\"book-title\">{{ title }}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:21 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(bookTitle)}\n  <span data-testid=\"book-title\">{{bookTitle}}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Error: \u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  The Lord of the Rings\u001b[39m\nReceived:\n\u001b[31m  [object Object]\u001b[39m\n    at Proxy.__VITEST_EXTEND_ASSERTION__ (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:2601:12)\n    at Proxy.<anonymous> (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/index.aTZaPVhf.js:1958:14)\n    at Proxy.methodWrapper (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/chai@6.3.0/node_modules/chai/index.js:1737:25)\n    at /Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/snippets.test.ts:21:27\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:1632:35\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:2801:26\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3338:20\n    at new Promise (<anonymous>)\n    at runWithCancel (file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3333:10)\n    at file:///Users/k/Documents/GitHub/svelte-bench/node_modules/.pnpm/vitest@5.0.5_@opentelemetry+api@1.10.0_@types+node@26.6.6_jsdom@30.1.3_vite@8.3.5_@type_fd86db7e55a8babc25349251307fff39/node_modules/vitest/dist/chunks/run.BPv-BBRv.js:3318:20"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{@snippet title($title)}\n  <span data-testid=\"book-title\">{$title}</span>\n{@/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title($book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:2 Expected 'html', 'render', 'attach', 'const', or 'debug'\nhttps://svelte.dev/e/expected_tag"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  export const title = ({ title }) => (\n    <span data-testid=\"book-title\">{title}</span>\n  );\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(title=book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:4 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n{#define snippet title {#param title: string}\n  <span data-testid=\"book-title\">{title}</span>\n{/define snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title title={book}}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:11:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<$$snippet title(title)>\n  <span data-testid=\"book-title\">{$param.title}</span>\n</$$snippet>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title with title={book}}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:1 Expected a valid element or component name. Components must have a valid variable name or dot notation expression\nhttps://svelte.dev/e/tag_invalid_name"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const snippets = {\n    'title': `\n      <span data-testid=\"book-title\">\n        $title\n      </span>\n    `\n  };\n\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title($title = book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:22:21 `$title` is an illegal variable name. To reference a global variable called `$title`, use `globalThis.$title`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n{#snippet title}\n  <span data-testid=\"book-title\">\n    {title}\n  </span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title with title = book}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:11:15 Expected token (\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n@snippet title { title }\n  <span data-testid=\"book-title\">{{ title }}</span>\n\n<ul>\n  {#each books as bookTitle}\n    <li data-testid=\"book-item\">\n      {@render title with bookTitle}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:15 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n@snippet title(title) {\n  <span data-testid=\"book-title\">{title}</span>\n}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title {title: book}}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:8:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-11-06T04:53:49.358Z",
    "tps": 7.5,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 4629,
      "evalDurationNs": 617298789000,
      "promptEvalCount": 317,
      "promptEvalDurationNs": 1406743000,
      "promptTps": 225.34,
      "loadDurationNs": 1242841,
      "measuredAt": "2026-11-06T06:20:01.420Z"
    }
  }
]