[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { count: 0 };\n\n  $effect(() => {\n    console.log($state.count);\n  });\n\n  function decrement() {\n    $state.count--;\n  }\n\n  function increment() {\n    $state.count++;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{ $state.count }</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { count: 0 };\n\n  $effect(() => {\n    console.log($state.count);\n  }, [$state.count]);\n\n  function decrement() {\n    $state.count--;\n  }\n\n  function increment() {\n    $state.count++;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{ $state.count }</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { count: 0 };\n\n  export let $effect = () => {\n    const decrementButton = document.querySelector('[data-testid=\"decrement-button\"]');\n    const incrementButton = document.querySelector('[data-testid=\"increment-button\"]');\n    const countValue = document.querySelector('[data-testid=\"count-value\"]');\n\n    decrementButton.addEventListener('click', () => {\n      $state.count -= 1;\n    });\n\n    incrementButton.addEventListener('click', () => {\n      $state.count += 1;\n    });\n  };\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\" class=\"count-value\">{ $state.count }</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { count: 0 };\n\n  $effect(() => {\n    console.log($state.count);\n  });\n\n  function increment() {\n    $state.count++;\n  }\n\n  function decrement() {\n    $state.count--;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{ $state.count }</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let count = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let count = 0;\n\n  $: increment() {\n    count++;\n  }\n\n  $: decrement() {\n    count--;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:17 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let count = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let count = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { count: 0 };\n\n  $effect(() => {\n    console.log($state.count);\n  }, [$state.count]);\n\n  function decrement() {\n    $state.count--;\n  }\n\n  function increment() {\n    $state.count++;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{ $state.count }</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { count: 0 };\n\n  $effect(() => {\n    if ($state.count < 0) {\n      $state.count = 0;\n    }\n  }, [$state.count]);\n\n  function increment() {\n    $state.count++;\n  }\n\n  function decrement() {\n    $state.count--;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{ $state.count }</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived {\n    get doubled() { return number * 2; }\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>\n    Increment\n  </button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button on:click={() => number++} data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $effect(() => {\n    let number = 5;\n    let doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$state.doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:11:40 `$state.number` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button on:click={() => number++} data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: number = $state(5);\n  $: doubled = number * 2;\n\n  increment() {\n    $state(number + 1);\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:14 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button on:click={() => $state.number++} data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $derived doubled = number * 2;\n\n  increment() {\n    number++;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => increment()}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state text = '';\n  $derived.by({\n    wordCount: () => text.split(' ').length,\n    charCount: () => text.length,\n    isLong: () => text.split(' ').length > 15\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: {\n    wordCount = $state.trim().split(/\\s+/).length;\n    charCount = $state.length;\n    isLong = wordCount > 15;\n  }\n\n  $effect(() => {\n    $state = '';\n  }, ['clear']);\n\n  function clear() {\n    $state = '';\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state} />\n  <button data-testid=\"clear-button\" on:click={clear}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">{isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: words = $state.text.split(' ').length\n  $: charCount = $state.text.length\n  $: isLong = words > 15\n\n  $effect(() => {\n    if ($state.clear) {\n      $state.text = ''\n    }\n  }, [$state.clear])\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.clear = true}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {words}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\" class:long={$isLong}>Status: { $isLong ? 'Long text' : 'Short text' }</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state text = '';\n  $derived.by calculateDerived = () => ({\n    wordCount: text.split(' ').length,\n    charCount: text.length,\n    isLong: text.split(' ').length > 15\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.calculateDerived().wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.calculateDerived().charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.calculateDerived().isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state text = '';\n  $derived.by({\n    wordCount: () => text.split(' ').length,\n    charCount: () => text.length,\n    isLong: () => text.split(' ').length > 15\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state text = '';\n  $derived.by countWords = text.split(' ').length;\n  $derived.by countChars = text.length;\n  $derived.by isLong = countWords > 15;\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value=$text />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$countWords}</p>\n    <p data-testid=\"char-count\">Characters: {$countChars}</p>\n    <p data-testid=\"length-indicator\">Status: {$isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = '';\n  export let $derived = {\n    words: 0,\n    chars: 0,\n    isLong: false,\n  };\n\n  export function $effect() {\n    const words = $state.trim().split(/\\s+/).length;\n    const chars = $state.trim().length;\n    $derived.words = words;\n    $derived.chars = chars;\n    $derived.isLong = words > 15;\n  }\n\n  function clearText() {\n    $state = '';\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state} />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.chars}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = '';\n  export let $derived = {\n    by: (text) => {\n      const words = text.trim().split(/\\s+/).length;\n      const chars = text.length;\n      const isLong = words > 15;\n      return {\n        wordCount: words,\n        charCount: chars,\n        lengthIndicator: isLong ? 'Long text' : 'Short text'\n      };\n    }\n  };\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state} />\n  <button data-testid=\"clear-button\" on:click={() => $state = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.lengthIndicator}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state text = '';\n\n  $derived.by(({ text }) => {\n    const words = text.split(' ').length;\n    const chars = text.length;\n    const isLong = words > 15;\n\n    return { words, chars, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.chars}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state text = '';\n  $derived.by(\n    ($state.text) => ({\n      wordCount: $state.text.split(/\\s+/).length,\n      charCount: $state.text.length,\n      isLong: $state.text.split(/\\s+/).length > 15\n    })\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  $: characters = [...characters];\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  $: $effect(() => {\n    characters.push('George');\n  }, [addGeorge]);\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  let characterToAdd = \"George\";\n\n  $: addCharacter = () => {\n    characters.push(characterToAdd);\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addCharacter}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:7:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  $: characters = [...characters];\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  $: characters.sort();\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  $: count = characters.length;\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button on:click={() => characters.push('George')} data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $effect(() => {\n    characters.push(\"George\");\n  }, { events: [\"click\", \"add-george-button\"] });\n\n  let characters = [\"Jerry\", \"Elaine\", \"Kramer\"];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$effect` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  $: console.log(characters);\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $effect doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { number: 5 };\n  export let $effect = (state) => ({ doubled: state.number * 2 });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$effect.state.doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n  $effect doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button on:click={() => number++} data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { number: 5 };\n  export let $effect = () => ({ doubled: $state.number * 2 });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$effect().doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.number++}>\n    Increment\n  </button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "doubled is not defined\n\n\tin <unknown>\n",
          "doubled is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state number = 5}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>\n\n<style>\n  button {\n    margin-left: 10px;\n  }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:48 Expected token }\nhttps://svelte.dev/e/expected_token\n\n- Did you forget to add a lang attribute to your style tag?\n- Did you forget to add a style preprocessor? See https://github.com/sveltejs/vite-plugin-svelte/blob/main/docs/preprocess.md for more information."
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { number: 5 };\n  export let $effect = $state => ({ doubled: $state.number * 2 });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$effect.number.doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5;\n\n  $effect watch(() => {\n    doubled = number * 2;\n  });\n\n  let number;\n  let doubled;\n\n  function increment() {\n    number++;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>\n\n<script>\n  export let number = 5;\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state number = 5\n  $effect doubled = number * 2\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let text = \"Hello, World!\";\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  {text}\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { color: 'blue' };\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <style>\n    .greeting {\n      text-align: center;\n      margin: 10px;\n      color: {$state.color};\n    }\n  </style>\n  Hello, World!\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let message = 'Hello, World!';\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting text-center\" style=\"margin: 10px; color: blue;\">\n  {$message}\n</div>\n<style>\n  .greeting {\n    font-size: 24px;\n    font-weight: bold;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let greeting = \"Hello, World!\";\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\" style=\"text-align: center; margin: 10px;\">\n  {greeting}\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting text-center\" style=\"margin: 10px; color: blue;\">Hello, World!</div>\n\n<style>\n  .greeting {\n    text-align: center;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let title = \"Hello, World!\";\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting text-center mx-4\">\n  {title}\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let state = \"\";\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <p style=\"color: blue; text-align: center; margin: 20px;\">Hello, World!</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let state = { message: \"Hello, World!\" };\n  $: style = { color: \"blue\", textAlign: \"center\", margin: \"10px\" };\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\" style={$style}>{state.message}</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let $state = { greeting: 'Hello, World!' };\n\n  $effect(() => {\n    document.title = $state.greeting;\n  });\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting text-center mb-2\">\n  {$state.greeting}\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n  }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let text = 'Hello, World!';\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <p style=\"color: blue; text-align: center; margin: 10px;\">{$text}</p>\n</div>\n\n<style>\n  .greeting {\n    text-align: center;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $effect(() => {\n    $inspect.trace();\n  }, [$state.text]);\n\n  $inspect\n    .with((text) => {\n      console.log('Input updated:', text);\n    })\n    .with((text) => {\n      $state.charCount = text.length;\n    });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <p data-testid=\"text-value\">Current text: \"{$state.text}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state.charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$effect` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state text = 'Hello world'\n\n  $inspect\n    .with((value) => console.log(`Input value changed to: ${value}`))\n    .trace()\n\n  $effect(() => {\n    $inspect\n      .with((value) => console.log(`Input value changed to: ${value}`))\n      .trace()\n  }, [text])\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = 'Hello world';\n  $inspect(text);\n  $inspect(text).with((value) => {\n    console.log('Updated text:', value);\n  });\n  $effect(() => {\n    $inspect(text).trace();\n  }, [text]);\n</script>\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:9:2 `$effect` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = 'Hello world';\n\n  $inspect\n    .with((input) => {\n      console.log('Input changed:', input);\n    })\n    .with((input) => {\n      console.log('Input changed:', input);\n    });\n\n  $effect(() => {\n    $inspect.trace().with((input) => {\n      console.log('Input changed:', input);\n    });\n  }, [text]);\n\n  $inspect\n    .with((input) => {\n      console.log('Input changed:', input);\n    });\n\n  $: charCount = text.length;\n\n  $effect(() => {\n    console.log('Text changed:', text);\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{$text}\"</p>\n  <p data-testid=\"char-count\">Character count: {$charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:2 `$inspect.with` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $effect(() => {\n    $inspect.trace().with((e) => console.error(e));\n  });\n\n  let text = \"Hello world\";\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{$inspect(text)}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:4 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $effect(() => {\n    $inspect.trace().log('$inspect is running');\n  }, { flush: 'post' });\n\n  $inspect\n    .with((text) => {\n      console.log('Text updated:', text);\n    })\n    .with(({ trace }) => {\n      trace.log('Text value:', $state.text);\n    });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <p data-testid=\"text-value\">Current text: \"{$state.text}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state.text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$effect` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $inspect(\"Initial text\");\n  let text = 'Hello world';\n  $effect(() => {\n    $inspect.trace();\n  }, [text]);\n  $inspect.with({\n    on: ['text'],\n    callback: (value) => console.log(`Input changed to: ${value}`),\n  });\n</script>\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:2 `$effect` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = 'Hello world';\n  $effect(() => {\n    $inspect.trace();\n  });\n  $effect(() => {\n    $inspect('input').with((value) => {\n      console.log('Input updated:', value);\n    });\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "expected false to be true // Object.is equality",
          "expected false to be true // Object.is equality",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Character count: 0\u001b[39m\nReceived:\n\u001b[31m  Character count: 11\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $effect(() => {\n    $inspect(trace).with((text) => {\n      console.log('Input changed:', text);\n    });\n  }, [text]);\n\n  let text = 'Hello world';\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$effect` must be called with exactly one argument\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = 'Hello world';\n\n  $inspect\n    .with((value) => {\n      console.log('Input updated:', value);\n    })\n    .with((value) => {\n      console.log('Input updated:', value);\n    });\n\n  $inspect.text('Initial text: {text}').with((value) => {\n    console.log('Text updated:', value);\n  });\n\n  $inspect.trace().with((args) => {\n    console.log('Tracing input changes:', args);\n  });\n\n  $effect(() => {\n    $inspect.trace().with((args) => {\n      console.log('Tracing input changes:', args);\n    });\n  }, [text]);\n\n  $: charCount = text.length;\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:2 `$inspect.with` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = 'World';\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count = count;\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$state.count}</p>\n<button data-testid=\"increment-button\" on:click={() => count++}>Increment</button>\n\n{#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name}</p>\n    <p>Count is {$state.count}</p>\n    <p>ShowDetails is {$props.showDetails}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = 'World';\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count += 1;\n\n  $: let details = {\n    name,\n    count,\n    showDetails\n  };\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {$props.count}</p>\n  <button data-testid=\"increment-button\" on:click={$state.count += 1}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$props.count}</p>\n      <p>ShowDetails is {$props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:5 The keyword 'let' is reserved\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = 'World';\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count = count;\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$state.count}</p>\n<button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n\n{#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name}</p>\n    <p>Count is {$state.count}</p>\n    <p>ShowDetails is {$props.showDetails}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = \"World\";\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count = count;\n  $: $state.showDetails = showDetails;\n\n  function incrementCount() {\n    count++;\n  }\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$state.count}</p>\n<button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n{#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name}</p>\n    <p>Count is {$state.count}</p>\n    <p>ShowDetails is {$props.showDetails}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = \"World\";\n  export let count = 0;\n  export let showDetails = false;\n  $: $state.count = count;\n\n  $effect(() => {\n    if (showDetails) {\n      const details = document.querySelector('[data-testid=\"details\"]');\n      if (details) {\n        details.innerHTML = `\n          <p>Name is ${name}</p>\n          <p>Count is ${count}</p>\n          <p>ShowDetails is ${showDetails}</p>\n        `;\n      }\n    } else {\n      const details = document.querySelector('[data-testid=\"details\"]');\n      if (details) {\n        details.innerHTML = '';\n      }\n    }\n  }, [showDetails, name, count]);\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$props.count}</p>\n<button data-testid=\"increment-button\" on:click={() => {$props.count++}}>Increment</button>\n\n<div data-testid=\"details\" style:display={$props.showDetails ? 'block' : 'none'}>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = \"World\";\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count = count;\n  $: $props.showDetails = showDetails;\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$state.count}</p>\n<button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n\n{#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name}</p>\n    <p>Count is {$state.count}</p>\n    <p>ShowDetails is {$props.showDetails}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = 'World';\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count = count;\n\n  function increment() {\n    $state.count++;\n  }\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$state.count}</p>\n<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n{#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name}</p>\n    <p>Count is {$state.count}</p>\n    <p>ShowDetails is {$props.showDetails}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = 'World';\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count = count;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$name}!</h1>\n  <p data-testid=\"count-display\">Count: {$count}</p>\n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$name}</p>\n      <p>Count is {$count}</p>\n      <p>ShowDetails is {$showDetails}</p>\n    </div>\n  {/if}\n</div>\n\n<script>\n  function incrementCount() {\n    $state.count++;\n  }\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:25:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = \"World\";\n  export let count = 0;\n  export let showDetails = false;\n\n  $: state.count = count;\n\n  $effect(() => {\n    state.count = count;\n  }, [count]);\n\n  function increment() {\n    count++;\n  }\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$props.count}</p>\n<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n{#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name}</p>\n    <p>Count is {$props.count}</p>\n    <p>ShowDetails is {$props.showDetails}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let name = 'World';\n  export let count = 0;\n  export let showDetails = false;\n\n  $: $state.count += 1;\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n<p data-testid=\"count-display\">Count: {$props.count}</p>\n<button data-testid=\"increment-button\" on:click={$state.count += 1}>Increment</button>\n\n{#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name}</p>\n    <p>Count is {$props.count}</p>\n    <p>ShowDetails is {$props.showDetails}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "llama3.1:8b-instruct-q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n  {@# $each(books, book)}\n    <li data-testid=\"book-item\">\n      {@render title book}\n    </li>\n  {/each}\n</ul>\n\n<script>\n  export function title(book) {\n    return `<span data-testid=\"book-title\">${book}</span>`;\n  }\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:8:4 Expected 'html', 'render', 'attach', 'const', or 'debug'\nhttps://svelte.dev/e/expected_tag"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  $: snippet = book => ({ title: book });\n\n  $: renderedTitles = books.map(snippet);\n\n  $: state = { renderedTitles };\n</script>\n\n{@render $renderedTitles as title}\n  <li data-testid=\"book-item\">{title}</li>\n{/render}\n\n<!-- $renderedTitles is an array of rendered snippet elements, but it's not possible to directly render an array in Svelte.\n     The above code will not work as expected. Instead, you can use the following code to achieve the desired result -->\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{snippets.title(book)}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:9 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let titles = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  $: snippet = ({ title }) => {\n    return span({ data: { testid: \"book-title\" } }, title);\n  };\n</script>\n\n<ul>\n  {@render snippet(titles[0])}\n  {@render snippet(titles[1])}\n  {@render snippet(titles[2])}\n</ul>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n  {@$render each(book, books) (\n    <li data-testid=\"book-item\">{@title(book)}</li>\n  )}\n</ul>\n\n{@$snippet title(book) {\n  <span data-testid=\"book-title\">{book}</span>\n}}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:8:4 Expected 'html', 'render', 'attach', 'const', or 'debug'\nhttps://svelte.dev/e/expected_tag"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n  const title = ({title}) => (\n    <span data-testid=\"book-title\">{title}</span>\n  );\n</script>\n\n<ul>\n  {@render books, (book) => (\n    <li data-testid=\"book-item\">{title(book)}</li>\n  )}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:4 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  function title({ title }) {\n    return <span data-testid=\"book-title\">{title}</span>;\n  }\n</script>\n\n<ul>\n  {@$books.forEach(book => (\n    <li data-testid=\"book-item\" key={book}>{{@render title, { title: book }}}</li>\n  ))}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:11 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  export let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{@render title, books}\n  <ul>\n    {#each books as book}\n      <li data-testid=\"book-item\">{@render title, {title: book}}</li>\n    {/each}\n  </ul>\n{/render}\n\n<script>\n  export function title({ title }) {\n    return `<span data-testid=\"book-title\">${title}</span>`;\n  }\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:9 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: titles = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  const title = ($title) => ({\n    span: $title,\n  });\n</script>\n\n<ul>\n  {@render({$: titles, $each: title})}\n    <li data-testid=\"book-item\">{span}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:10 Expected whitespace\nhttps://svelte.dev/e/expected_whitespace"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  const title = (title) => ({ props: { title } });\n\n  const fragment = [\n    { type: 'li', props: { 'data-testid': 'book-item' }, children: [\n      { type: 'span', props: { 'data-testid': 'book-title' }, children: '{@title}' },\n    ]},\n  ];\n</script>\n\n<ul>\n  {@each books as book}\n    {@render title(book)}\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:16:4 Expected 'html', 'render', 'attach', 'const', or 'debug'\nhttps://svelte.dev/e/expected_tag"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  $: title = ({ title }) => span({ data: { testid: \"book-title\" } }, title);\n</script>\n\n<ul>\n  {@each books as book}\n    <li data-testid=\"book-item\">{@render title({ title: book })}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:14:4 Expected 'html', 'render', 'attach', 'const', or 'debug'\nhttps://svelte.dev/e/expected_tag"
        ]
      }
    ],
    "timestamp": "2026-09-14T00:27:09.618Z",
    "tps": 15.07,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 123,
      "evalDurationNs": 8159533000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 1917888000,
      "promptTps": 146.52,
      "loadDurationNs": 2048222,
      "measuredAt": "2026-09-16T18:26:22.269Z"
    }
  }
]