[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function increment() {\n    $state += 1;\n  }\n\n  function decrement() {\n    $state -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function decrement() {\n    $state -= 1;\n  }\n\n  function increment() {\n    $state += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button on:click={decrement} data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button on:click={increment} data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function increment() {\n    $state += 1;\n  }\n\n  function decrement() {\n    $state -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function increment() {\n    $state += 1;\n  }\n\n  function decrement() {\n    $state -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function increment() {\n    $state += 1;\n  }\n\n  function decrement() {\n    $state -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function decrement() {\n    $state -= 1;\n  }\n\n  function increment() {\n    $state += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button on:click={decrement} data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button on:click={increment} data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function increment() {\n    $state += 1;\n  }\n\n  function decrement() {\n    $state -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function decrement() {\n    $state -= 1;\n  }\n\n  function increment() {\n    $state += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function increment() {\n    $state += 1;\n  }\n\n  function decrement() {\n    $state -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = 0;\n\n  function decrement() {\n    $state -= 1;\n  }\n\n  function increment() {\n    $state += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{$state}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: number = 5;\n  $derived doubled: number = $state * 2;\n\n  function increment() {\n    $state += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:5:11 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: number = 5;\n  $derived: number = $state * 2;\n\n  function increment() {\n    $state += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:21 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5, setNumber\n  $derived: $state, doubled = $state * 2\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={() => setNumber($state + 1)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:12 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5, number = $state;\n  $derived: $state * 2, doubled = $derived;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:22 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: number = 5;\n  $derived(number, { doubled: number }) = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:5:2 Assigning to rvalue\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '@svelte/system';\n\n  $state(number, 5);\n\n  $derived(doubled, $number, {\n    get: () => $number() * 2\n  });\n\n  function increment() {\n    $number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $effect } from '@svelte/runtime';\n\n  $state(number, 5);\n\n  $derived(doubled, $number, {\n    get: () => $number * 2\n  });\n\n  $effect(() => {\n    document.getElementById('increment-button').addEventListener('click', () => {\n      $number += 1;\n    });\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5 as number\n  $derived: $state * 2\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:12 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5, setNumber = (number) => number + 1;\n  $derived: $state * 2, double = ($state * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state = setNumber($state)}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:12 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5 as number, $derived = $state * 2;\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>\n    Increment\n  </button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:12 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '@svelte/runtime';\n\n  $state(text, '');\n  $derived.by(\n    text,\n    (text) => {\n      const words = text.split(/\\s+/).filter((word) => word !== '').length;\n      const chars = text.length;\n      const isLong = words > 15;\n\n      return { words, chars, isLong };\n    },\n    (derived) => {\n      const { words, chars, isLong } = derived;\n\n      return (\n        <div>\n          <input\n            data-testid=\"text-input\"\n            type=\"text\"\n            bind:value={$state(text)}\n          />\n          <button\n            data-testid=\"clear-button\"\n            on:click={() => $state(text, '')}\n          >\n            Clear\n          </button>\n          <div>\n            <p data-testid=\"word-count\">Words: {words}</p>\n            <p data-testid=\"char-count\">Characters: {chars}</p>\n            <p data-testid=\"length-indicator\">\n              Status: {isLong ? 'Long text' : 'Short text'}\n            </p>\n          </div>\n        </div>\n      );\n    }\n  );\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:20:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$lib/state';\n  import { $effect } from '$lib/effect';\n\n  $state(text, 'text', '');\n  $derived.by(text, derived => {\n    const words = derived.split(/\\s+/).filter(Boolean).length;\n    const chars = derived.length;\n    const isLong = words > 15;\n    return { words, chars, isLong };\n  }, $derivedDeps);\n\n  $effect(\n    () => {\n      const clearButton = document.querySelector('[data-testid=\"clear-button\"]');\n      clearButton.addEventListener('click', () => {\n        $state(text).set('');\n      });\n    },\n    $effectDeps\n  );\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={$state(text)}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.by(text).words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.by(text).chars}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {$derived.by(text).isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: '';\n  $derived.by(\n    state => ({\n      words: state.split(' ').filter(word => word !== '').length,\n      characters: state.length,\n      isLong: state.split(' ').filter(word => word !== '').length > 15,\n    }),\n    { store: $state }\n  );\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={$state}\n  />\n  <button\n    data-testid=\"clear-button\"\n    on:click={() => $state = ''}\n  >\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.characters}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {$derived.isLong ? 'Long' : 'Short'} text\n    </p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$derived.by(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $effect } from '@runecase/svelte';\n  \n  $state(text, '');\n  \n  $derived.by(\n    text,\n    {\n      words: (prev, text) => text.split(' ').filter(word => word !== '').length,\n      chars: (prev, text) => text.length,\n      long: (prev, text) => text.split(' ').filter(word => word !== '').length > 15\n    },\n    $state(wordCount, 0),\n    $state(charCount, 0),\n    $state(isLong, false)\n  );\n  \n  $effect(\n    () => {\n      document.querySelector('[data-testid=\"word-count\"]').textContent = `Words: ${wordCount}`;\n      document.querySelector('[data-testid=\"char-count\"]').textContent = `Characters: ${charCount}`;\n      document.querySelector('[data-testid=\"length-indicator\"]').textContent = `Status: ${isLong ? 'Long' : 'Short'} text`;\n    }\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:input={$state(text, '')} />\n  <button data-testid=\"clear-button\" on:click={() => $state(text, '')}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$state(wordCount, 0)}</p>\n    <p data-testid=\"char-count\">Characters: {$state(charCount, 0)}</p>\n    <p data-testid=\"length-indicator\">Status: {$state(isLong, false) ? 'Long' : 'Short'} text</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '@sveltejs/kit';\n\n  $state.text = '';\n\n  $derived.by(\n    $state.text,\n    (text) => ({\n      wordCount: text.split(' ').filter((word) => word !== '').length,\n      charCount: text.length,\n      isLong: text.split(' ').filter((word) => word !== '').length > 15,\n    }),\n    {\n      store: 'textStats',\n    }\n  );\n\n  function clearText() {\n    $state.text = '';\n  }\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={$state.text}\n  />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.by.textStats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.by.textStats.charCount}</p>\n    <p\n      data-testid=\"length-indicator\"\n    >Status: {$derived.by.textStats.isLong ? 'Long text' : 'Short text'}</p\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:38:2 Expected token >\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $effect } from '@sveltejs/kit';\n\n  $state(text, '');\n  $derived.by(\n    $state(text),\n    'wordCount',\n    (text) => text.split(/\\s+/).filter((word) => word !== '').length,\n    [text]\n  );\n  $derived.by(\n    $state(text),\n    'charCount',\n    (text) => text.length,\n    [text]\n  );\n  $derived.by(\n    $state(text),\n    'isLongText',\n    (text) => text.split(/\\s+/).filter((word) => word !== '').length > 15,\n    [text]\n  );\n\n  $effect(\n    () => {\n      document.querySelector('[data-testid=\"text-input\"]').addEventListener('input', (e) => {\n        text = e.target.value;\n      });\n    },\n    [text]\n  );\n  $effect(\n    () => {\n      document.querySelector('[data-testid=\"clear-button\"]').addEventListener('click', () => {\n        text = '';\n      });\n    },\n    []\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLongText ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: '';\n  $derived.by(\n    ($state, prev) => {\n      const words = $state.trim().split(/\\s+/).filter(Boolean).length;\n      const chars = $state.length;\n      const isLong = words > 15;\n\n      return { words, chars, isLong };\n    },\n    ($state) => $state\n  );\n\n  function clearText() {\n    $state = '';\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state} />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.chars}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLong ? 'Long' : 'Short'} text</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:2 `$derived.by(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $effect } from '@sveltejs/core';\n\n  $state(text, ' ');\n  $derived.by(\n    text,\n    (text) => {\n      const words = text.trim().split(/\\s+/).length;\n      const characters = text.length;\n      const isLong = words > 15;\n\n      return { words, characters, isLong };\n    },\n    (derived) => {\n      $effect(derived, ({ words, characters, isLong }) => {\n        wordCount = words;\n        charCount = characters;\n        isLongText = isLong;\n      });\n    }\n  );\n\n  let wordCount = 0;\n  let charCount = 0;\n  let isLongText = false;\n\n  function clearText() {\n    $state(text, ' ');\n  }\n</script>\n\n<div>\n  <input bind:value={$state(text)} data-testid=\"text-input\" type=\"text\" />\n  <button on:click={clearText} data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLongText ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: '';\n  $derived.by(\n    state => {\n      const words = state.split(/\\s+/).filter(word => word !== '').length;\n      const chars = state.length;\n      const isLong = words > 15;\n      return { words, chars, isLong };\n    },\n    ($state),\n    ($derived)\n  );\n\n  function clearText() {\n    $state = '';\n  }\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={$state}\n  />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.chars}</p>\n    <p\n      data-testid=\"length-indicator\"\n    >Status: {$derived.isLong ? 'Long' : 'Short'} text</p\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:34:2 Expected token >\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { onMount } from '$utils';\n\n  $state = '';\n\n  $derived.by(\n    ($state) => {\n      const words = $state.split(/\\s+/).filter((word) => word !== '').length;\n      const chars = $state.length;\n      const isLong = words > 15;\n\n      return { words, chars, isLong };\n    },\n    ($state),\n    ({ words, chars, isLong }) => {\n      const wordCount = `Words: ${words}`;\n      const charCount = `Characters: ${chars}`;\n      const lengthIndicator = `Status: ${isLong ? 'Long' : 'Short'} text`;\n\n      return { wordCount, charCount, lengthIndicator };\n    }\n  );\n\n  function clearText() {\n    $state = '';\n  }\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={$state}\n    on:input={(e) => ($state = e.target.value)}\n  />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.chars}</p>\n    <p data-testid=\"length-indicator\">Status: {#if($derived.isLong) Long {#else} Short{/if} text</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:42:50 Expected whitespace\nhttps://svelte.dev/e/expected_whitespace"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    const button = document.querySelector('[data-testid=\"add-george-button\"]');\n    button.addEventListener('click', () => {\n      $state = [...$state, \"George\"];\n    });\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $state as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    const addGeorge = () => {\n      $state = [...$state, \"George\"];\n    };\n\n    return { addGeorge };\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {@pagination($state, { as: 'li', testid: 'character' })}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={$effect.addGeorge}>Add George</button>\n</div>\n\n<style>\n  li {\n    list-style: none;\n  }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:16:6 Expected 'html', 'render', 'attach', 'const', or 'debug'\nhttps://svelte.dev/e/expected_tag"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    let button = document.querySelector('[data-testid=\"add-george-button\"]');\n    button.addEventListener('click', () => {\n      $state = [...$state, \"George\"];\n    });\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {@html:$state($state, (character) => {\n      <li data-testid=\"character\">{character}</li>\n    })}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:10 Expected whitespace\nhttps://svelte.dev/e/expected_whitespace"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    const button = document.querySelector('[data-testid=\"add-george-button\"]');\n    button.addEventListener('click', () => {\n      $state = [...$state, \"George\"];\n    });\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $state as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    const addGeorge = () => {\n      $state = [...$state, \"George\"];\n    };\n    return { addGeorge };\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $state as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={$effect.addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    const button = document.querySelector('[data-testid=\"add-george-button\"]');\n    button.addEventListener('click', () => {\n      $state = [...$state, \"George\"];\n    });\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {@html:$state.map((character) => {\n      return `<li data-testid=\"character\">${character}</li>`;\n    })}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:10 Expected whitespace\nhttps://svelte.dev/e/expected_whitespace"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = ($state, $event) => {\n    if ($event.type === \"click\") {\n      $state = [...$state, \"George\"];\n    }\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $state as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={$effect}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    let button = document.querySelector('[data-testid=\"add-george-button\"]');\n    button.addEventListener(\"click\", () => {\n      $state = [...$state, \"George\"];\n    });\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $state as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  $effect = () => {\n    let button = document.querySelector('[data-testid=\"add-george-button\"]');\n    button.addEventListener('click', () => {\n      $state = [...$state, \"George\"];\n    });\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {@html:$state.map((character, index) => `\n      <li data-testid=\"character\">${character}</li>\n    `)}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:10 Expected whitespace\nhttps://svelte.dev/e/expected_whitespace"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\n    { name: 'Jerry' },\n    { name: 'Elaine' },\n    { name: 'Kramer' }\n  ];\n\n  $effect = () => {\n    const button = document.querySelector('[data-testid=\"add-george-button\"]');\n\n    button.addEventListener('click', () => {\n      $state = [...$state, { name: 'George' }];\n    });\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $state as character (character.name)}\n      <li data-testid=\"character\">{character.name}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5, number = $state;\n  $effect: () => {\n    doubled = number * 2;\n  };\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:22 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5 as number\n  $effect: ({ number }) => {\n    $state = number\n    return number * 2\n  } as number\n\n  let doubled = $effect($state)\n\n  function increment() {\n    $state += 1\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:12 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5\n  $effect: () => {\n    $state = $state + 1\n  }, [$state]\n  $derived: ({ state: $state }) => $state * 2\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state = $state + 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:4 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5, number = $state;\n  $effect: () => {\n    doubled = number * 2;\n  };\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$effect, doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:22 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: number = 5\n  $effect: $state => {\n    $state = $state + 1\n  }\n  $derived: {\n    doubled: $state * 2\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived.doubled}</p>\n  <button data-testid=\"increment-button\" on:click>{$effect}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:9:13 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5\n  $effect: () => {\n    $state: $state\n    $state = $state * 2\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$state}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:12 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: number = 5;\n  $effect: () => {\n    const doubled = $state * 2;\n    return doubled;\n  };\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$effect}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:20 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5 as number, $number;\n  $effect: $number => $number * 2, $doubled;\n\n  function increment() {\n    $number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:12 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5 as number, $number = $state;\n  $effect: () => { $number *= 1; $doubled = $number * 2; };\n  let doubled = $doubled;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:12 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: 5 as number;\n  $effect: () => {\n    $state = $state + 1;\n  };\n\n  let doubled = $state * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click/{$effect}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:12 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 5,
    "pass1": 0.5,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let state = { greeting: 'Hello, World!' }\n  let effect = { greetingStyle: 'color: blue; text-align: center; margin: 10px;' }\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{state.greeting}</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = { greeting: 'Hello, World!' }\n  $effect = () => {}\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{$state.greeting}</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    greeting: 'Hello, World!'\n  };\n\n  $effect = (\n    () => {\n      const style = {\n        color: 'blue',\n        textAlign: 'center',\n        margin: '10px'\n      };\n\n      return style;\n    },\n    {}\n  );\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{$state.greeting}</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let state = { greeting: 'Hello, World!' };\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{state.greeting}</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = { greeting: 'Hello, World!' };\n  $effect = ($state, { greeting: 'Hello, World!' }) => {\n    document.body.style.textAlign = 'center';\n  };\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{$state.greeting}</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:5:33 Assigning to rvalue\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let state = { greeting: 'Hello, World!' }\n  let effect = { greetingStyle: 'color: blue; text-align: center; margin: 10px;' }\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{state.greeting}</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let greeting = 'Hello, World!';\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">{greeting}</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let state = 'Hello, World!';\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{state}</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    greeting: 'Hello, World!'\n  };\n\n  $effect\n  (($state) => {\n    document.title = $state.greeting;\n  }, $state);\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{$state.greeting}</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let style = `\n    .greeting {\n      color: blue;\n      text-align: center;\n      margin: 10px;\n    }\n  `;\n</script>\n\n<style>{style}</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:7 Expected a valid CSS identifier\nhttps://svelte.dev/e/css_expected_identifier\n\n- Did you forget to add a lang attribute to your style tag?\n- Did you forget to add a style preprocessor? See https://github.com/sveltejs/vite-plugin-svelte/blob/main/docs/preprocess.md for more information."
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '@sveltejs/core';\n\n  $state(text, 'Hello world');\n\n  $inspect(text, (value) => {\n    console.log('Text changed:', value);\n  }).with((value) => {\n    console.log('Text updated:', value);\n  });\n\n  $effect(\n    () => {\n      $inspect.trace(text);\n    },\n    text\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '@svelte/runtime';\n\n  $state('text', 'Hello world');\n\n  $inspect($state('text'), (value) => {\n    console.log('Text changed:', value);\n  }).with((value) => {\n    console.log('Tracked text:', value);\n  });\n\n  $effect(\n    () => {\n      $inspect($state('text')).trace();\n    },\n    [$state('text')]\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state('text')} />\n  <p data-testid=\"text-value\">Current text: \"{$state('text')}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state('text').length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: text = $state('Hello world')\n  $: $inspect(text)\n  $: $inspect(text).with((newValue) => {\n    console.log('Text updated:', newValue)\n  })\n  $: $effect(\n    () => {\n      $: $inspect.trace()\n    },\n    [text]\n  )\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: text = 'Hello world', \n     $inspect(text).with((value) => {\n       console.log('Text updated:', value);\n     }),\n     $effect(() => {\n       $inspect(text).trace();\n     }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <p data-testid=\"text-value\">Current text: \"{$state.text}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state.text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$utils';\n  import { on } from '$utils';\n\n  $state(text = 'Hello world', textProps = {});\n\n  $inspect(text, (value) => {\n    console.log('Text changed to:', value);\n  }).with((prev, next) => {\n    if (prev !== next) {\n      console.log('Text updated from:', prev, 'to:', next);\n    }\n  });\n\n  $effect(\n    () => {\n      $inspect.trace();\n    },\n    [text]\n  );\n\n  textProps = {\n    on: {\n      input: () => {\n        $inspect.trace();\n      },\n    },\n  };\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    {...textProps}\n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$lib';\n\n  $state('text', 'Hello world');\n\n  $inspect($state('text'));\n\n  $inspect($state('text')).with((text) => {\n    console.log('Text updated:', text);\n  });\n\n  $effect(\n    () => {\n      $inspect($state('text')).trace();\n    },\n    $state('text')\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state('text')} />\n  <p data-testid=\"text-value\">Current text: \"{$state('text')}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state('text').length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$runes';\n\n  $state('text', 'Hello world');\n\n  $inspect($state('text')).with((val) => {\n    console.log('Text updated:', val);\n  });\n\n  $effect(\n    $state('text'),\n    () => {\n      $inspect($state('text')).trace();\n    }\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state('text')} />\n  <p data-testid=\"text-value\">Current text: \"{$state('text')}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state('text').length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '@sveltejs/core';\n  \n  $state('text', 'Hello world', {\n    subscribe: (value) => {\n      $inspect(value).with((newValue) => {\n        console.log('Text value updated:', newValue);\n      });\n    }\n  });\n\n  $effect(\n    () => {\n      $inspect($state('text')).trace();\n    },\n    [$state('text')]\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state('text')} />\n  <p data-testid=\"text-value\">Current text: \"{$state('text')}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state('text').length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $inspect, $state, $effect } from '@sveltejs/core';\n\n  $state('text', 'Hello world');\n\n  $inspect($state('text')).with((value) => {\n    console.log('Input changed:', value);\n  });\n\n  $effect($state('text'), (text) => {\n    $inspect($state('text')).trace();\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state('text')} />\n  <p data-testid=\"text-value\">Current text: \"{$state('text')}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state('text').length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '@sveltejs/core';\n\n  $state(text, 'Hello world');\n\n  $inspect(text, (value) => {\n    console.log('Text updated:', value);\n  }).with((previous, current) => {\n    if (previous !== current) {\n      console.log('Text changed from:', previous, 'to:', current);\n    }\n  });\n\n  $effect(\n    () => {\n      $inspect.trace(text);\n    },\n    [text]\n  );\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state } from '$state';\n  import { $props } from '$props';\n\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  $state(count, $props.count);\n\n  function incrementCount() {\n    $state(count, count + 1);\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {($props.showDetails ? 'true' : 'false')}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state count = $props.count || 0;\n  $effect increment = () => {\n    count += 1;\n  };\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name || 'World'}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n  {#if $props.showDetails || false}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name || 'World'}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {($props.showDetails || false).toString()}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $props, $state, $effect } from 'svelte';\n\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  $state(count, 0);\n\n  $effect(() => {\n    const increment = () => {\n      count = count + 1;\n    };\n\n    return { increment };\n  });\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$name}!</h1>\n  <p data-testid=\"count-display\">Count: {$count}</p>\n  <button data-testid=\"increment-button\" on:click={$effect().increment}>Increment</button>\n\n  {#if $showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$name}</p>\n      <p>Count is {$count}</p>\n      <p>ShowDetails is { $showDetails ? 'true' : 'false'}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state: { count: 0 };\n  $props: { name: \"World\", count: 0, showDetails: false };\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {$props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:5:32 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $props, $state, $effect } from '$core';\n  $state(count, 0);\n\n  $effect(() => {\n    if ($props.showDetails) {\n      console.log('Showing details');\n    }\n  }, $props.showDetails);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {($props.showDetails ? 'true' : 'false')}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = { count: 0 }\n  $props = {\n    name: { default: 'World' },\n    count: { default: 0 },\n    showDetails: { default: false }\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n  \n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {$props.showDetails ? 'true' : 'false'}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props(name = 'World', count = 0, showDetails = false)\n  $state(count)\n\n  function increment() {\n    $state(count = count + 1)\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$name}!</h1>\n  <p data-testid=\"count-display\">Count: {$count}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n  {#if $showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$name}</p>\n      <p>Count is {$count}</p>\n      <p>ShowDetails is {($showDetails ? 'true' : 'false')}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:41 `$name` is an illegal variable name. To reference a global variable called `$name`, use `globalThis.$name`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state } from '$utils';\n\n  $state(count, 0);\n  $state(name, 'World');\n  $state(showDetails, false);\n\n  function increment() {\n    $state(count, count + 1);\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$name}!</h1>\n  <p data-testid=\"count-display\">Count: {$count}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$name}</p>\n      <p>Count is {$count}</p>\n      <p>ShowDetails is {$showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    count: $props.count || 0\n  }\n  \n  $effect\n  $: $state.count, $props.showDetails\n  \n  function increment() {\n    $state.count += 1\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name || \"World\"}</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n  \n  {#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$props.name || \"World\"}</p>\n    <p>Count is {$state.count}</p>\n    <p>ShowDetails is {($props.showDetails ? \"true\" : \"false\")}</p>\n  </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state } from '@runes/state';\n  import { $props } from '@runes/props';\n\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false,\n  });\n\n  $state(count, 0);\n\n  function incrementCount() {\n    $state(count, count + 1);\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {($props.showDetails ? 'true' : 'false')}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "llama3.3:70b-instruct-q2_K",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    books: [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]\n  }\n  \n  $effect = {\n    document: {\n      body: {\n        innerHTML: ''\n      }\n    }\n  }\n\n  $: render = {\n    title: (bookTitle) => {\n      return `<span data-testid=\"book-title\">${bookTitle}</span>`\n    }\n  }\n\n  $: render = {\n    books: $state.books.map((book) => {\n      return `<li data-testid=\"book-item\">${$render.title(book)}</li>`\n    })\n  }\n</script>\n\n<ul>\n  {@render books}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:30:11 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    bookTitles: [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]\n  }\n  \n  $effect = {\n    document: {\n      mount: {\n        bookTitles: $state.bookTitles\n      }\n    }\n  }\n\n  $: render = {\n    title: (bookTitle) => {\n      return `<span data-testid=\"book-title\">${bookTitle}</span>`\n    }\n  }\n\n  $: render = {\n    bookList: () => {\n      return `\n        <ul>\n          {@render bookList}\n        </ul>\n      `\n    }\n  }\n\n  $: render = {\n    bookList: () => {\n      return `\n        <ul>\n          {#each $state.bookTitles as bookTitle}\n            <li data-testid=\"book-item\">\n              {@render title bookTitle}\n            </li>\n          {/each}\n        </ul>\n      `\n    }\n  }\n</script>\n\n<render bookList/>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    bookTitles: [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]\n  };\n  \n  $effect = (title) => {\n    return `<span data-testid=\"book-title\">${title}</span>`;\n  };\n\n  $: render = ({ state, effect }) => {\n    return `\n      <ul>\n        ${state.bookTitles.map((title) => `\n          <li data-testid=\"book-item\">${effect(title)}</li>\n        `).join('')}\n      </ul>\n    `;\n  };\n</script>\n\n<render />",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    books: [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]\n  }\n</script>\n\n<ul>\n  {@render $state.books}\n    {@for $state.books as book (book)}\n      <li data-testid=\"book-item\">\n        {@render title(book)}\n          {@snippet title}\n            <span data-testid=\"book-title\">{book}</span>\n          {@/snippet}\n        {@/render}\n      </li>\n    {@/for}\n  {@/render}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:11 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    books: [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]\n  }\n</script>\n\n<ul>\n  {#each $state.books as book}\n    <li data-testid=\"book-item\">\n      {@render title book}\n    </li>\n  {/each}\n</ul>\n\n$effect = {\n  title: (book) => {\n    return `<span data-testid=\"book-title\">${book}</span>`\n  }\n}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:15 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    books: [\n      \"The Lord of the Rings\",\n      \"To Kill a Mockingbird\",\n      \"1984\"\n    ]\n  }\n\n  $effect = ({$state}, {$state = { books: $state.books }}) => {\n    return {\n      title: (book) => {\n        return <span data-testid=\"book-title\">{book}</span>;\n      }\n    };\n  }\n</script>\n\n<ul>\n  {@render $effect.title as title}\n  {#each $state.books as book (book)}\n    <li data-testid=\"book-item\">\n      {@render title book={book} />\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:24 Argument name clash\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    bookTitles: [\n      'The Lord of the Rings',\n      'To Kill a Mockingbird',\n      '1984'\n    ]\n  };\n\n  $effect = {\n    document: {\n      body: {\n        $: {\n          bookTitles: $state.bookTitles\n        }\n      }\n    }\n  };\n\n  $: {\n    title: (bookTitle) => {\n      return `<span data-testid=\"book-title\">${bookTitle}</span>`\n    }\n  }\n</script>\n\n<ul>\n  {@render $effect.document.body.$}\n  {#each $state.bookTitles as bookTitle}\n    <li data-testid=\"book-item\">\n      {@render title(bookTitle)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:30:11 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = ['The Lord of the Rings', 'To Kill a Mockingbird', '1984'];\n  $effect = ($render) => {\n    $: $state = $state.map((title) => $render(title));\n  };\n</script>\n\n<ul>\n  {#each $state as title (title)}\n    <li data-testid=\"book-item\">\n      {@render title as $title (title)}\n        <span data-testid=\"book-title\">{$title}</span>\n      {@/}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:15 `{@render ...}` tags can only contain call expressions\nhttps://svelte.dev/e/render_tag_invalid_expression"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = [\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ];\n\n  $effect = (title) => {\n    return (\n      <span data-testid=\"book-title\">{title}</span>\n    );\n  };\n</script>\n\n<ul>\n  {@render $state, (item) => (\n    <li data-testid=\"book-item\">\n      {$effect(item)}\n    </li>\n  )}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state = {\n    bookTitles: ['The Lord of the Rings', 'To Kill a Mockingbird', '1984']\n  }\n  \n  $effect = function($state, $prefix) {\n    return {\n      bookTitles: $state.bookTitles,\n      title: function($params) {\n        return <span data-testid=\"book-title\">{$params.title}</span>;\n      }\n    };\n  }\n</script>\n\n<ul>\n  {@render $prefix.bookTitles, ($bookTitle, $index) => \n    <li data-testid=\"book-item\">\n      {@render $prefix.title, { title: $bookTitle }}\n    </li>\n  }\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:15 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-14T23:04:56.442Z",
    "tps": 2.66,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 106,
      "evalDurationNs": 39821921000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 18329781000,
      "promptTps": 15.33,
      "loadDurationNs": 2413478,
      "measuredAt": "2026-09-16T18:36:16.335Z"
    }
  }
]