[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n\n  function handleDecrement() {\n    count -= 1;\n  }\n\n  function handleIncrement() {\n    count += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={handleDecrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={handleIncrement}>+</button>\n</div>\n\n<style>\n  .counter {\n    font-family: Arial, sans-serif;\n    padding: 10px;\n    border: 1px solid #ccc;\n    border-radius: 5px;\n    text-align: center;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => $count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => $count++}>+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n    font-family: Arial, sans-serif;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $let count = 0;\n\n  $effect(() => {\n    console.log('Count updated to', count);\n  });\n</script>\n\n<style>\n  .counter {\n    font-size: 24px;\n    margin: 20px;\n    padding: 10px;\n    border: 1px solid #ccc;\n    border-radius: 5px;\n    text-align: center;\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:7 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n\n  function handleDecrement() {\n    count -= 1;\n  }\n\n  function handleIncrement() {\n    count += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={handleDecrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={handleIncrement}>+</button>\n</div>\n\n<style>\n  .counter {\n    font-family: Arial, sans-serif;\n    text-align: center;\n    padding: 10px;\n    border: 1px solid #ccc;\n    border-radius: 5px;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count--}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count++}>+</button>\n</div>\n\n<style>\n  .counter {\n    font-family: Arial, sans-serif;\n    color: #333;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n\n  function handleIncrement() {\n    count += 1;\n  }\n\n  function handleDecrement() {\n    count -= 1;\n  }\n</script>\n\n<div class=\"counter\" data-testid=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={handleDecrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={handleIncrement}>+</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:0 `<div>` was left open\nhttps://svelte.dev/e/element_unclosed"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $let count = 0;\n\n  $effect(() => {\n    console.log('Count updated to', count);\n  });\n</script>\n\n<style>\n  .counter {\n    font-size: 24px;\n    margin: 20px;\n    padding: 10px;\n    border: 1px solid #ccc;\n    border-radius: 5px;\n    text-align: center;\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count -= 1}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count += 1}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:7 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n\n  function handleDecrement() {\n    count -= 1;\n  }\n\n  function handleIncrement() {\n    count += 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={handleDecrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={handleIncrement}>+</button>\n</div>\n\n<style>\n  .counter {\n    font-family: Arial, sans-serif;\n    padding: 10px;\n    border: 1px solid #ccc;\n    border-radius: 5px;\n  }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n\n  function handleDecrement() {\n    count--;\n  }\n\n  function handleIncrement() {\n    count++;\n  }\n</script>\n\n<style>\n  .counter {\n    font-family: Arial, sans-serif;\n    color: #333;\n    text-align: center;\n    padding: 20px;\n    border: 1px solid #ccc;\n    border-radius: 5px;\n    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={handleDecrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={handleIncrement}>+</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:27:0 `<div>` was left open\nhttps://svelte.dev/e/element_unclosed"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state let count = 0;\n</script>\n\n<style>\n  .counter {\n    font-size: 24px;\n    margin: 20px;\n    padding: 10px;\n    border: 1px solid #ccc;\n    border-radius: 5px;\n    cursor: pointer;\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count -= 1}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count += 1}>+</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$lib/runes';\n\n  $state('number', 5);\n  $derived('doubled', () => number * 2);\n\n  function increment() {\n    $state('number', number + 1);\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state('number')}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived('doubled')}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$lib/runes';\n\n  $state.number = 5;\n  $derived.doubled = () => $state.number * 2;\n\n  $effect.on('Increment', () => {\n    $state.number += 1;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived.doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$lib';\n\n  $state.number = 5;\n  $derived.doubled = () => $state.number * 2;\n\n  $effect.on('increment-button', () => {\n    $state.number += 1;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">{number}</p>\n  <p data-testid=\"doubled-value\">{doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$runtime';\n\n  let number = $state(5);\n\n  $derived('doubled', () => number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number = number + 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '@sveltejs/runner';\n\n  $state.number = 5;\n  $derived.doubled = ($state.number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived.doubled}</p>\n  <button data-testid=\"increment- button\" on:click={() => $state.number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$lib/runes';\n\n  let number = $state(5);\n  const doubled = $derived(number, n => n * 2);\n\n  function increment() {\n    number = number + 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">{number}</p>\n  <p data-testid=\"doubled-value\">{doubled}</p>\n  <button data-testid=\"increment-button' on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:18:6 Unexpected end of input\nhttps://svelte.dev/e/unexpected_eof"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$lib/runes';\n\n  $state.number = 5;\n  $derived.doubled = () => $state.number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived.doubled}</p>\n  <button data-testid=\"increment- button\" on:click={() => $state.number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '@sveltejs/runner';\n\n  $state.number = 5;\n  $derived.doubled = () => $state.number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived.doubled}</p>\n  <button data-testid=\"increment- button\" on:click={() => $state.number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$lib/runes';\n\n  let number = $state(5);\n  const doubled = $derived(number, n => n * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment- button\" on:click={() => number = number + 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '@sveltejs/runner';\n\n  $state.number = 5;\n  $derived.doubled = () => $state.number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$derived.doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $on } from 'svelte/rune';\n\n  $state.text = '';\n  $derived.wordCount = $state.text.split(' ').length;\n  $derived.charCount = $state.text.length;\n  $derived.isLong = () => $state.text.split(' ').length > 15;\n\n  $on('input', (e) => {\n    $state.text = e.target.value;\n  });\n\n  $on('click', () => {\n    $state.text = '';\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-identifier\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>\n\n<style>\n  div { margin-top: 20px; }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $on } from 'svelte/rune';\n\n  $state.text = '';\n  $derived.wordCount = $state.text => {\n    return $state.text.trim().split(/\\s+/).length;\n  };\n  $derived.charCount = $state.text => {\n    return $state.text.length;\n  };\n  $derived.isLong = $state.text => {\n    return $derived.wordCount > 15;\n  };\n\n  $on('Clear', () => {\n    $state.text = '';\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $on('Clear')}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.charCount}</p>\n    <p data-testid=\"length-indicator\">\n      {$derived.isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:7:35 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $on } from '$lib';\n\n  $state.text = '';\n\n  $derived.countWords = () => {\n    return this.text.trim().split(/\\s+/).length;\n  };\n\n  $derived.countChars = () => {\n    return this.text.length;\n  };\n\n  $derived.isLongText = () => {\n    return this.countWords() > 15;\n  };\n\n  $on('input', (e) => {\n    $state.text = e.target.value;\n  });\n\n  $on('click', () => {\n    $state.text = '';\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\">Clear</button>\n\n  <div>\n    <p data-testid=\"word-count\">Words: {countWords()}</p>\n    <p data-testid=\"char-count\">Characters: {countChars()}</p>\n    <p data-testid=\"length-identifier\">Status: {isLongText ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $on } from 'svelte/rune';\n\n  $state.text = '';\n\n  $derived.countWords = () => {\n    return this.text.split(/\\s+/).length;\n  };\n\n  $derived.countChars = () => {\n    return this.text.length;\n  };\n\n  $derived.isLongText = () => {\n    return this.countWords() > 15;\n  };\n\n  $on('input', (e) => {\n    $state.text = e.target.value;\n  });\n\n  $on('click', () => {\n    $state.text = '';\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $on('click')()} >Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {countWords()}</p>\n    <p data-testid=\"char-count\">Characters: {countChars()}</p>\n    <p data-testid=\"length-indicator\">{isLongText() ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $on } from '@sveltejs/runner';\n\n  $state.setText('');\n  $derived.set('wordCount', () => {\n    return $state.getText().split(' ').length;\n  });\n  $derived.set('charCount', () => {\n    return $state.getText().length;\n  });\n  $derived.set('isLong', () => {\n    return $state.getText().split(' ').length > 15;\n  });\n\n  $on('input', (e) => {\n    $state.setText(e.target.value);\n  });\n\n  const clear = () => {\n    $state.setText('');\n  };\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.getText()} />\n  <button data-testid=\"clear-button\" on:click={clear}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.get('wordCount')}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.get('charCount')}</p>\n    <p data-testid=\"length-indicator\">\n      {$derived.get('isLong') ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $effect } from 'svelte/rune';\n\n  $state.text = '';\n\n  $derived.countWords = $state.text.split(' ').length;\n  $derived.countChars = $state.text.length;\n  $derived.isLong = $state.text.split(' ').length > 15;\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" on:input={$state.text = $event.target.value} />\n  <button data-testid=\"clear-button\" on:click={() => $state.text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {countWords}</p>\n    <p data-testid=\"char-count\">Characters: {countChars}</p>\n    <p data-testid=\"length-indicator\">{isLong ? 'Status: Long text' : 'Status: Short text'}</p>\n  </div>\n</div>\n\n<style>\n  div { margin-top: 20px; }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $on } from 'svelte/runes';\n\n  const textInput = $state('');\n  const { words, chars, isLong } = $derived.by({\n    words: () => textInput.value.split(/\\s+/).length,\n    chars: () => textInput.value.length,\n    isLong: () => words > 15,\n  });\n\n  $on('Clear', () => {\n    textInput('');\n  });\n</script>\n\n<div>\n  <input type=\"text\" data-testid=\"text-input\" bind:value={textInput} />\n  <button data-testid=\"clear-button\" on:click={() => $on('Clear')}>Clear</button>\n\n  <div>\n    <p data-testid=\"word-count\">Words: {words}</p>\n    <p data-testid=\"char-count\">Characters: {chars}</p>\n    <p data-testid=\"length-identifier\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from 'svelte/rune';\n\n  $state.text = '';\n  $derived.wordCount = () => {\n    return this.text.trim().split(/\\s+/).length;\n  };\n  $derived.charCount = () => {\n    return this.text.length;\n  };\n  $derived.isLong = () => {\n    return this.wordCount > 15;\n  };\n\n  function clearText() {\n    $state.text = '';\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">{isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>\n\n<style>\n  div { margin-top: 20px; }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from '$svelte';\n\n  let text = $state('');\n  const derived = $derived.by(() => ({\n    wordCount: () => text.split(' ').length,\n    charCount: () => text.length,\n    isLong: () => text.split(' ').length > 15\n  }));\n</script>\n\n<div>\n  <input type=\"text\" data-testid=\"text-input\" bind:value={text} />\n  <button data-testid=\"clear-button\" on:click={() => text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount()}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount()}</p>\n    <p data-testid=\"length-indicator\">\n      {derived.isLong() ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>\n\n<style>\n  div { margin-top: 20px; }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived, $on } from 'svelte/rune';\n\n  $state.text = '';\n  $derived.wordCount = $state.text.split(' ').length;\n  $derived.charCount = $state.text.length;\n  $derived.isLong = () => $state.text.split(' ').length > 15;\n\n  $on('Clear', () => {\n    $state.text = '';\n  });\n</script>\n\n<div>\n  <input type=\"text\" data-testid=\"text-input\" bind:value={$state.text} />\n  <button data-testid=\"clear-button\" on:click={() => $on('Clear')}>Clear</button>\n\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong() ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  \n  function addToCharacters() {\n    characters.push('George');\n  }\n</script>\n\n<style>\n  ul {\n    list-style-type: none;\n    padding: 0;\n  }\n  li {\n    margin: 5px 0;\n  }\n</style>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addToCharacters}>Add George</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:21:0 `<div>` was left open\nhttps://svelte.dev/e/element_unclosed"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addToCharacters() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as person}\n      <li data-testid=\"character\">{person}</li>\n    {/each}\n  </ul>\n\n  <button data-testid=\"add-george-button\" on:click={addToCharacters}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  \n  function addToCharacters() {\n    characters.push('George');\n  }\n</script>\n\n<style>\n  ul { list-style-type: none; padding: 0; }\n  li { margin: 5px 0; }\n  button { padding: 5px 10px; }\n</style>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as person}\n      <li data-testid=\"character\">{person}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addToCharacters}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  \n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as person}\n      <li data-testid=\"character\">{person}</li>\n    {/each}\n  </ul>\n  \n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  function addToCharacters() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as name}\n      <li data-testid=\"character\">{name}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addToCharacters}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addToCharacters() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n\n  <button data-testid=\"add-george-button\" on:click={addToCharacters}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  function addButton() {\n    characters = [...characters, 'George'];\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as name}\n      <li data-testid=\"character\">{name}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addButton}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addButton() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as person (person)}\n      <li data-testid=\"character\">{person}</li>\n    {/each}\n  </ul>\n\n  <button data-testid=\"add-george-button\" on:click={addButton}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  \n  function addButtonClicked() {\n    characters.push('George');\n  }\n</script>\n\n<style>\n  ul {\n    margin: 0;\n    padding: 0;\n  }\n  li {\n    list-style: none;\n    padding: 8px;\n    margin: 4px 0;\n    background: none;\n    border: none;\n    color: black;\n  }\n</style>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addButtonClicked}>Add George</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:26:0 `<div>` was left open\nhttps://svelte.dev/e/element_unclosed"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  \n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<style>\n  ul { list-style-type: none; padding: 0; }\n  li { margin: 4px 0; }\n  button { padding: 4px; }\n</style>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as person (person)}\n      <li data-testid=\"character\">{person}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george- button\" on:click={addGeorge}>Add George</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:0 `<div>` was left open\nhttps://svelte.dev/e/element_unclosed"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from '$svelte/runes';\n\n  $state('number', 5);\n  $state('doubled', 0);\n\n  $effect(() => {\n    $state('doubled', $state('number') * 2);\n  });\n\n  function increment() {\n    $state('number', $state('number') + 1);\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">{number}</p>\n  <p data-testid=\"doubled-value\">{doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from 'svelte';\n\n  $state.number = 5;\n  $state.doubled = $state.number * 2;\n\n  $effect(() => {\n    $state.doubled = $state.number * 2;\n  });\n\n  function increment() {\n    $state.number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">{number}</p>\n  <p data-testid=\"doubled-value\">{doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from '$svelte/runes';\n\n  $state('number', 5);\n  $state('doubled');\n\n  $effect(() => {\n    $set('doubled', $get('number') * 2);\n  });\n\n  function increment() {\n    $set('number', $get('number') + 1);\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$get('number')}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$get('doubled')}</p>\n  <button data-testid=\"increment-button' on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:22:6 Unexpected end of input\nhttps://svelte.dev/e/unexpected_eof"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from '$svelte/stdlib';\n\n  let number = $state(5);\n  let doubled;\n\n  $effect(() => {\n    doubled = number.value * 2;\n  });\n\n  function increment() {\n    number = number + 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">{number}</p>\n  <p data-testid=\"doubled-value\">{doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from '$lib/runes';\n\n  let number = $state(5);\n  let doubled = $effect(() => number.value * 2);\n\n  function increment() {\n    number = number + 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment- button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from '$svelte/runes';\n\n  let number = $state(5);\n  let doubled;\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number = number + 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">{number}</p>\n  <p data-testid=\"doubled-value\">{doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: number = 5;\n  $: doubled = number * 2;\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from '$lib';\n\n  $state.number = 5;\n  let doubled = 0;\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$state.number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: number = 5;\n  $: doubled = $number * 2;\n\n  $effect(() => {\n    doubled = $number * 2;\n  });\n\n  function increment() {\n    $number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: number = 5;\n  $: doubled = number * 2;\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 8,
    "pass1": 0.7999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this simple component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data- testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"greeting svelte-13c97w8\"\u001b[39m\n      \u001b[33mdata-\u001b[39m=\u001b[32m\"\"\u001b[39m\n      \u001b[33mtestid\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[0mHello, World!\u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"greeting svelte-13c97w8\"\u001b[39m\n      \u001b[33mdata-\u001b[39m=\u001b[32m\"\"\u001b[39m\n      \u001b[33mtestid\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[0mHello, World!\u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data- testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"greeting svelte-13c97w8\"\u001b[39m\n      \u001b[33mdata-\u001b[39m=\u001b[32m\"\"\u001b[39m\n      \u001b[33mtestid\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[0mHello, World!\u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"greeting svelte-13c97w8\"\u001b[39m\n      \u001b[33mdata-\u001b[39m=\u001b[32m\"\"\u001b[39m\n      \u001b[33mtestid\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[0mHello, World!\u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No script needed for this component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$svelte/rune';\n\n  let text = $state('Hello world');\n\n  $inspect(text).log();\n  $inspect(text).with((value) => console.log('Input updated:', value));\n  $inspect(text).trace();\n\n  $effect(() => {\n    console.log('Trace:', text);\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $state.text = 'Hello world';\n  $effect(() => {\n    $inspect($state.text).with((value, prevState) => {\n      console.log('Input updated:', value, prevState);\n    });\n  });\n  $inspect($state.text).trace();\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <p data-testid=\"text-value\">Current text: \"{$state.text}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state.text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$state.text` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$self';\n\n  $state(text = 'Hello world');\n\n  $inspect(text).log();\n  $inspect(text).with((value) => {\n    console.log('Input updated:', value);\n  });\n\n  $effect(() => {\n    $inspect.trace();\n  }, [text]);\n\n  $state(charCount = () => text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$lib';\n\n  $state.text = 'Hello world';\n\n  $inspect(text).log();\n  $inspect(text).with((value) => console.log('Text updated:', value));\n\n  $effect(() => {\n    $inspect.trace('Input changed', { text });\n  }, [text]);\n\n  let charCount;\n  $effect(() => {\n    charCount = text.length;\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$svelte/rune';\n\n  const text = $state('Hello world');\n\n  $inspect(text)\n    .with((value) => console.log('Text updated:', value))\n    .trace();\n\n  $effect(() => {\n    $inspect.trace('Input changed');\n  }, [text]);\n\n  const charCount = $state(0);\n  $effect(() => {\n    charCount.set(text.length);\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from 'svelte/rune';\n\n  let text = $state('Hello world');\n\n  $inspect(text).log();\n  $inspect(text).with((value) => {\n    console.log('Input updated with:', value);\n  });\n\n  $effect(() => {\n    $inspect.trace();\n  }, [text]);\n\n  $state('character-count').subscribe(count => {\n    console.log('Character count:', count);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $import { $state, $inspect, $effect } from '$svelte/runes';\n\n  $const initialState = 'Hello world';\n  $let text = $state(initialState);\n\n  $inspect(text).log();\n  $inspect(text).with((value) => {\n    console.log('Input updated:', value);\n  });\n\n  $effect(() => {\n    $inspect.trace('Input changed');\n  }, [text]);\n\n  $let charCount = $state(0);\n  $effect(() => {\n    charCount = $state(text.length);\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:10 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '@sveltejs/runner';\n\n  $state.text = 'Hello world';\n\n  $inspect(text).log();\n  $inspect(text).with((value) => console.log('Input updated:', value));\n  $inspect(text).trace();\n\n  $effect(() => {\n    console.log('Trace:', text);\n  }, [text]);\n\n  let charCount;\n  $effect(() => {\n    charCount = text.length;\n  }, [text]);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $inspect, $effect } from '$lib/runes';\n\n  $state.text = 'Hello world';\n\n  $inspect(text).log();\n  $inspect(text).with((value) => {\n    console.log('Input updated with:', value);\n  });\n\n  $effect(() => {\n    $inspect().trace('Input changed to:', text);\n  }, [text]);\n\n  let charCount = 0;\n\n  $inspect(charCount).dependOn(text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>\n\n<script>\n  $effect(() => {\n    charCount = text.length;\n  }, [text]);\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:28:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $import { $state, $inspect, $effect } from 'svelte/rune';\n\n  $const initialState = $state({ text: 'Hello world' });\n\n  $inspect(initialState.text).with((value) => {\n    console.log('Input value updated:', value);\n  });\n\n  $effect(() => {\n    $inspect.trace(() => `Input value: ${initialState.text}`);\n  }, [initialState.text]);\n\n  const { text } = initialState;\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:10 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  $state(count = 0);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={() => count++}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>\n\n<style>\n  div { margin-bottom: 20px; }\n</style>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$props()` can only be used at the top level of components as a variable declaration initializer\nhttps://svelte.dev/e/props_invalid_placement"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  $state(let: count) {\n    count = 0;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">$name</h1>\n  <p data-testid=\"count-display\">Count: $count</p>\n  <button data-testid=\"increment-button\" on:click={() => $count++}>Increment</button>\n\n  {$showDetails ? \n    <div data-testid=\"details\">\n      <p>Name is $name</p>\n      <p>Count is $count</p>\n      <p>ShowDetails is $showDetails</p>\n    </div> : null}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:9 The keyword 'let' is reserved\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $props, $state } from 'svelte';\n\n  const { name = 'World', count, showDetails } = $props;\n\n  $state.count = count;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n  $state(count = 0);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={() => count++}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$props()` can only be used at the top level of components as a variable declaration initializer\nhttps://svelte.dev/e/props_invalid_placement"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n  \n  $state(let: count) {\n    count;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={() => count = count + 1}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:9 The keyword 'let' is reserved\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  $state(countState = 0);\n\n  $effect(() => {\n    document.getElementById('count-display').textContent = `Count: ${countState}`;\n  });\n\n  function incrementCount() {\n    countState += 1;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {countState}</p>\n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {countState}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:21:0 `<div>` was left open\nhttps://svelte.dev/e/element_unclosed"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  $state(count = 0);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-Button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>\n\n<script>\n  $effect(() => {\n    document.querySelector('[data-testid=\"increment-button\"]').addEventListener('click', () => {\n      count += 1;\n    });\n  });\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:27:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $props, $state } from 'svelte';\n\n  const { name, count, showDetails } = $props({\n    name: 'World',\n    count: 0,\n    showDetails: false,\n  });\n\n  const reactiveCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {reactiveCount}</p>\n  <button data-testid=\"increment-button\" on:click={() => reactiveCount(($count) => $count + 1)}>\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {reactiveCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>\n\n<script>\n  export let props = {};\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:31:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n  $state.count = 0;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$props()` can only be used at the top level of components as a variable declaration initializer\nhttps://svelte.dev/e/props_invalid_placement"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  $state(countState = 0);\n\n  $effect(() => {\n    document.getElementById('count-display').textContent = `Count: ${countState}`;\n  });\n\n  $effect(() => {\n    document.getElementById('increment-button').addEventListener('click', () => {\n      countState++;\n    });\n  });\n\n  $effect(() => {\n    if (showDetails) {\n      document.getElementById('details').style.display = '';\n    } else {\n      document.getElementById('details').style.display = 'none';\n    }\n  });\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {countState}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {countState}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:2 `$props()` can only be used at the top level of components as a variable declaration initializer\nhttps://svelte.dev/e/props_invalid_placement"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "barvhaim/granite-switch-4.1-3b-preview:latest",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const title = ({ title }) => html`\n    <li data-testid=\"book-item\">${title}</li>\n  `;\n</script>\n\n{#render books.map(book => title({ title: book }))}{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const title = {\n    render: ({ book }) => `<span data- testid=\"book-title\">${book}</span>`\n  };\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{title.render({ book })}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m&lt;span data- testid=\"book-title\"&gt;The Lord of the Rings&lt;/span&gt;\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m&lt;span data- testid=\"book-title\"&gt;To Kill a Mockingbird&lt;/span&gt;\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m&lt;span data- testid=\"book-title\"&gt;1984&lt;/span&gt;\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m&lt;span data- testid=\"book-title\"&gt;The Lord of the Rings&lt;/span&gt;\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m&lt;span data- testid=\"book-title\"&gt;To Kill a Mockingbird&lt;/span&gt;\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m&lt;span data- testid=\"book-title\"&gt;1984&lt;/span&gt;\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const title = (book) => {\n    return `<span data- testid=\"book-title\">${book}</span>`;\n  };\n</script>\n\n{#render}\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{title(book)}</li>\n  {/each}\n</ul>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const titleSnippet = (title) => `<span data- testid=\"book-title\">${title}</span>`;\n</script>\n\n{#render}\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{titleSnippet(book)}</li>\n  {/each}\n</ul>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const titleSnippet = {\n    title: (book) => `<span data- testid=\"book-title\">${book}</span>`\n  };\n</script>\n\n{#render titleSnippet.title each book}\n  <li data-testid=\"book-item\">{props}</li>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const title = {\n    snippet: (bookTitle) => html`\n      <span data-testid=\"book-title\">${bookTitle}</span>\n    `\n  };\n</script>\n\n{#render}\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {$snippet.title(book)}\n    </li>\n  {/each}\n</ul>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:17:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const snippet = {\n    title: ({ title }) => `<span data-testid=\"book-title\">${title}</span>`\n  };\n</script>\n\n{#render snippet.title({ title: book }) for book in books #}\n  <li data-testid=\"book-item\">{#computed title = snippet.title({ title: book }) #}{title}#{/computed}</li>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ];\n\n  $snippets.title = (title) => `<span data- testid=\"book-title\">${title}</span>`;\n</script>\n\n{#render}\n<ul>\n  {#each $snippets.title(books) as title (index)}\n    <li data-testid=\"book-item\">{title}</li>\n  {/each}\n</ul>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const title = ({ book }) => html`\n    <span data-TESTID=\"book-title\">${book}</span>\n  `;\n</script>\n\n{#render}\n<ul>\n  {#each books as book}\n    <li data-TESTID=\"book-item\">${title({ book })}</li>\n  {/each}\n</ul>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const snippet = {\n    title: (title) => `<span data- testid=\"book-title\">${title}</span>`\n  };\n</script>\n\n{#render}\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{title(book)}</li>\n  {/each}\n</ul>\n{/render}",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:2 Expected 'if', 'each', 'await', 'key' or 'snippet'\nhttps://svelte.dev/e/expected_block_type"
        ]
      }
    ],
    "timestamp": "2026-10-22T03:16:04.651Z",
    "tps": 4.84,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 163,
      "evalDurationNs": 33659006999,
      "promptEvalCount": 315,
      "promptEvalDurationNs": 211467000,
      "promptTps": 1489.59,
      "loadDurationNs": 1107615,
      "measuredAt": "2026-10-22T03:16:39.395Z"
    }
  }
]