[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\" class=\"count\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n\t.count {\n\t\tmin-width: 2rem;\n\t\ttext-align: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\" class=\"count-display\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 1rem;\n\t\tpadding: 0.5rem;\n\t}\n\t\n\t.count-display {\n\t\tmin-width: 3rem;\n\t\ttext-align: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { count = 0 } = $state();\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={() => count--}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={() => count++}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'count')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'count')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'count')\n\n\tin <unknown>\n",
          "Cannot read properties of undefined (reading 'count')\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\t\n\tfunction increment() {\n\t\tcount++;\n\t}\n\t\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\tgap: 0.5rem;\n\t\talign-items: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\" class=\"count-display\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t\tpadding: 0.5rem 1rem;\n\t\tbackground: #f5f5f5;\n\t\tborder-radius: 0.25rem;\n\t}\n\t.count-display {\n\t\tmin-width: 2rem;\n\t\ttext-align: center;\n\t}\n\tbutton {\n\t\twidth: 1.5rem;\n\t\theight: 1.5rem;\n\t\tbackground: #eee;\n\t\tborder: 1px solid #ccc;\n\t\tborder-radius: 0.25rem;\n\t\tcursor: pointer;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\tgap: 0.5rem;\n\t\talign-items: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 7,
    "pass1": 0.7,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet { wordCount, charCount, isLong } = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.length;\n\t\tconst isLong = words > 15;\n\t\treturn { wordCount: words, charCount: chars, isLong };\n\t});\n\n\tfunction clearText() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    placeholder=\"Type some text...\"\n  />\n  <button data-testid=\"clear-button\" onclick={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet { wordCount, charCount, isLong } = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.length;\n\t\tconst isLong = words > 15;\n\t\treturn { wordCount: words, charCount: chars, isLong };\n\t});\n\n\tfunction clear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    placeholder=\"Enter some text...\"\n  />\n  <button\n    data-testid=\"clear-button\"\n    onclick={clear}\n  >\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet { wordCount, charCount, isLong } = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.length;\n\t\tconst isLong = words > 15;\n\t\treturn { wordCount: words, charCount: chars, isLong };\n\t});\n\n\tfunction clear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n  <button data-testid=\"clear-button\" onclick={clear}>Clear</button>\n  <div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet wordCount = $derived.by(() => {\n\t\tif (!text) return 0;\n\t\treturn text.trim().split(/\\s+/).length;\n\t});\n\n\tlet charCount = $derived.by(() => {\n\t\tif (!text) return 0;\n\t\treturn text.trim().length;\n\t});\n\n\tlet isLong = $derived.by(() => wordCount > 15);\n\n\tlet clearState = $state(false);\n\n\tfunction clearText() {\n\t\ttext = '';\n\t\tclearState = true;\n\t}\n\n\t$effect(() => {\n\t\tif (clearState) {\n\t\t\tclearText();\n\t\t\tclearState = false;\n\t\t}\n\t});\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    placeholder=\"Enter some text...\"\n  />\n  <button\n    data-testid=\"clear-button\"\n    onclick={clearText}\n  >\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet { wordCount, charCount, isLong } = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.length;\n\t\tconst isLong = words > 15;\n\t\treturn { wordCount: words, charCount: chars, isLong };\n\t});\n\n\tfunction clear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n  <button\n\t\tdata-testid=\"clear-button\"\n\t\tonclick={clear}\n\t>\n\t\tClear\n\t</button>\n  <div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\tif (!text) return 0;\n\t\treturn text.trim().split(/\\s+/).filter(Boolean).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => {\n\t\tif (!text) return 0;\n\t\treturn text.replace(/\\s+/g, ' ').length;\n\t});\n\t\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\t\n\tfunction clearText() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n    placeholder=\"Type some text...\"\n  />\n  \n  <button \n    data-testid=\"clear-button\" \n    onclick={clearText}\n  >\n    Clear\n  </button>\n  \n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet derived = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.replace(/\\s+/g, '').length;\n\t\tconst isLong = words > 15;\n\n\t\treturn {\n\t\t\twords,\n\t\t\tchars,\n\t\t\tisLong\n\t\t};\n\t});\n\n\tfunction clear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" onclick={clear}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.words}</p>\n    <p data-testid=\"char-count\">Characters: {derived.chars}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {derived.isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Characters: 11\u001b[39m\nReceived:\n\u001b[31m  Characters: 10\u001b[39m"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => text.trim().split(/\\s+/).filter(Boolean).length);\n\tlet charCount = $derived.by(() => text.length);\n\tlet lengthStatus = $derived.by(() => {\n\t\tif (wordCount === 0) return 'Status: Empty';\n\t\treturn wordCount > 15 ? 'Status: Long text' : 'Status: Short text';\n\t});\n\t\n\tfunction clearText() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    value={text} \n    oninput={(e) => text = e.target.value}\n  />\n  <button data-testid=\"clear-button\" onclick={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">{lengthStatus}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Status: Short text\u001b[39m\nReceived:\n\u001b[31m  Status: Empty\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Status: Short text\u001b[39m\nReceived:\n\u001b[31m  Status: Empty\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet stats = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean).length;\n\t\tconst chars = text.length;\n\t\tconst isLong = words > 15;\n\t\treturn { words, chars, isLong };\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button data-testid=\"clear-button\" onclick={() => text = ''}>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {stats.words}</p>\n\t\t<p data-testid=\"char-count\">Characters: {stats.chars}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {stats.isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\n\tlet wordCount = $derived.by(() => {\n\t\tif (!text) return 0;\n\t\treturn text.split(/\\s+/).filter(Boolean).length;\n\t});\n\n\tlet charCount = $derived.by(() => {\n\t\tif (!text) return 0;\n\t\treturn text.replace(/\\s+/g, '').length;\n\t});\n\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\n\tfunction clearText() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    placeholder=\"Enter some text...\"\n  />\n  <button\n    data-testid=\"clear-button\"\n    onclick={clearText}\n  >\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Characters: 11\u001b[39m\nReceived:\n\u001b[31m  Characters: 10\u001b[39m"
        ]
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\n\tfunction addGeorge() {\n\t\tcharacters.push('George');\n\t}\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/lib/SeinfeldCharacters.svelte\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\n\tfunction addGeorge() {\n\t\tcharacters.push('George');\n\t}\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\t\n\t$effect(() => {\n\t\tconsole.log('Characters updated:', characters);\n\t});\n\t\n\tfunction addGeorge() {\n\t\tcharacters.push('George');\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character (character)}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>\n\t\tAdd George\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/lib/SeinfeldCharacters.svelte\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\tlet count = $derived(characters.length);\n\n\t// $effect for manual cleanup demonstration (not strictly needed here)\n\t$effect(() => {\n\t\tconst cleanup = () => {\n\t\t\t// No actual cleanup needed for this simple example\n\t\t\tconsole.log('Cleanup triggered for characters:', characters);\n\t\t};\n\t\treturn cleanup;\n\t});\n\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, 'George'];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character (character)}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, 'George'];\n\t}\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as name (name)}\n      <li data-testid=\"character\">{name}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={() => characters.push('George')}>\n    Add George\n  </button>\n</div>\n\n<script>\n  let characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as name (name)}\n      <li data-testid=\"character\">{name}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={() => characters.push('George')}>\n    Add George\n  </button>\n</div>\n\n<script>\n  let characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\n\tfunction addGeorge() {\n\t\tcharacters.push('George');\n\t}\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\t\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, 'George'];\n\t}\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\tlet showGeorge = $state(false);\n\n\t$effect(() => {\n\t\tif (showGeorge) {\n\t\t\tcharacters.push('George');\n\t\t\tshowGeorge = false;\n\t\t}\n\t});\n\n\tfunction addGeorge() {\n\t\tshowGeorge = true;\n\t}\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 8,
    "pass1": 0.7999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(10);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button onclick={increment} data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n// src/lib/DoubledValue.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived.by(() => number * 2);\n\n\t// Example effect that runs whenever number changes\n\t$effect(() => {\n\t\tconsole.log(`Number changed to: ${number}, Doubled: ${doubled}`);\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "number is not defined\n\n\tin <unknown>\n",
          "number is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(10);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $effect(() => number * 2);\n</script>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:11:15 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No reactive state needed for a simple display\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\tHello, World!\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px auto;\n\t\tpadding: 10px;\n\t\tmax-width: 600px;\n\t}\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No state or logic needed for this simple component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\tHello, World!\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px auto;\n\t\tpadding: 10px;\n\t\tfont-size: 24px;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No reactive state needed for this simple component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\tHello, World!\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\tdisplay: flex;\n\t\tjustify-content: center;\n\t\talign-items: center;\n\t\tmin-height: 100vh;\n\t\tfont-size: 2rem;\n\t\tmargin: 1rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No state needed for this simple component\n</script>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 1rem;\n\t\tfont-size: 1.5rem;\n\t}\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No props or state needed for this simple component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\tHello, World!\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px auto;\n\t\tpadding: 10px;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px auto;\n\t\tmax-width: 600px;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No reactive state needed for this simple component\n</script>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px auto;\n\t\tpadding: 10px;\n\t\tmax-width: 600px;\n\t}\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\tfont-size: 2em;\n\t\ttext-align: center;\n\t\tmargin: 1rem;\n\t}\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No reactive state needed for a static \"Hello, World!\" message\n</script>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px auto;\n\t\tpadding: 10px;\n\t\tfont-size: 24px;\n\t}\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// No reactive state needed for this simple component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 1rem;\n\t\tfont-size: 1.5rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Basic inspect to log changes\n\t$inspect(text);\n\n\t// Inspect with custom callback\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Custom inspect callback:', value);\n\t});\n\n\t// Trace inspection\n\t$inspect.trace(() => {\n\t\tconsole.log('Trace inspection triggered');\n\t});\n\n\t// Derived state for character count\n\tlet charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Basic inspection\n\t$inspect(text);\n\n\t// Inspection with custom callback\n\t$inspect(text).with((v) => {\n\t\tconsole.log('Text updated to:', v);\n\t});\n\n\t// Inspection with trace\n\t$inspect.trace(() => {\n\t\tconsole.log('Text changed:', text);\n\t});\n\n\t// Derived state for character count\n\tlet charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Basic inspection\n\t$inspect(text);\n\n\t// Inspection with custom callback\n\t$inspect(text).with((val) => {\n\t\tconsole.log('Custom log:', val);\n\t});\n\n\t// Trace inspection\n\t$inspect.trace();\n\n\t// Derived state for character count\n\tconst charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\tlet inspect = $inspect(text);\n\tinspect.with((value) => {\n\t\tconsole.log('Custom inspect callback:', value);\n\t});\n\n\t$inspect.trace(() => {\n\t\tconsole.log('Trace: text changed to', text);\n\t});\n\n\tlet charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:12:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\t// Use $state for reactive text input\n\tlet text = $state('Hello world');\n\n\t// Basic $inspect to log input value changes\n\t$inspect(text);\n\n\t// $inspect.with to track updates with custom callback\n\t$inspect.with(text, (value) => {\n\t\tconsole.log(`Text updated to: ${value}`);\n\t});\n\n\t// $inspect.trace() inside an effect that runs on change\n\t$inspect.trace(text, (value) => {\n\t\tconsole.log(`Trace: Text now is \"${value}\"`);\n\t});\n\n\t// Derived value for character count\n\tconst charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t\tplaceholder=\"Type something...\"\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:12:1 `$inspect.with` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/lib/InspectComponent.svelte\n<script>\n\t// Use $state for reactive text input\n\tlet text = $state(\"Hello world\");\n\n\t// Use basic $inspect to log the input value\n\t$inspect(text);\n\n\t// Use $inspect with a custom callback to track updates\n\t$inspect(text).with((value) => {\n\t\tconsole.log(`Text value changed to: \"${value}\"`);\n\t});\n\n\t// Use $inspect.trace() inside an effect to show call stack\n\t$inspect.trace(() => {\n\t\tconsole.log(\"Trace effect triggered\");\n\t});\n\n\t// Derived value for character count\n\tlet charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:17:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Basic inspection\n\t$inspect(text);\n\n\t// Inspection with custom callback\n\tconst inspected = $inspect(text).with((value) => {\n\t\tconsole.log('Custom inspection:', value);\n\t});\n\n\t// Trace inspection\n\t$inspect.trace(() => {\n\t\tconsole.log('Trace inspection: text =', text);\n\t});\n\n\t// Derived character count\n\tconst charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Basic inspection\n\t$inspect(text);\n\n\t// Inspection with custom callback\n\t$inspect(text).with((v) => {\n\t\tconsole.log('Text changed to:', v);\n\t});\n\n\t// Inspection with trace\n\t$inspect.trace(text);\n\n\t// Derived value for character count\n\tlet textLength = $derived(text.length);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {textLength}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\t\n\t// Basic inspection\n\t$inspect(text);\n\t\n\t// Inspection with custom callback\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Custom inspection: value changed to', value);\n\t});\n\t\n\t// Inspection with trace\n\t$inspect.trace(text);\n\t\n\t// Derived value for character count\n\tlet charCount = $derived(text.length);\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Basic inspection\n\t$inspect(text);\n\n\t// Inspection with custom callback\n\t$inspect(text).with((value) => {\n\t\tconsole.log(`Text updated to: ${value}`);\n\t});\n\n\t// Trace inspection\n\t$inspect.trace(() => {\n\t\tconsole.log(`Text value is: ${text}`);\n\t});\n\n\t// Derived character count\n\tlet charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:1 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 8,
    "pass1": 0.7999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\t\n\tlet countValue = $state(count);\n\t\n\tfunction increment() {\n\t\tcountValue += 1;\n\t}\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {countValue}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {countValue}</p>\n      <p>ShowDetails is {showDetails ? 'true' : 'false'}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/lib/PropsDemo.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet localCount = $state(0);\n\n\tfunction increment() {\n\t\tlocalCount += 1;\n\t}\n\n\tconst props = {\n\t\tname: name,\n\t\tcount: count,\n\t\tshowDetails: showDetails\n\t};\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "Cannot access 'props' before initialization\n\n\tin <unknown>\n",
          "Cannot access 'props' before initialization\n\n\tin <unknown>\n",
          "Cannot access 'props' before initialization\n\n\tin <unknown>\n",
          "Cannot access 'props' before initialization\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\t\n\t// Use $state for reactive count (even though props have defaults, we want to track changes)\n\t// This is safe because $props returns a reactive object that updates when props change\n\tlet expanded = $state(false);\n\t\n\t// Increment button handler\n\tfunction increment() {\n\t\tcount += 1;\n\t}\n\t\n\t// Toggle details handler\n\tfunction toggleDetails() {\n\t\texpanded = !expanded;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button \n\t\tdata-testid=\"increment-button\" \n\t\tonclick={increment}\n\t>\n\t\tIncrement\n\t</button>\n\n\t{#if expanded || showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\n\tlet countState = $state(count);\n\n\tfunction increment() {\n\t\tcountState += 1;\n\t}\n\n\tconst nameDisplay = $derived(name);\n\tconst countDisplay = $derived(countState);\n</script>\n\n<h1 data-testid=\"name-display\">{nameDisplay}</h1>\n<p data-testid=\"count-display\">Count: {countDisplay}</p>\n<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n{#if showDetails}\n\t<div data-testid=\"details\">\n\t\t<p>Name is {nameDisplay}</p>\n\t\t<p>Count is {countDisplay}</p>\n\t\t<p>ShowDetails is {showDetails}</p>\n\t</div>\n{/if}",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/lib/PropsDemo.svelte\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\n\tlet localCount = $state(count);\n\n\tfunction increment() {\n\t\tlocalCount++;\n\t}\n\n\tconst details = showDetails\n\t\t? `Name is ${name}, Count is ${localCount}, ShowDetails is true`\n\t\t: '';\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {localCount}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {localCount}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/lib/PropsDemo.svelte\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\tlet countValue = $state(count);\n\n\tfunction increment() {\n\t\tcountValue++;\n\t}\n\n\tlet details = $derived.by(() => {\n\t\treturn showDetails ? {\n\t\t\tname: name,\n\t\t\tcount: countValue,\n\t\t\tshowDetails: showDetails\n\t\t} : null;\n\t});\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countValue}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countValue}</p>\n\t\t\t<p>ShowDetails is {showDetails.toString()}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/lib/PropsDemo.svelte\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\n\t// Use $state for count to demonstrate both $props and $state\n\tlet countState = $state(count);\n\n\tfunction increment() {\n\t\tcountState++;\n\t}\n\n\t// Effect to sync count prop with state (optional for this task)\n\t$effect(() => {\n\t\tif (count !== countState) {\n\t\t\tcountState = count;\n\t\t}\n\t});\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countState}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "expected 'Count: 10' to match /11/"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\t\n\tfunction increment() {\n\t\tcount += 1;\n\t}\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails ? 'true' : 'false'}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/lib/PropsDemo.svelte\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\n\tlet countValue = $state(count);\n\n\tfunction increment() {\n\t\tcountValue++;\n\t}\n\n\tconst nameDisplay = $derived(name);\n\tconst countDisplay = $derived(countValue);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{nameDisplay}</h1>\n  <p data-testid=\"count-display\">Count: {countDisplay}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {nameDisplay}</p>\n      <p>Count is {countDisplay}</p>\n      <p>ShowDetails is {showDetails ? 'true' : 'false'}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/lib/PropsDemo.svelte\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\n\tlet countValue = $state(count);\n\n\tfunction increment() {\n\t\tcountValue += 1;\n\t}\n\n\tconst details = showDetails\n\t\t? [\n\t\t\t\t`Name is ${name}`,\n\t\t\t\t`Count is ${countValue}`,\n\t\t\t\t`ShowDetails is true`\n\t\t  ]\n\t\t: [];\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countValue}</p>\n\t<button\n\t\tdata-testid=\"increment-button\"\n\t\tonclick={increment}\n\t>\n\t\tIncrement\n\t</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t{#each details as line}\n\t\t\t\t<p>{line}</p>\n\t\t\t{/each}\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder-8b:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet books = $state([\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t]);\n\n\tfunction title(book) {\n\t\treturn (\n\t\t\t<span data-testid=\"book-title\">{book}</span>\n\t\t);\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:3 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\timport { title } from './BookList.svelte';\n\n\tlet books = $state([\n\t\t'The Lord of the Rings',\n\t\t'To Kill a Mockingbird',\n\t\t'1984'\n\t]);\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>\n\n// src/routes/BookList.svelte\n<script>\n\texport let title;\n</script>\n\n<span data-testid=\"book-title\">{title}</span>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:23:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = $state(['The Lord of the Rings', 'To Kill a Mockingbird', '1984']);\n\n\t// Define a snippet that renders a single book title\n\tfunction title({ title }) {\n\t\treturn `<span data-testid=\"book-title\">${title}</span>`;\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title({ title: book })}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet books = $state([\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t]);\n\n\tfunction title(book) {\n\t\treturn (\n\t\t\t<span data-testid=\"book-title\">{book}</span>\n\t\t);\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:3 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet books = [\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t];\n\n\t// Snippet definition: receives a single book title parameter\n\tfunction title(book) {\n\t\treturn `<span data-testid=\"book-title\">${book}</span>`;\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m// src/routes/+page.svelte \u001b[0m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m// src/routes/+page.svelte \u001b[0m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<ul>\n  {@render title(\"The Lord of the Rings\")}\n  {@render title(\"To Kill a Mockingbird\")}\n  {@render title(\"1984\")}\n</ul>\n\n{#snippet title(book)}\n  <li data-testid=\"book-item\">\n    <span data-testid=\"book-title\">{book}</span>\n  </li>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet books = [\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t];\n\n\t// Snippet definition: title receives a book string\n\t// Snippets are declared with `snippet` keyword\n\tsnippet title(book) {\n\t\treturn (\n\t\t\t<span data-testid=\"book-title\">{book}</span>\n\t\t);\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = $state([\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t]);\n\n\t// Snippet that renders a single book title with data-testid\n\tfunction title(book) {\n\t\treturn (\n\t\t\t<span data-testid=\"book-title\">{book}</span>\n\t\t);\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:3 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/lib/BookList.svelte\n<script>\n\t// Hardcoded array of book titles\n\tlet books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n\t// Snippet definition: receives a single book title\n\tfunction title({ title }) {\n\t\treturn `<span data-testid=\"book-title\">${title}</span>`;\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title({ title: book })}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m// src/lib/BookList.svelte \u001b[0m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m// src/lib/BookList.svelte \u001b[0m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet books = [\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t];\n\n\t// Snippet: renders a book title in a span with data-testid\n\tfunction title(book) {\n\t\treturn (\n\t\t\t<span data-testid=\"book-title\">{book}</span>\n\t\t);\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:14:3 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-21T09:33:01.155Z",
    "tps": 14.81,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 213,
      "evalDurationNs": 14384484000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 2002822000,
      "promptTps": 140.3,
      "loadDurationNs": 2439589,
      "measuredAt": "2026-09-21T13:26:00.892Z"
    }
  }
]