[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n\t\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\t\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\tgap: 0.5rem;\n\t\talign-items: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.5rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\tgap: 0.5rem;\n\t\talign-items: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/lib/Counter.svelte\n<script>\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount++;\n\t}\n\n\tfunction decrement() {\n\t\tcount--;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={increment}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\tgap: 0.5rem;\n\t\talign-items: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={() => count--}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={() => count++}>+</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\t\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\t\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\t\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\t\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\tif (!text.trim()) return 0;\n\t\treturn text.trim().split(/\\s+/).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => text.length);\n\t\n\tlet isLong = $derived.by(() => wordCount > 15);\n\t\n\tfunction handleClear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\" \n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\"\n\t\tonclick={handleClear}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\t\n\tlet { words, characters, isLong } = $derived.by(() => {\n\t\tconst wordArray = text.trim().split(/\\s+/).filter(Boolean);\n\t\tconst wordCount = wordArray.length;\n\t\tconst charCount = text.length;\n\t\tconst isLongText = wordCount > 15;\n\t\t\n\t\treturn {\n\t\t\twords: wordCount,\n\t\t\tcharacters: charCount,\n\t\t\tisLong: isLongText\n\t\t};\n\t});\n\t\n\tfunction clearInput() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\" \n\t\ttype=\"text\" \n\t\tvalue={text} \n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button data-testid=\"clear-button\" onclick={clearInput}>Clear</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {words}</p>\n\t\t<p data-testid=\"char-count\">Characters: {characters}</p>\n\t\t<p data-testid=\"length-indicator\">{isLong ? 'Status: Long text' : 'Status: Short text'}</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\treturn text.trim().split(/\\s+/).filter(word => word.length > 0).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => {\n\t\treturn text.length;\n\t});\n\t\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\t\n\tfunction clearText() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\" \n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\"\n\t\tonclick={clearText}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\treturn text.trim().split(/\\s+/).filter(Boolean).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => {\n\t\treturn text.length;\n\t});\n\t\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\t\n\tfunction clearInput() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\" \n\t\ttype=\"text\" \n\t\tvalue={text} \n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\" \n\t\tonclick={clearInput}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\treturn text.trim().split(/\\s+/).filter(Boolean).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => {\n\t\treturn text.length;\n\t});\n\t\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\t\n\tfunction handleClear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\" \n\t\ttype=\"text\" \n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\" \n\t\tonclick={handleClear}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\tif (!text.trim()) return 0;\n\t\treturn text.split(/\\s+/).filter(word => word.length > 0).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => text.length);\n\t\n\tlet isLong = $derived.by(() => wordCount > 15);\n\t\n\tfunction handleClear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\" \n\t\tbind:value={text}\n\t\tplaceholder=\"Enter text here\"\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\"\n\t\tonclick={handleClear}\n\t\tdisabled={!text.trim()}\n\t>\n\t\tClear\n\t</button>\n\t\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\tif (!text.trim()) return 0;\n\t\treturn text.split(/\\s+/).filter(word => word.length > 0).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => text.length);\n\t\n\tlet isLong = $derived.by(() => wordCount > 15);\n\t\n\tfunction handleClear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\" \n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\"\n\t\tonclick={handleClear}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\treturn text.trim().split(/\\s+/).filter(word => word.length > 0).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => {\n\t\treturn text.length;\n\t});\n\t\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\t\n\tfunction clearInput() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\" \n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\"\n\t\tonclick={clearInput}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\t\n\t// Derived values using $derived.by for complex calculations\n\tlet wordCount = $derived.by(() => {\n\t\treturn text.trim().split(/\\s+/).filter(word => word.length > 0).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => {\n\t\treturn text.length;\n\t});\n\t\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\t\n\tfunction handleClear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\" \n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\"\n\t\tonclick={handleClear}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">\n\t\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t\t</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\t\n\tlet wordCount = $derived.by(() => {\n\t\treturn text.trim().split(/\\s+/).filter(Boolean).length;\n\t});\n\t\n\tlet charCount = $derived.by(() => {\n\t\treturn text.length;\n\t});\n\t\n\tlet isLong = $derived.by(() => {\n\t\treturn wordCount > 15;\n\t});\n\t\n\tfunction handleClear() {\n\t\ttext = '';\n\t}\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\" \n\t\tvalue={text}\n\t\toninput={(e) => text = e.target.value}\n\t/>\n\t<button \n\t\tdata-testid=\"clear-button\"\n\t\tonclick={handleClear}\n\t>\n\t\tClear\n\t</button>\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\t$effect(() => {\n\t\tconsole.log(\"Characters updated:\", characters);\n\t});\n\t\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, \"George\"];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\t$effect(() => {\n\t\tif (characters.length === 3 && !characters.includes(\"George\")) {\n\t\t\tcharacters.push(\"George\");\n\t\t}\n\t});\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={() => {\n\t\tif (!characters.includes(\"George\")) {\n\t\t\tcharacters.push(\"George\");\n\t\t}\n\t}}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 4 to be 3 // Object.is equality",
          "expected 4 to be 3 // Object.is equality"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\t$effect(() => {\n\t\t// Ensure initial state is set\n\t\tif (characters.length === 0) {\n\t\t\tcharacters = [\"Jerry\", \"Elaine\", \"Kramer\"];\n\t\t}\n\t});\n\t\n\tfunction addGeorge() {\n\t\tcharacters.push(\"George\");\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// Hardcoded initial characters array\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\t// Derived state for the {#each} block\n\tlet listCharacters = $derived(characters);\n\t\n\t// Function to add George\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, \"George\"];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each listCharacters as character (character)}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, \"George\"];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\t$effect(() => {\n\t\tconsole.log(\"Characters updated:\", characters);\n\t});\n\t\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, \"George\"];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>\n\t\tAdd George\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/routes/seinfeld/+page.svelte\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, \"George\"];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as char}\n\t\t\t<li data-testid=\"character\">{char}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// Declare a reactive state variable for the characters array\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\t// Function to add George to the array\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, \"George\"];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as name}\n\t\t\t<li data-testid=\"character\">{name}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>\n\t\tAdd George\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\t$effect(() => {\n\t\tconsole.log(\"Characters updated:\", characters);\n\t});\n\t\n\tfunction addGeorge() {\n\t\tcharacters.push(\"George\");\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as char}\n\t\t\t<li data-testid=\"character\">{char}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>\n\t\tAdd George\n\t</button>\n</div>\n\n<style>\n\tul {\n\t\tlist-style-type: none;\n\t\tpadding: 0;\n\t}\n\t\n\tli {\n\t\tpadding: 8px;\n\t\tborder-bottom: 1px solid #ccc;\n\t}\n\t\n\tbutton {\n\t\tmargin-top: 10px;\n\t\tpadding: 8px 16px;\n\t\tcursor: pointer;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\t\n\tfunction addGeorge() {\n\t\tcharacters = [...characters, \"George\"];\n\t}\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 7,
    "pass1": 0.7,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\t\n\t// Use $effect to compute doubled value reactively\n\tlet doubled = $state(0);\n\t\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\t\n\tlet doubled = $effect(() => {\n\t\treturn number * 2;\n\t});\n\t\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:7:15 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\t\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\t\n\tlet doubled = $effect(() => {\n\t\treturn number * 2;\n\t});\n\t\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:15 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\n\t$effect(() => {\n\t\tconsole.log(`Number is now ${number}, doubled is ${doubled}`);\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\t\n\tlet doubled = $effect(() => {\n\t\treturn number * 2;\n\t});\n\t\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:15 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n\t\n\tfunction increment() {\n\t\tnumber++;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = \"World\", color = \"#0070f3\" } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\" style=\"color: {color}; margin: 1rem;\">\n\tHello, {name}!\n</div>\n\n<style>\n\t.greeting {\n\t\ttext-align: center;\n\t}\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">$state(\"Hello, World!\")</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n\tlet { name = \"World\" } = $props();\n\t\n\t$effect(() => {\n\t\tconsole.log(`Hello component mounted with name: ${name}`);\n\t\treturn () => {\n\t\t\tconsole.log(`Hello component unmounted`);\n\t\t};\n\t});\n</script>\n\n<div style=\"display: flex; justify-content: center; align-items: center; height: 100vh; margin: 2rem;\">\n\t<div data-testid=\"greeting\" class=\"greeting\" style=\"color: blue;\">\n\t\tHello, {name}!\n\t</div>\n</div>\n\n<style>\n\t.greeting {\n\t\tfont-size: 1.5rem;\n\t\tfont-weight: bold;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<div class=\"greeting\" style=\"color: blue; text-align: center; margin: 20px auto; max-width: 600px;\">\n\tHello, World!\n</div>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n      \u001b[33mstyle\u001b[39m=\u001b[32m\"color: blue; text-align: center; margin: 20px auto; max-width: 600px;\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[0mHello, World!\u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n      \u001b[33mstyle\u001b[39m=\u001b[32m\"color: blue; text-align: center; margin: 20px auto; max-width: 600px;\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[0mHello, World!\u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px auto;\n\t\tpadding: 10px 20px;\n\t\tfont-size: 1.2em;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet greeting = $derived(\"Hello, World!\");\n</script>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px;\n\t}\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">{greeting}</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet greeting = $state(\"Hello, World!\");\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\t{greeting}\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 1em;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<div class=\"greeting\" data-testid=\"greeting\">Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 20px;\n\t}\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\timport { inspect } from 'svelte/inspector';\n\t\n\tlet text = $state('Hello world');\n\t\n\t// Basic $inspect to log the input value\n\t$inspect(text);\n\t\n\t// $inspect(...).with to track updates with a custom callback\n\t$inspect(text).with((value, prev) => {\n\t\tconsole.log(`Input changed from \"${prev}\" to \"${value}\"`);\n\t});\n\t\n\t// $inspect.trace() inside an effect that runs when the input changes\n\t$effect(() => {\n\t\tconst unsubscribe = inspect(text).trace();\n\t\treturn () => unsubscribe();\n\t});\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\" \n\t\ttype=\"text\" \n\t\tbind:value={text} \n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "\"./inspector\" is not exported under the conditions [\"browser\", \"node\", \"development\", \"svelte\", \"svelte\", \"svelte\", \"import\"] from package /Users/k/Documents/GitHub/svelte-bench/node_modules/svelte (see exports field in /Users/k/Documents/GitHub/svelte-bench/node_modules/svelte/package.json)"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Use $inspect to log the input value\n\t$inspect(text);\n\n\t// Use $inspect(...).with to track updates with a custom callback\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Input updated via with:', value);\n\t});\n\n\t// Use $inspect.trace() inside an effect that runs when the input changes\n\t$effect(() => {\n\t\t$inspect.trace();\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state(\"Hello world\");\n\n\t$inspect(text);\n\t$inspect(text).with((value, prev) => {\n\t\tconsole.log(`Text changed from \"${prev}\" to \"${value}\"`);\n\t});\n\n\t$effect(() => {\n\t\tconst handle = $inspect(trace => {\n\t\t\tconsole.log(`Trace: ${trace}`);\n\t\t});\n\t\treturn () => handle.unsubscribe();\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot convert object to primitive value"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t// Log the input value using $inspect\n\t$inspect(text);\n\n\t// Log with custom callback using $inspect.with\n\t$inspect(text).with(() => {\n\t\tconsole.log(`Input updated to: \"${text}\"`);\n\t});\n\n\t// Log changes with trace in an effect\n\t$effect(() => {\n\t\tconst subscription = $inspect.trace(() => {\n\t\t\tconsole.log(`Text changed to: \"${text}\"`);\n\t\t});\n\t\treturn () => subscription.unsubscribe();\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t\tplaceholder=\"Type something...\"\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:23 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\timport { inspect } from 'svelte/state';\n\t\n\tlet text = $state('Hello world');\n\t\n\t// Basic inspection\n\t$inspect(text);\n\t\n\t// Inspection with custom callback\n\t$inspect(text).with((value, event) => {\n\t\tconsole.log(`Text changed to: \"${value}\"`);\n\t});\n\t\n\t// Inspection trace\n\t$effect(() => {\n\t\tconst trace = $inspect(trace).with((value, event) => {\n\t\t\tconsole.log(`Trace: ${event.type} - \"${value}\"`);\n\t\t});\n\t});\n\t\n\t// Character count derived from text length\n\tconst charCount = $derived(text.length);\n</script>\n\n<div>\n\t<input \n\t\tdata-testid=\"text-input\" \n\t\ttype=\"text\" \n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "\"./state\" is not exported under the conditions [\"browser\", \"node\", \"development\", \"svelte\", \"svelte\", \"svelte\", \"import\"] from package /Users/k/Documents/GitHub/svelte-bench/node_modules/svelte (see exports field in /Users/k/Documents/GitHub/svelte-bench/node_modules/svelte/package.json)"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\timport { onMount } from 'svelte';\n\n\tlet text = $state('Hello world');\n\n\t// Basic $inspect to log the input value\n\t$inspect(text);\n\n\t// $inspect(...).with() to track updates with a custom callback\n\t$inspect(text).with((value, old) => {\n\t\tif (value !== old) {\n\t\t\tconsole.log(`Input changed from \"${old}\" to \"${value}\"`);\n\t\t}\n\t});\n\n\t// $inspect.trace() to log changes in a trace-like format\n\t$effect(() => {\n\t\t$inspect(trace)(text);\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t\tplaceholder=\"Type something...\"\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "inspect(...) is not a function\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "inspect(...) is not a function\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "inspect(...) is not a function\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "inspect(...) is not a function\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value, previous) => {\n\t\tif (value !== previous) {\n\t\t\tconsole.log(`Text changed from \"${previous}\" to \"${value}\"`);\n\t\t}\n\t});\n\n\t$effect(() => {\n\t\tconst trace = $inspect.trace();\n\t\tconsole.log('Effect ran with trace:', trace);\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t\tplaceholder=\"Type something...\"\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:16:16 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet text = $state('Hello world');\n\n\t// Basic $inspect to log the input value\n\t$inspect(text);\n\n\t// Custom callback with $inspect(.with())\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Text value changed via .with() callback:', value);\n\t});\n\n\t// $inspect.trace() in an effect\n\t$effect(() => {\n\t\t$inspect(text).trace();\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state(\"Hello world\");\n\n\t// Basic $inspect to log the input value\n\t$inspect(text);\n\n\t// $inspect with custom callback to track updates\n\t$inspect(text).with((value, previous) => {\n\t\tif (value !== previous) {\n\t\t\tconsole.log(`Input changed from \"${previous}\" to \"${value}\"`);\n\t\t}\n\t});\n\n\t// Effect with $inspect.trace() to log changes\n\t$effect(() => {\n\t\tconst unsubscribe = $inspect(text).trace();\n\t\treturn () => unsubscribe();\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t\tplaceholder=\"Type something...\"\n\t/>\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "Cannot read properties of undefined (reading 'trace')\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\timport { inspect } from 'svelte';\n\n\tlet text = $state('Hello world');\n\n\t// Basic $inspect to log the input value\n\t$inspect(text);\n\n\t// $inspect with custom callback to track updates\n\t$inspect(text).with((value, prev) => {\n\t\tconsole.log(`Input changed from \"${prev}\" to \"${value}\"`);\n\t});\n\n\t// $inspect.trace() inside an effect to log changes\n\t$effect(() => {\n\t\t$inspect.trace();\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<p data-testid=\"text-value\">\n\t\tCurrent text: \"{text}\"\n\t</p>\n\t<p data-testid=\"char-count\">\n\t\tCharacter count: {text.length}\n\t</p>\n</div>",
        "success": false,
        "errors": [
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n"
        ]
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet countState = $state(count);\n\n\tfunction increment() {\n\t\tcountState++;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countState}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is {showDetails ? \"true\" : \"false\"}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet countState = $state(count);\n\n\tfunction increment() {\n\t\tcountState++;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countState}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet localCount = $state(count);\n\n\tfunction increment() {\n\t\tlocalCount++;\n\t}\n\n\tconst details = $derived(showDetails ? (\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {localCount}</p>\n\t\t\t<p>ShowDetails is {showDetails ? \"true\" : \"false\"}</p>\n\t\t</div>\n\t) : null);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {localCount}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>\n\t\tIncrement\n\t</button>\n\n\t{#if details}\n\t\t{details}\n\t{/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet localCount = $state(count);\n\n\tfunction increment() {\n\t\tlocalCount++;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {localCount}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {localCount}</p>\n\t\t\t<p>ShowDetails is {showDetails ? \"true\" : \"false\"}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\t\n\tlet countState = $state(count);\n\t\n\tfunction increment() {\n\t\tcountState++;\n\t}\n\t\n\tlet derivedCount = $derived(countState);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {derivedCount}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\t\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is {showDetails ? \"true\" : \"false\"}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet countState = $state(count);\n\n\tfunction increment() {\n\t\tcountState++;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countState}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n// src/lib/PropsDemo.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\t\n\tlet countState = $state(count);\n\t\n\tfunction increment() {\n\t\tcountState += 1;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countState}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\t\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\t\n\tlet localCount = $state(count);\n\t\n\tfunction increment() {\n\t\tlocalCount++;\n\t}\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n<p data-testid=\"count-display\">Count: {localCount}</p>\n<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n{#if showDetails}\n\t<div data-testid=\"details\">\n\t\t<p>Name is {name}</p>\n\t\t<p>Count is {localCount}</p>\n\t\t<p>ShowDetails is {showDetails ? 'true' : 'false'}</p>\n\t</div>\n{/if}",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet localCount = $state(count);\n\t\n\tlet increment = () => {\n\t\tlocalCount++;\n\t};\n\t\n\tlet detailsVisible = $derived(showDetails);\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n<p data-testid=\"count-display\">Count: {localCount}</p>\n<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n{#if detailsVisible}\n\t<div data-testid=\"details\">\n\t\t<p>Name is {name}</p>\n\t\t<p>Count is {localCount}</p>\n\t\t<p>ShowDetails is {showDetails ? \"true\" : \"false\"}</p>\n\t</div>\n{/if}",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet { name = \"World\", count = 0, showDetails = false } = $props();\n\tlet countState = $state(count);\n\n\tfunction increment() {\n\t\tcountState++;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countState}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countState}</p>\n\t\t\t<p>ShowDetails is {showDetails ? \"true\" : \"false\"}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "hf.co/rockypod/svelte-coder:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 5,
    "pass1": 0.5,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// Hardcoded array of book titles\n\tconst books = [\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t];\n\n\t// Snippet definition\n\t{#snippet title(name)}\n\t\t<span data-testid=\"book-title\">{name}</span>\n\t{/snippet}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{#snippet title(book)}\n\t\t\t\t<span data-testid=\"book-title\">{book}</span>\n\t\t\t{/snippet}\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\t\n\t/** @type {Snippet<[string]>} */\n\tlet title = ({ title }) => {\n\t\treturn <span data-testid=\"book-title\">{title}</span>;\n\t};\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:8:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>\n\n{#snippet title(book)}\n\t<span data-testid=\"book-title\">{book}</span>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t// Hardcoded array of book titles\n\tconst books = $state([\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]);\n\t\n\t// Snippet definition\n\tfunction title({ title }) {\n\t\treturn <span data-testid=\"book-title\">{title}</span>;\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title({ title: book })}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:9:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\t\n\t{#snippet title(title)}\n\t\t<span data-testid=\"book-title\">{title}</span>\n\t{/snippet}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:6:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\n\t\t{ name: \"The Lord of the Rings\" },\n\t\t{ name: \"To Kill a Mockingbird\" },\n\t\t{ name: \"1984\" }\n\t];\n</script>\n\n<ul>\n\t{#snippet title(name)}\n\t\t<span data-testid=\"book-title\">{name}</span>\n\t{/snippet}\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book.name)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n// src/routes/+page.svelte\n<script>\n\tlet books = $state([\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"]);\n</script>\n\n<main>\n\t<ul>\n\t\t{#snippet title(title)}\n\t\t\t<span data-testid=\"book-title\">{title}</span>\n\t\t{/snippet}\n\n\t\t{#each books as book}\n\t\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t\t{/each}\n\t</ul>\n</main>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{#snippet title(book)}\n\t\t\t\t<span data-testid=\"book-title\">{book}</span>\n\t\t\t{/snippet}\n\t\t\t{@render title()}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  The Lord of the Rings\u001b[39m\nReceived:\n\u001b[31m\u001b[39m"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(title)}\n\t<span data-testid=\"book-title\">{title}</span>\n{/snippet}\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T09:07:55.504Z",
    "tps": 8.12,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 412,
      "evalDurationNs": 50748850000,
      "promptEvalCount": 281,
      "promptEvalDurationNs": 3555589000,
      "promptTps": 79.03,
      "loadDurationNs": 1517897,
      "measuredAt": "2026-09-21T13:25:41.388Z"
    }
  }
]