[
  {
    "testName": "counter",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" on:click={() => count -= 1}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" on:click={() => count += 1}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 1rem;\n\t}\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" on:click={() => count -= 1}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" on:click={() => count += 1}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 1rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button type=\"button\" data-testid=\"decrement-button\" on:click={() => count -= 1}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button type=\"button\" data-testid=\"increment-button\" on:click={() => count += 1}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button type=\"button\" data-testid=\"decrement-button\" on:click={() => count -= 1}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button type=\"button\" data-testid=\"increment-button\" on:click={() => count += 1}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: inline-flex;\n\t\talign-items: center;\n\t\tgap: 1rem;\n\t}\n\n\tbutton {\n\t\twidth: 2.5rem;\n\t\theight: 2.5rem;\n\t\tfont-size: 1.25rem;\n\t\tcursor: pointer;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button\n\t\ttype=\"button\"\n\t\tdata-testid=\"decrement-button\"\n\t\taria-label=\"Decrease count\"\n\t\tonclick={() => count -= 1}\n\t>\n\t\t-\n\t</button>\n\n\t<span data-testid=\"count-value\">{count}</span>\n\n\t<button\n\t\ttype=\"button\"\n\t\tdata-testid=\"increment-button\"\n\t\taria-label=\"Increase count\"\n\t\tonclick={() => count += 1}\n\t>\n\t\t+\n\t</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 1rem;\n\t\tfont-size: 1.5rem;\n\t}\n\n\tbutton {\n\t\twidth: 2.5rem;\n\t\theight: 2.5rem;\n\t\tfont-size: 1.25rem;\n\t\tcursor: pointer;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n\tlet count = $state(0);\n\n\tfunction increment() {\n\t\tcount += 1;\n\t}\n\n\tfunction decrement() {\n\t\tcount -= 1;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button type=\"button\" data-testid=\"decrement-button\" aria-label=\"Decrease count\" onclick={decrement}>\n\t\t-\n\t</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button type=\"button\" data-testid=\"increment-button\" aria-label=\"Increase count\" onclick={increment}>\n\t\t+\n\t</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 1rem;\n\t}\n\n\tbutton {\n\t\tpadding: 0.5rem 1rem;\n\t\tborder: 1px solid #0070f3;\n\t\tborder-radius: 0.25rem;\n\t\tbackground: #fff;\n\t\tcursor: pointer;\n\t}\n\n\tbutton:hover {\n\t\tbackground: #eaf6ff;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" onclick={() => (count -= 1)}>−</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" onclick={() => (count += 1)}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button type=\"button\" data-testid=\"decrement-button\" onclick={() => count -= 1} aria-label=\"Decrease count\">-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button type=\"button\" data-testid=\"increment-button\" onclick={() => count += 1} aria-label=\"Increase count\">+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 1rem;\n\t}\n\n\tbutton {\n\t\tpadding: 0.5rem 1rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button type=\"button\" data-testid=\"decrement-button\" onclick={() => count -= 1}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button type=\"button\" data-testid=\"increment-button\" onclick={() => count += 1}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\tdisplay: flex;\n\t\talign-items: center;\n\t\tgap: 0.75rem;\n\t}\n\n\tbutton {\n\t\twidth: 2.5rem;\n\t\theight: 2.5rem;\n\t\tborder: 1px solid #2563eb;\n\t\tborder-radius: 0.375rem;\n\t\tbackground: #2563eb;\n\t\tcolor: white;\n\t\tfont-size: 1.25rem;\n\t\tcursor: pointer;\n\t}\n\n\tbutton:hover {\n\t\tbackground: #1d4ed8;\n\t}\n\n\tspan {\n\t\tmin-width: 2rem;\n\t\tfont-size: 1.25rem;\n\t\tfont-weight: 600;\n\t\ttext-align: center;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n\t<button type=\"button\" data-testid=\"decrement-button\" onclick={() => count -= 1}>−</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button type=\"button\" data-testid=\"increment-button\" onclick={() => count += 1}>+</button>\n</div>\n\n<style>\n\t.counter {\n\t\talign-items: center;\n\t\tdisplay: flex;\n\t\tfont-size: 1.5rem;\n\t\tgap: 1rem;\n\t}\n\n\tbutton {\n\t\tfont-size: 1rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "derived",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<main>\n\t<label for=\"number-input\">Number</label>\n\t<input\n\t\tid=\"number-input\"\n\t\ttype=\"number\"\n\t\tmin=\"0\"\n\t\tstep=\"1\"\n\t\tbind:value={number}\n\t/>\n\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\n\t<button\n\t\ttype=\"button\"\n\t\tdata-testid=\"increment-button\"\n\t\tonclick={() => number += 1}\n\t>\n\t\tIncrement\n\t</button>\n</main>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tconst doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button type=\"button\" data-testid=\"increment-button\" on:click={() => number += 1}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button type=\"button\" data-testid=\"increment-button\" onclick={() => (number += 1)}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $derived(number * 2);\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "derived-by",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst wordCount = $derived.by(() => {\n\t\treturn text.trim() ? text.trim().split(/\\s+/).length : 0;\n\t});\n\n\tconst charCount = $derived.by(() => text.length);\n\n\tconst isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<input\n\tdata-testid=\"text-input\"\n\ttype=\"text\"\n\tbind:value={text}\n\taria-label=\"Text input\"\n/>\n<button data-testid=\"clear-button\" on:click={() => (text = '')}>Clear</button>\n\n<p data-testid=\"word-count\">Words: {wordCount}</p>\n<p data-testid=\"char-count\">Characters: {charCount}</p>\n<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst wordCount = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/);\n\t\treturn text.trim() ? words.length : 0;\n\t});\n\n\tconst charCount = $derived.by(() => text.length);\n\n\tconst lengthStatus = $derived.by(() =>\n\t\twordCount > 15 ? 'Status: Long text' : 'Status: Short text'\n\t);\n</script>\n\n<input\n\tdata-testid=\"text-input\"\n\ttype=\"text\"\n\tbind:value={$text}\n/>\n\n<button data-testid=\"clear-button\" on:click={() => (text = '')}>\n\tClear\n</button>\n\n<p data-testid=\"word-count\">Words: {wordCount}</p>\n<p data-testid=\"char-count\">Characters: {charCount}</p>\n<p data-testid=\"length-indicator\">{lengthStatus}</p>",
        "success": false,
        "errors": [
          "store_invalid_shape\n`text` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n",
          "store_invalid_shape\n`text` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n",
          "store_invalid_shape\n`text` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst wordCount = $derived.by(() => {\n\t\tconst words = text.trim().split(/\\s+/).filter(Boolean);\n\t\treturn words.length;\n\t});\n\n\tconst charCount = $derived.by(() => text.length);\n\n\tconst isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<button data-testid=\"clear-button\" on:click={() => (text = '')}>Clear</button>\n\n\t<div>\n\t\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t\t<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n\t</div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const wordCount = $derived.by(() => {\n    return text.trim() ? text.trim().split(/\\s+/).length : 0;\n  });\n\n  const charCount = $derived.by(() => text.length);\n\n  const isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    aria-label=\"Text input\"\n  />\n  <button type=\"button\" data-testid=\"clear-button\" disabled={!text} on:click={() => (text = '')}>\n    Clear\n  </button>\n\n  <p data-testid=\"word-count\">Words: {wordCount}</p>\n  <p data-testid=\"char-count\">Characters: {charCount}</p>\n  <p data-testid=\"length-indicator\">\n    Status: {isLong ? 'Long text' : 'Short text'}\n  </p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst wordCount = $derived.by(() =>\n\t\ttext.trim() ? text.trim().split(/\\s+/).length : 0\n\t);\n\n\tconst charCount = $derived.by(() => text.length);\n\n\tconst isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n\t<button data-testid=\"clear-button\" on:click={() => (text = '')}>\n\t\tClear\n\t</button>\n\n\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t<p data-testid=\"length-indicator\">\n\t\tStatus: {isLong ? 'Long text' : 'Short text'}\n\t</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tlet stats = $derived.by(() => {\n\t\tconst words = text.trim() ? text.trim().split(/\\s+/).length : 0;\n\t\tconst characters = text.length;\n\t\tconst isLong = words > 15;\n\n\t\treturn { words, characters, isLong };\n\t});\n</script>\n\n<div>\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t\tplaceholder=\"Enter some text\"\n\t/>\n\t<button data-testid=\"clear-button\" on:click={() => (text = '')}>\n\t\tClear\n\t</button>\n\n\t<p data-testid=\"word-count\">Words: {stats.words}</p>\n\t<p data-testid=\"char-count\">Characters: {stats.characters}</p>\n\t<p data-testid=\"length-indicator\">\n\t\tStatus: {stats.isLong ? 'Long text' : 'Short text'}\n\t</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tlet wordCount = $derived.by(() => {\n\t\tconst trimmed = text.trim();\n\t\treturn trimmed === '' ? 0 : trimmed.split(/\\s+/).length;\n\t});\n\n\tlet charCount = $derived.by(() => text.length);\n\n\tlet isLong = $derived.by(() => wordCount > 15);\n\n\tfunction clearText() {\n\t\ttext = '';\n\t}\n</script>\n\n<label>\n\tEnter text\n\t<input\n\t\tdata-testid=\"text-input\"\n\t\ttype=\"text\"\n\t\tbind:value={text}\n\t/>\n</label>\n\n<button type=\"button\" data-testid=\"clear-button\" on:click={clearText}>\n\tClear\n</button>\n\n<p data-testid=\"word-count\">Words: {wordCount}</p>\n<p data-testid=\"char-count\">Characters: {charCount}</p>\n<p data-testid=\"length-indicator\">\n\tStatus: {isLong ? 'Long text' : 'Short text'}\n</p>\n\n<style>\n\tlabel {\n\t\tdisplay: grid;\n\t\tgap: 0.5rem;\n\t}\n\n\tinput {\n\t\tbox-sizing: border-box;\n\t\tpadding: 0.75rem;\n\t\tfont: inherit;\n\t}\n\n\tbutton {\n\t\tpadding: 0.5rem 1rem;\n\t\tfont: inherit;\n\t}\n\n\tp {\n\t\tfont: inherit;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst wordCount = $derived.by(() => {\n\t\treturn text.trim().split(/\\s+/).filter(Boolean).length;\n\t});\n\n\tconst charCount = $derived.by(() => text.length);\n\n\tconst isLong = $derived.by(() => wordCount > 15);\n\n\tfunction clearText() {\n\t\ttext = '';\n\t}\n</script>\n\n<input\n\tdata-testid=\"text-input\"\n\ttype=\"text\"\n\tbind:value={text}\n\taria-label=\"Text input\"\n/>\n<button type=\"button\" data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n\n<p data-testid=\"word-count\" aria-live=\"polite\">Words: {wordCount}</p>\n<p data-testid=\"char-count\" aria-live=\"polite\">Characters: {charCount}</p>\n<p data-testid=\"length-indicator\" aria-live=\"polite\">\n\tStatus: {isLong ? 'Long text' : 'Short text'}\n</p>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst counts = $derived.by(() => {\n\t\tconst wordCount = text.trim() ? text.trim().split(/\\s+/).length : 0;\n\n\t\treturn {\n\t\t\twordCount,\n\t\t\tcharCount: text.length,\n\t\t\tisLong: wordCount > 15\n\t\t};\n\t});\n</script>\n\n<label for=\"text-input\">Enter text</label>\n<input\n\tid=\"text-input\"\n\tdata-testid=\"text-input\"\n\ttype=\"text\"\n\tbind:value={text}\n/>\n\n<button data-testid=\"clear-button\" on:click={() => (text = '')}>\n\tClear\n</button>\n\n<p data-testid=\"word-count\">Words: {counts.wordCount}</p>\n<p data-testid=\"char-count\">Characters: {counts.charCount}</p>\n<p data-testid=\"length-indicator\">\n\tStatus: {counts.isLong ? 'Long text' : 'Short text'}\n</p>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('');\n\n\tconst wordCount = $derived.by(() => {\n\t\treturn text.trim() === '' ? 0 : text.trim().split(/\\s+/).length;\n\t});\n\n\tconst charCount = $derived.by(() => text.length);\n\n\tconst isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<button data-testid=\"clear-button\" onclick={() => (text = '')}>Clear</button>\n\n\t<p data-testid=\"word-count\">Words: {wordCount}</p>\n\t<p data-testid=\"char-count\">Characters: {charCount}</p>\n\t<p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "each",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" on:click={() => characters.push('George')}>\n\t\tAdd George\n\t</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>\n\n<ul data-testid=\"characters-list\">\n\t{#each characters as character}\n\t\t<li data-testid=\"character\">{character}</li>\n\t{/each}\n</ul>\n\n<button data-testid=\"add-george-button\" onclick={() => characters = [...characters, 'George']}>\n\tAdd George\n</button>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\n\t<button data-testid=\"add-george-button\" onclick={() => characters.push('George')}>\n\t\tAdd George\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" on:click={() => characters.push('George')}>\n\t\tAdd George\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\n\tfunction addGeorge() {\n\t\tif (!characters.includes('George')) {\n\t\t\tcharacters = [...characters, 'George'];\n\t\t}\n\t}\n</script>\n\n<ul data-testid=\"characters-list\">\n\t{#each characters as character (character)}\n\t\t<li data-testid=\"character\">{character}</li>\n\t{/each}\n</ul>\n\n<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>\n\n<ul data-testid=\"characters-list\">\n\t{#each characters as character}\n\t\t<li data-testid=\"character\">{character}</li>\n\t{/each}\n</ul>\n\n<button data-testid=\"add-george-button\" onclick={() => characters.push('George')}>\n\tAdd George\n</button>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet count = $state(0);\n\tconst numbers = [1, 2, 3];\n\n\tfunction increment() {\n\t\tif (count < 10) count += 1;\n\t}\n\n\tfunction decrement() {\n\t\tif (count > 0) count -= 1;\n\t}\n</script>\n\n<div>\n\t<button on:click={decrement} disabled={count === 0}>−</button>\n\t<span>{count}</span>\n\t<button on:click={increment} disabled={count === 10}>\n\t\tAdd {count < 10 ? 1 : 10}\n\t</button>\n\n\t<ul data-testid=\"characters-list\">\n\t\t{#each numbers as number}\n\t\t\t<li data-testid=\"character\">{number}</li>\n\t\t{/each}\n\t</ul>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Jerry\u001b[39m\nReceived:\n\u001b[31m  1\u001b[39m",
          "Unable to find an element by: [data-testid=\"add-george-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div>\u001b[39m\n      \u001b[36m<button\u001b[39m\n        \u001b[33mdisabled\u001b[39m=\u001b[32m\"\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m−\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<span>\u001b[39m\n        \u001b[0m0\u001b[0m\n      \u001b[36m</span>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0mAdd 1\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<ul\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"characters-list\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[36m<li\u001b[39m\n          \u001b[33mdata-testid\u001b[39m=\u001b[32m\"character\"\u001b[39m\n        \u001b[36m>\u001b[39m\n          \u001b[0m1\u001b[0m\n        \u001b[36m</li>\u001b[39m\n        \u001b[36m<li\u001b[39m\n          \u001b[33mdata-testid\u001b[39m=\u001b[32m\"character\"\u001b[39m\n        \u001b[36m>\u001b[39m\n          \u001b[0m2\u001b[0m\n        \u001b[36m</li>\u001b[39m\n        \u001b[36m<li\u001b[39m\n          \u001b[33mdata-testid\u001b[39m=\u001b[32m\"character\"\u001b[39m\n        \u001b[36m>\u001b[39m\n          \u001b[0m3\u001b[0m\n        \u001b[36m</li>\u001b[39m\n        \u001b[0m\u001b[0m\n      \u001b[36m</ul>\u001b[39m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<ul data-testid=\"characters-list\">\n\t{#each characters as character}\n\t\t<li data-testid=\"character\">{character}</li>\n\t{/each}\n</ul>\n\n<button type=\"button\" data-testid=\"add-george-button\" onclick={() => characters.push(\"George\")}>\n\tAdd George\n</button>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>\n\n<div>\n\t<ul data-testid=\"characters-list\">\n\t\t{#each characters as character}\n\t\t\t<li data-testid=\"character\">{character}</li>\n\t\t{/each}\n\t</ul>\n\t<button data-testid=\"add-george-button\" onclick={() => (characters = [...characters, 'George'])}>\n\t\tAdd George\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\n</script>\n\n<ul data-testid=\"characters-list\">\n\t{#each characters as character (character)}\n\t\t<li data-testid=\"character\">{character}</li>\n\t{/each}\n</ul>\n\n<button data-testid=\"add-george-button\" on:click={() => characters.push('George')}>\n\tAdd George\n</button>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "effect",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => number += 1}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = 0;\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 10\u001b[39m\nReceived:\n\u001b[31m  Doubled: 0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 12\u001b[39m\nReceived:\n\u001b[31m  Doubled: 10\u001b[39m"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => (number += 1)}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button data-testid=\"increment-button\" onclick={() => (number += 1)}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(10);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n\n\tfunction increment() {\n\t\tnumber += 1;\n\t}\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button type=\"button\" data-testid=\"increment-button\" on:click={increment}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet number = $state(5);\n\tlet doubled = $state(0);\n\n\t$effect(() => {\n\t\tdoubled = number * 2;\n\t});\n</script>\n\n<div>\n\t<p data-testid=\"number-value\">Number: {number}</p>\n\t<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n\t<button type=\"button\" data-testid=\"increment-button\" onclick={() => (number += 1)}>\n\t\tIncrement\n\t</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "hello-world",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<div class=\"page\">\n  <div class=\"greeting\" data-testid=\"greeting\">Hello, World!</div>\n</div>\n\n<style>\n  .page {\n    min-height: 100vh;\n    display: grid;\n    place-items: center;\n  }\n\n  .greeting {\n    margin: 1rem;\n    padding: 1rem;\n    color: blue;\n    font-size: 2rem;\n    font-weight: 600;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { message = 'Hello, World!' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\t{message}\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 1rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const greeting = 'Hello, World!';\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  {greeting}\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const greeting = 'Hello, World!';\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  {greeting}\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { class: greetingClass = 'greeting' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class={greetingClass}>Hello, World!</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 1rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    font-size: 2rem;\n    text-align: center;\n    margin: 2rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n\tlet { message = 'Hello, World!' } = $props();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n\t{message}\n</div>\n\n<style>\n\t.greeting {\n\t\tcolor: blue;\n\t\ttext-align: center;\n\t\tmargin: 2rem;\n\t}\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "inspect",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 5,
    "pass1": 0.5,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Updated text:', value);\n\t});\n\n\t$effect(() => {\n\t\t$inspect.trace(text);\n\t\tconsole.log('Input changed:', text);\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Custom update:', value);\n\t});\n\n\t$effect(() => {\n\t\tconsole.log('Effect ran for:', text);\n\t\t$inspect.trace();\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/Component.svelte:14:2 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Custom input inspection:', value);\n\t});\n\n\t$effect(() => {\n\t\t$inspect.trace();\n\t\tconsole.log('Input changed:', text);\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value, previous) => {\n\t\tconsole.log('Text changed:', value, 'Previous:', previous);\n\t});\n\n\t$effect(() => {\n\t\t$inspect.trace().with((value, previous) => {\n\t\t\tconsole.log('Traced text:', value, 'Previous:', previous);\n\t\t});\n\t});\n</script>\n\n<input\n\tdata-testid=\"text-input\"\n\ttype=\"text\"\n\tbind:value={text}\n/>\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/Component.svelte:13:2 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value, previous) => {\n\t\tconsole.log(`Input changed from ${previous} to ${value}`);\n\t});\n\n\t$effect(() => {\n\t\tconst current = text;\n\t\t$inspect.trace();\n\t\tconsole.log(`Effect ran for ${current}`);\n\t});\n</script>\n\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/Component.svelte:14:2 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value) => {\n\t\tconsole.log('Custom inspection:', value);\n\t});\n\n\t$effect(() => {\n\t\t$inspect.trace(text);\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value, previous) => {\n\t\tconsole.log(`Input changed from \"${previous}\" to \"${value}\"`);\n\t});\n\n\teffect(() => {\n\t\t$inspect.trace();\n\t\tconsole.log(`Traced input: ${text}`);\n\t});\n</script>\n\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>",
        "success": false,
        "errors": [
          "effect is not defined\n\n\tin <unknown>\n",
          "effect is not defined\n\n\tin <unknown>\n",
          "effect is not defined\n\n\tin <unknown>\n",
          "effect is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value, previous) => {\n\t\tconsole.log(`Text changed from \"${previous}\" to \"${value}\"`);\n\t});\n\n\t$effect(() => {\n\t\t$inspect.trace();\n\t\tvoid text.length;\n\t});\n</script>\n\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value, previous) => {\n\t\tconsole.log('Text updated:', { value, previous });\n\t});\n\n\t$effect(() => {\n\t\t$inspect.trace();\n\t\tconsole.log(text);\n\t});\n</script>\n\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nText updated: [object Ob…' to contain 'init'",
          "expected 'Hello world\\nText updated: [object Ob…' to contain 'init'",
          "expected 'Hello world\\nText updated: [object Ob…' to contain 'init'"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet text = $state('Hello world');\n\n\t$inspect(text);\n\n\t$inspect(text).with((value) => {\n\t\tconsole.log(`Custom callback: ${value}`);\n\t});\n\n\t$effect(() => {\n\t\t$inspect.trace();\n\t\tconsole.log(`Effect: ${text}`);\n\t});\n</script>\n\n<div>\n\t<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\t<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\t<p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "props",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\tlet localCount = $state(count);\n\n\tfunction increment() {\n\t\tlocalCount += 1;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {localCount}</p>\n\t<button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {localCount}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\tlet displayCount = $state(count);\n\n\tfunction incrementCount() {\n\t\tdisplayCount += 1;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {displayCount}</p>\n\t<button type=\"button\" data-testid=\"increment-button\" on:click={incrementCount}>\n\t\tIncrement\n\t</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {displayCount}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet {\n\t\tname = 'World',\n\t\tcount: initialCount = 0,\n\t\tshowDetails = false\n\t} = $props();\n\n\tlet count = $state(initialCount);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button data-testid=\"increment-button\" onclick={() => count += 1}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count: initialCount = 0, showDetails = false } = $props();\n\tlet count = $state(initialCount);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button data-testid=\"increment-button\" onclick={() => (count += 1)}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\n\tlet localCount = $state(count);\n\n\t$effect(() => {\n\t\tlocalCount = count;\n\t});\n\n\tfunction incrementCount() {\n\t\tlocalCount += 1;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {localCount}</p>\n\t<button data-testid=\"increment-button\" onclick={incrementCount}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {localCount}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst { name = 'World', count: initialCount = 0, showDetails = false } = $props();\n\tlet count = $state(initialCount);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button data-testid=\"increment-button\" onclick={() => (count += 1)}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\tlet countValue = $state(count);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countValue}</p>\n\t<button data-testid=\"increment-button\" type=\"button\" onclick={() => (countValue += 1)}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countValue}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet {\n\t\tname = \"World\",\n\t\tcount: initialCount = 0,\n\t\tshowDetails = false\n\t} = $props();\n\n\tlet count = $state(initialCount);\n\n\tfunction increment() {\n\t\tcount += 1;\n\t}\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {count}</p>\n\t<button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {count}</p>\n\t\t\t<p>ShowDetails is {showDetails}</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet {\n\t\tname = 'World',\n\t\tcount: initialCount = 0,\n\t\tshowDetails = false\n\t} = $props();\n\n\tlet count = $state(initialCount);\n</script>\n\n<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n<p data-testid=\"count-display\">Count: {count}</p>\n<button type=\"button\" data-testid=\"increment-button\" onclick={() => (count += 1)}>\n\tIncrement\n</button>\n\n{#if showDetails}\n\t<div data-testid=\"details\">\n\t\t<p>Name is {name}</p>\n\t\t<p>Count is {count}</p>\n\t\t<p>ShowDetails is true</p>\n\t</div>\n{/if}",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet { name = 'World', count = 0, showDetails = false } = $props();\n\tlet countValue = $state(count);\n</script>\n\n<div>\n\t<h1 data-testid=\"name-display\">Hello, {name}!</h1>\n\t<p data-testid=\"count-display\">Count: {countValue}</p>\n\t<button data-testid=\"increment-button\" onclick={() => countValue += 1}>Increment</button>\n\n\t{#if showDetails}\n\t\t<div data-testid=\"details\">\n\t\t\t<p>Name is {name}</p>\n\t\t\t<p>Count is {countValue}</p>\n\t\t\t<p>ShowDetails is true</p>\n\t\t</div>\n\t{/if}\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  },
  {
    "testName": "snippets",
    "provider": "OpenRouter",
    "modelId": "nex-agi/nex-n2.5-mini:free",
    "numSamples": 10,
    "numCorrect": 6,
    "pass1": 0.5999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = [\n\t\t'The Lord of the Rings',\n\t\t'To Kill a Mockingbird',\n\t\t'1984'\n\t];\n</script>\n\n{#snippet title(book)}\n\t<span data-testid=\"book-title\">{book}</span>\n{/snippet}\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\n\t\t'The Lord of the Rings',\n\t\t'To Kill a Mockingbird',\n\t\t'1984'\n\t];\n\n\tlet title = snippet((title) => (\n\t\t<span data-testid=\"book-title\">{title}</span>\n\t));\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/Component.svelte:11:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(book)}\n\t<span data-testid=\"book-title\">{book}</span>\n{/snippet}\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = [\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t];\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>\n\n{#snippet title(book)}\n\t<span data-testid=\"book-title\">{book}</span>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = ['The Lord of the Rings', 'To Kill a Mockingbird', '1984'];\n\n\tsnippet title(title) {\n\t\t<span data-testid=\"book-title\">{title}</span>\n\t}\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/Component.svelte:6:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n\tlet title = snippet((bookTitle) => {\n\t\treturn `<span data-testid=\"book-title\">${bookTitle}</span>`;\n\t});\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "snippet is not defined\n\n\tin <unknown>\n",
          "snippet is not defined\n\n\tin <unknown>\n",
          "snippet is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = [\n\t\t\"The Lord of the Rings\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\"\n\t];\n</script>\n\n{#snippet title(book)}\n\t<span data-testid=\"book-title\">{book}</span>\n{/snippet}\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(book)}\n\t<span data-testid=\"book-title\">{book}</span>\n{/snippet}\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">\n\t\t\t{@render title(book)}\n\t\t</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tconst books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n{#snippet title(book)}\n\t<span data-testid=\"book-title\">{book}</span>\n{/snippet}\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n\tlet books = [\n\t\t'The Lord of the Rings',\n\t\t'To Kill a Mockingbird',\n\t\t'1984'\n\t];\n\n\tlet title = snippet((bookTitle) => {\n\t\treturn `<span data-testid=\"book-title\">${bookTitle}</span>`;\n\t});\n</script>\n\n<ul>\n\t{#each books as book}\n\t\t<li data-testid=\"book-item\">{@render title(book)}</li>\n\t{/each}\n</ul>",
        "success": false,
        "errors": [
          "snippet is not defined\n\n\tin <unknown>\n",
          "snippet is not defined\n\n\tin <unknown>\n",
          "snippet is not defined\n\n\tin <unknown>\n"
        ]
      }
    ],
    "timestamp": "2026-09-08T22:57:13.039Z"
  }
]