[
  {
    "testName": "counter",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { createCounter } from 'svelte';\n  \n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { createSignal } from 'svelte';\n\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { create } from 'svelte';\n  \n  const count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $: count = 0;\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/counter_sample4_95786aba413287b5/Component.svelte:4:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { create } from 'svelte';\n  \n  const count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={() => count -= 1}>\n    -\n  </button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={() => count += 1}>\n    +\n  </button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/counter_sample5_d3b098c26cbf558d/Component.svelte:10:57 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state } from 'svelte';\n  \n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/counter_sample6_541661f3faedf019/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { create } from 'svelte';\n\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { create } from 'svelte';\n  \n  const count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { create } from 'svelte';\n  \n  const count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "derived",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 7,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n  \n  const doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n<button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>",
        "success": false,
        "errors": [
          "Found multiple elements by: [data-testid=\"increment-button\"]\n\nHere are the matching elements:\n\nIgnored nodes: comments, script, style\n\u001b[36m<button\u001b[39m\n  \u001b[33mdata-testid\u001b[39m=\u001b[32m\"increment-button\"\u001b[39m\n\u001b[36m>\u001b[39m\n  \u001b[0mIncrement\u001b[0m\n\u001b[36m</button>\u001b[39m\n\nIgnored nodes: comments, script, style\n\u001b[36m<button\u001b[39m\n  \u001b[33mdata-testid\u001b[39m=\u001b[32m\"increment-button\"\u001b[39m\n\u001b[36m>\u001b[39m\n  \u001b[0mIncrement\u001b[0m\n\u001b[36m</button>\u001b[39m\n\n(If this is intentional, then use the `*AllBy*` variant of the query (like `queryAllByText`, `getAllByText`, or `findAllByText`)).\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div>\u001b[39m\n      \u001b[36m<p\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"number-value\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mNumber: 5\u001b[0m\n      \u001b[36m</p>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<p\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"doubled-value\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mDoubled: 10\u001b[0m\n      \u001b[36m</p>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"increment-button\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mIncrement\u001b[0m\n      \u001b[36m</button>\u001b[39m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m \u001b[0m\n    \u001b[36m<button\u001b[39m\n      \u001b[33mdata-testid\u001b[39m=\u001b[32m\"increment-button\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[0mIncrement\u001b[0m\n    \u001b[36m</button>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ],
        "temperature": 0
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n  const doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5$state;\n  \n  const doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/derived_sample4_8ebf5438f1a2f94a/Component.svelte:4:16 Identifier directly after number\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5$state;\n  \n  const doubled = $derived(number * 2);\n  \n  function increment() {\n    number += 1$state;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/derived_sample6_f0c8f35b0f11af77/Component.svelte:4:16 Identifier directly after number\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n  const doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = 5;\n\n  $: double = count * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {count}</p>\n  <p data-testid=\"doubled-value\">Doubled: {double}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/derived_sample8_7d4b72402800be9b/Component.svelte:6:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = 5$state\n  \n  const double = $derived(count * 2)\n  \n  function increment() {\n    count += 1\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {count}</p>\n  <p data-testid=\"doubled-value\">Doubled: {double}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/derived_sample9_bb17c1adc48e19e2/Component.svelte:4:15 Identifier directly after number\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "derived-by",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { derivedBy } from 'svelte/derived-by';\n\n  // Store the current text value\n  let text = '';\n\n  // Derive multiple values from the text state using $derived.by\n  const derivedValues = derivedBy('text', {\n    wordCount: () => text.split(/\\s+/).filter(Boolean).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derivedValues.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derivedValues.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derivedValues.isLong ? 'Long' : 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "Missing \"./derived-by\" specifier in \"svelte\" package"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = '';\n  \n  const derived = derivedBy('text', {\n    wordCount: () => text.split(/\\s+/).filter(Boolean).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" on:click={() => text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long' : 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "derivedBy is not defined\n\n\tin <unknown>\n",
          "derivedBy is not defined\n\n\tin <unknown>\n",
          "derivedBy is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { derivedBy } from 'svelte/derived-by';\n\n  // Store the current text value using $state\n  let text = '';\n\n  // Calculate derived values using $derived.by\n  const derived = derivedBy('text', {\n    wordCount: () => text.split(/\\s+/).filter(Boolean).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long' : 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "Missing \"./derived-by\" specifier in \"svelte\" package"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { derivedBy } from 'svelte/derived';\n\n  let text = '';\n\n  const by = derivedBy({\n    wordCount: () => {\n      if (!text) return 0;\n      return text.split(/\\s+/).filter(Boolean).length;\n    },\n    charCount: () => {\n      if (!text) return 0;\n      return text.length;\n    },\n    isLong: () => {\n      if (!text) return false;\n      return text.split(/\\s+/).filter(Boolean).length > 15;\n    }\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {by.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {by.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {if (by.isLong) 'Long' else 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/derived-by_sample3_19a2aa8a880ffcf9/Component.svelte:34:47 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // State variable for the text input\n  let text = '';\n\n  // Derived values calculated from the text state using $derived.by\n  const by = derivedBy(text, {\n    wordCount: () => text.split(/\\s+/).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).length > 15\n  });\n</script>\n\n<div>\n  <!-- Input field with test id -->\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  \n  <!-- Clear button with test id -->\n  <button data-testid=\"clear-button\">Clear</button>\n  \n  <!-- Derived values display -->\n  <div>\n    <p data-testid=\"word-count\">Words: {by.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {by.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {by.isLong ? 'Long' : 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "derivedBy is not defined\n\n\tin <unknown>\n",
          "derivedBy is not defined\n\n\tin <unknown>\n",
          "derivedBy is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Using $state to declare reactive state\n  $: text = '';\n\n  // Using $derived.by to compute multiple derived values from the same state\n  const by = derived(() => ({\n    wordCount: (text.split(/\\s+/)).filter(Boolean).length,\n    charCount: text.length,\n    isLong: (text.split(/\\s+/)).filter(Boolean).length > 15\n  }));\n</script>\n\n<div>\n  <!-- Input field with data-testid=\"text-input\" -->\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  \n  <!-- Clear button with data-testid=\"clear-button\" -->\n  <button data-testid=\"clear-button\">Clear</button>\n  \n  <!-- Container for derived values -->\n  <div>\n    <p data-testid=\"word-count\">Words: {by.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {by.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {if (by.isLong) 'Long' else 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/derived-by_sample5_388b9107a894e742/Component.svelte:30:47 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { derivedBy } from 'svelte/derived-by';\n\n  let text = '';\n\n  const derived = derivedBy('text', {\n    wordCount: () => text.split(/\\s+/).filter(Boolean).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input \n    bind:value={text} \n    type=\"text\" \n    data-testid=\"text-input\"\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long' : 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "Missing \"./derived-by\" specifier in \"svelte\" package"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { derivedBy } from 'svelte/derived-by';\n\n  // State variable for storing the text input value\n  let text = '';\n\n  // Derived values calculated from the text state using $derived.by\n  const derivedValues = derivedBy('text', {\n    wordCount: () => text.split(/\\s+/).filter(Boolean).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derivedValues.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derivedValues.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derivedValues.isLong ? 'Long' : 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "Missing \"./derived-by\" specifier in \"svelte\" package"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { derivedBy } from 'svelte/derived-by';\n\n  let text = '';\n\n  const derived = derivedBy('text', {\n    wordCount: () => text.split(/\\s+/).filter(Boolean).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {derived.isLong ? 'Long' : 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "Missing \"./derived-by\" specifier in \"svelte\" package"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = '';\n  \n  const by = derivedBy({\n    wordCount: () => text.split(/\\s+/).length,\n    charCount: () => text.length,\n    isLong: () => text.split(/\\s+/).length > 15\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {by.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {by.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {if (by.isLong) 'Long' else 'Short'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/derived-by_sample9_3e5b8e25f95b0ebd/Component.svelte:23:47 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "each",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<main>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  \n  <button data-testid=\"add-george-button\">Add George</button>\n</main>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {$#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={() => characters.push('George')}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/each_sample2_a5bca7f18398650c/Component.svelte:9:6 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<main>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  \n  <button data-testid=\"add-george-button\">Add George</button>\n</main>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {$#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/each_sample4_e706abfd7b07c5d2/Component.svelte:9:6 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  const addGeorge = () => {\n    characters.push('George');\n  };\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<main>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  \n  <button data-testid=\"add-george-button\">Add George</button>\n</main>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n      <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "effect",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $$state, $$effect } from 'svelte';\n\n  let number = $$state(5);\n\n  $$effect(() => {\n    const doubled = number * 2;\n    document.getElementById('doubled-value').textContent = `Doubled: ${doubled}`;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">{document.getElementById('doubled-value')?.textContent || ''}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/effect_sample0_aa65ca302d2b2188/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $$state, $$effect } from 'svelte';\n\n  let number = $$state(5);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {number * 2}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n$$effect(() => {\n  const doubled = number * 2;\n  document.getElementById('doubled-value').textContent = `Doubled: ${doubled}`;\n});",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/effect_sample1_5fb0cbaf2d67e3db/Component.svelte:17:2 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { create } from 'svelte';\n\n  let number = 5;\n\n  $effect(() => {\n    const doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "doubled is not defined\n\n\tin <unknown>\n",
          "doubled is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $$state(5);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {number * 2}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>\n\n$$effect(() => {\n  const doubled = number * 2;\n  document.getElementById('doubled-value').textContent = `Doubled: ${doubled}`;\n});",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/effect_sample3_74d91718621c592c/Component.svelte:19:2 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n\n  $effect(() => {\n    const doubled = number * 2;\n    return doubled;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "doubled is not defined\n\n\tin <unknown>\n",
          "doubled is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {number * 2}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>\n\n$$effect(() => {\n  const doubled = number * 2;\n  document.getElementById('doubled-value').textContent = 'Doubled: ' + doubled;\n});",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/effect_sample5_d3e8dc782364a471/Component.svelte:15:2 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n\n  // Calculate doubled value whenever number changes\n  $: doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/effect_sample6_1153e08c2cb6704d/Component.svelte:7:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n\n  $$effect(() => {\n    const doubled = number * 2;\n    const doubledEl = document.getElementById('doubled-value');\n    if (doubledEl) {\n      doubledEl.textContent = 'Doubled: ' + doubled;\n    }\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">{document.getElementById('doubled-value')?.textContent || ''}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/effect_sample7_dadb9dba1d0a35f0/Component.svelte:6:2 `$$effect` is an illegal variable name. To reference a global variable called `$$effect`, use `globalThis.$$effect`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = 5;\n\n  // $effect rune - calculates doubled value whenever number changes\n  $: doubled = number * 2;\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/effect_sample8_652262e6b9d62b3b/Component.svelte:7:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { create } from 'svelte';\n\n  let number = 5;\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Number: 6\u001b[39m\nReceived:\n\u001b[31m  Number: 5\u001b[39m"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "hello-world",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 9,
    "pass1": 0.9,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No reactive state needed for this static component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <span style=\"color: blue; margin: 10px;\">Hello, World!</span>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No reactive state needed for this static component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 8px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No reactive state needed for this static component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <p>Hello, World!</p>\n</div>\nWait, I need to add the centering and margin. Let me update the CSS classes to include those styles. In Svelte 5, I can use inline styles or CSS classes. The requirement says to add a CSS class \"greeting\" and make the text centered and have a small margin.\n\nLet me refine this:\n\n<script>\n  // No reactive state needed for this static component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <p style=\"color: blue; text-align: center; margin: 10px;\">Hello, World!</p>\n</div>\nActually, looking at the requirements more carefully:\n- The div itself has `data-testid=\"greeting\"` and `class=\"greeting\"`\n- The text needs to be blue (`style=\"color: blue;\"`)\n- Centered on the page (this could mean horizontal centering within its container)\n- Small margin around the text\n\nThe simplest approach is to apply all these styles via the `style` attribute on the `<p>` element since the main container already has the class \"greeting\". But wait - if I put the margin and color on the `<p>`, that would style the paragraph specifically. The requirement says \"Add a CSS class 'greeting' to style the text\" - so the class should be on the div, not the p tag.\n\nLet me reconsider. The div has `class=\"greeting\"`. I need to add additional styling for:\n- Blue text color\n- Centered text\n- Small margin\n\nI can do this by adding inline styles to the `<p>` element while keeping the class on the parent div. Or I could use a separate CSS block. Since this is a single component file, I'll use inline styles for simplicity.\n\nActually, looking at the example structure again:\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\nThis shows the text directly inside the div. So I should keep the text inside the div. The centering of the text on the page likely means the entire div should be centered, which can be done with flexbox on a wrapper or by setting the div's own display/flex properties.\n\nBut the requirement specifically mentions \"Add a CSS class 'greeting' to style the text\" - so the class should be on the div. And then \"Make the text color blue\" and \"Center the text on the page\" and \"Add a small margin around the text\".\n\nI think the most straightforward interpretation is:\n1. Div with `data-testid=\"greeting\"` and `class=\"greeting\"`\n2. Inside, the text \"Hello, World!\" \n3. The div gets styled (via CSS) to have blue text, be centered, and have margin\n\nSince Svelte 5 allows us to use CSS classes and inline styles, I'll set up the styles appropriately. The centering of the text on the page suggests using flexbox on the parent container. But there's no explicit parent mentioned...\n\nOne approach: wrap everything in a flex container. Another: use the div itself with flex properties.\n\nGiven the constraints, let me create a clean solution:\n\n<script>\n  // No reactive state needed for this static component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <p style=\"color: blue; text-align: center; margin: 8px;\">Hello, World!</p>\n</div>\nThis satisfies:\n- Component named HelloWorld (the filename will be HelloWorld.svelte)\n- Displays \"Hello, World!\"\n- Div has `data-testid=\"greeting\"` and `class=\"greeting\"`\n- Class \"greeting\" is added to the div\n- Text color is blue (via inline style)\n- Text is centered (text-align: center)\n- Small margin around text (margin: 8px)\n\nHowever, I'm not entirely satisfied about the \"center the text on the page\" requirement. If they want the entire greeting centered on the viewport, we might need a wrapper. But given the example structure shows just the div with the text, I think the above is sufficient.\n\nActually, re-reading: \"Center the text on the page\" - this could mean the text itself should be centered horizontally and vertically. With just the div containing the text, text-align: center centers it horizontally. For vertical centering, we'd need a wrapper with flex behavior.\n\nBut the example structure doesn't show a wrapper - it just shows the div with the text. So I'll stick with the simpler interpretation: center the text within its container (horizontal centering).\n\nLet me finalize the component:\n\n<script>\n  // No reactive state needed for this static component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <p style=\"color: blue; text-align: center; margin: 8px;\">Hello, World!</p>\n</div>\nThis looks correct. The component is named HelloWorld (implied by filename), it meets all requirements.<script>\n  // No reactive state needed for this static component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <p style=\"color: blue; text-align: center; margin: 8px;\">Hello, World!</p>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/hello-world_sample2_9cdf4f6b645fc8c7/Component.svelte:14:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No reactive state needed for this simple component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\" style=\"margin: 0 auto;\">\n  Hello, World!\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No reactive state needed for this simple component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No reactive state needed for this simple component\n</script>\n\n<div class=\"container\">\n  <div data-testid=\"greeting\" class=\"greeting\">\n    Hello, World!\n  </div>\n</div>\n\n<style>\n  .greeting {\n    color: blue;\n    margin: 16px;\n  }\n\n  .container {\n    display: flex;\n    justify-content: center;\n    align-items: center;\n    min-height: 100vh;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No state needed for this simple component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 8px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No state or effects needed for this simple component\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <style>\n    .greeting {\n      color: blue;\n      text-align: center;\n      margin: 10px;\n    }\n  </style>\n  Hello, World!\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No special state required for this simple component\n</script>\n\n<style>\n  .greeting {\n    display: flex;\n    justify-content: center;\n    align-items: center;\n    margin: 16px 0;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">\n  <span style=\"color: blue; margin: 8px;\">Hello, World!</span>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No state required for this simple component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    margin: 10px;\n    text-align: center;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "inspect",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte';\n  \n  // Svelte 5 $state for reactive text value, initialized with \"Hello world\"\n  let text = $state('Hello world');\n  \n  // Basic $inspect to log the input value\n  $inspect(text);\n  \n  // Track updates with a custom callback using $inspect(...).with()\n  $inspect(text).with(function() {\n    console.log('Input updated:', text);\n  });\n</script>\n\n<div>\n  <!-- Input field with data-testid=\"text-input\" -->\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  \n  <!-- Display current text with data-testid=\"text-value\" -->\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  \n  <!-- Character count with data-testid=\"char-count\" -->\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte';\n  \n  // Svelte 5 $state for reactive text input\n  let text = 'Hello world';\n  \n  // Basic $inspect to log the input value\n  inspect(text);\n  \n  // Custom callback to track updates\n  const handleUpdate = () => {\n    console.log('Input value changed:', text);\n  };\n  \n  // Track each change with a custom callback\n  $inspect(text).with(handleUpdate);\n  \n  // Derived value: character count\n  $: charCount = text.length;\n  \n  // Effect that runs when input changes - uses $inspect.trace()\n  effect(() => {\n    inspect.trace();\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample1_3f8aa114bf9eceee/Component.svelte:21:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { createEffect } from 'svelte';\n\n  // State for the input text, initialized with \"Hello world\"\n  $: text = 'Hello world';\n\n  // Basic $inspect to log the input value\n  $: $inspect(text);\n\n  // Track input changes with a custom callback using $inspect(...).with()\n  $: $inspect(text).with(() => {\n    console.log('Input updated:', text);\n  });\n\n  // Effect that uses $inspect.trace() to trace input changes\n  $: $inspect.trace().onUpdate((updatedText) => {\n    console.log('Input traced:', updatedText);\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample2_9dffe19d4211aa70/Component.svelte:7:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte';\n\n  // Svelte 5 $state for reactive state, initialized with \"Hello world\"\n  let text = $state('Hello world');\n\n  // Basic $inspect to log the input value\n  inspect(text);\n\n  // Track updates with custom callback using $inspect(...).with()\n  $inspect(text).with(function() {\n    console.log('Input updated:', text);\n  });\n\n  // Trace all state changes inside an effect\n  $: {\n    $inspect.trace();\n  }\n</script>\n\n<div>\n  <!-- Input field with data-testid=\"text-input\" -->\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n\n  <!-- Display current text with data-testid=\"text-value\" -->\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\n  <!-- Character count with data-testid=\"char-count\" -->\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample3_06ad7c0ded1823ad/Component.svelte:18:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { createEffect } from 'svelte';\n\n  // Reactive state for the input text, initialized with \"Hello world\"\n  let text = $state('Hello world');\n\n  // Basic $inspect to log the current input value\n  $: basicInspect = $inspect(text);\n\n  // Track updates to the input with a custom callback\n  $: trackedInspect = $inspect(text).with((newText) => {\n    console.log('Input updated:', newText);\n  });\n\n  // Effect that runs when the input changes, capturing trace\n  createEffect(() => {\n    $inspect.trace().then(() => {\n      console.log('Input change trace captured');\n    });\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample4_77246317c7aed2f4/Component.svelte:10:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte';\n\n  // Reactive state for the input text, initialized with \"Hello world\"\n  let text = $state('Hello world');\n\n  // Basic $inspect to log the input value\n  $inspect(text);\n\n  // Track updates with a custom callback using .with()\n  $inspect(text).with(function() {\n    console.log('Input updated:', text);\n  });\n</script>\n\n<div>\n  <!-- Input field with data-testid -->\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n\n  <!-- Display the input value with data-testid -->\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n\n  <!-- Display character count with data-testid -->\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>\n\n<style>\n  div {\n    padding: 16px;\n    border: 1px solid #ddd;\n    min-height: 60px;\n    font-family: sans-serif;\n  }\n</style>",
        "success": false,
        "errors": [
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte/inspect';\n\n  let text = $state('Hello world');\n\n  // Basic $inspect to log the input value\n  $inspect(text);\n\n  // Track updates with a custom callback\n  $inspect(text).with((updatedText) => {\n    console.log('Input updated via $inspect.with:', updatedText);\n  });\n\n  // Derived value: character count\n  const charCount = text.length;\n\n  // Effect that runs when input changes, tracing the update\n  $$effect(() => {\n    $inspect.trace(() => {\n      console.log('$inspect.trace triggered - input changed');\n    }, { target: text });\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/inspect_sample6_7dce44924a1e1fee/Component.svelte:20:2 `$$effect` is an illegal variable name. To reference a global variable called `$$effect`, use `globalThis.$$effect`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte';\n\n  let text = 'Hello world';\n\n  // Basic inspect - logs the current value whenever it changes\n  $inspect(text);\n\n  // Track updates with a custom callback\n  $inspect(text).with(function() {\n    console.log('Input changed:', text);\n  });\n\n  // Derived value: character count of the input text\n  const charCount = text.length;\n\n  // Effect with trace - runs when the input changes\n  $inspect.trace(() => {\n    console.log('Trace: Input updated');\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte';\n\n  // Reactive block (effect) that runs when the input changes\n  $: {\n    const trace = inspect.trace();\n  }\n\n  // Initial state with the input starting as \"Hello world\"\n  let text = 'Hello world';\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample8_9cbe53acaf65a00a/Component.svelte:7:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte';\n\n  let text = 'Hello world';\n\n  // Basic inspect - logs every change to the text value\n  $inspect(text);\n\n  // Track updates with a custom callback\n  $inspect(text).with((newValue) => {\n    console.log('Input value changed:', newValue);\n  });\n\n  // Derived value - character count\n  const charCount = text.length;\n\n  // Effect that runs when input changes - uses trace\n  $: inspect.trace(() => {\n    console.log('Input has been updated via trace');\n  }, { target: text });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text} \n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample9_9786d669dc037fb6/Component.svelte:20:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "props",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Accept props using $props rune\n  const props = {\n    name: 'World',\n    count: 0,\n    showDetails: false\n  };\n\n  // Reactive state for count using $state\n  let count = $state(props.count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{props.name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {props.name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"details\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div>\u001b[39m\n      \u001b[36m<h1\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"name-display\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mWorld\u001b[0m\n      \u001b[36m</h1>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<p\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"count-display\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mCount: 0\u001b[0m\n      \u001b[36m</p>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"increment-button\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mIncrement\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "expected 'Count: 0' to match /10/",
          "Unable to find an element by: [data-testid=\"details\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div />\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div>\u001b[39m\n      \u001b[36m<h1\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"name-display\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mWorld\u001b[0m\n      \u001b[36m</h1>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<p\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"count-display\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mCount: 0\u001b[0m\n      \u001b[36m</p>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"increment-button\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0mIncrement\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const props = {\n    name: { type: 'string', default: 'World' },\n    count: { type: 'number', default: 0 },\n    showDetails: { type: 'boolean', default: false }\n  };\n\n  let { name, count, showDetails } = $props();\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "store_invalid_shape\n`props` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n",
          "store_invalid_shape\n`props` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n",
          "store_invalid_shape\n`props` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n",
          "store_invalid_shape\n`props` is not a store with a `subscribe` method\nhttps://svelte.dev/e/store_invalid_shape\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Reactive state for count using $state\n  $: count = $state(count);\n\n  // Props received via $props rune\n  export let name = 'World';\n  export let count = 0;\n  export let showDetails = false;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/props_sample2_fe8cfaec0b6c1124/Component.svelte:5:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Accept props using the $props rune\n  const props = $props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  // Create a reactive state for the count\n  $: count = props.count;\n\n  // Keep state in sync with prop changes\n  $state(() => props.count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{props.name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {props.name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/props_sample3_94f30b50514c54fc/Component.svelte:5:16 `$props` cannot be called with arguments\nhttps://svelte.dev/e/rune_invalid_arguments"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Accept props using $props rune\n  const { name = \"World\", count = 0, showDetails = false } = $props();\n  \n  // Reactive variable for count using $state\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button data-testid=\"increment-button\" on:click={() => currentCount += 1}>Increment</button>\n  \n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state } from 'svelte';\n\n  // Accept props using $props rune\n  const props = $props();\n\n  // Reactive state for count\n  const count = $state(0);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{props.name}</h1>\n  <p data-testid=\"count-display\">Count: {count()}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {props.name}</p>\n      <p>Count is {count()}</p>\n      <p>ShowDetails is {props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/props_sample5_4d4d574a4ed0aebc/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Accept props using $props rune\n  const { name = \"World\", count = 0, showDetails = false } = $props();\n  \n  // Reactive state for count\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n  \n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "expected 'Count: 10' to match /11/"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const props = $props();\n  \n  export let name = 'World';\n  export let showDetails = false;\n  \n  // Reactive count using $state\n  let count = $state(0);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/props_sample7_f92e184d77632426/Component.svelte:6:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $props, $state } from 'svelte';\n\n  // Access props via $props rune\n  const props = $props();\n\n  // Export component properties\n  export let name;\n  export let count;\n  export let showDetails;\n\n  // Create reactive state for count using $state\n  $: displayedCount = count;\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {displayedCount}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {displayedCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/props_sample8_bc5681d95de89435/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const { name = 'World', count = 0, showDetails = false } = $props();\n  \n  // Reactive state for count\n  $: currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/props_sample9_1541ef604d024f6c/Component.svelte:7:2 `$:` is not allowed in runes mode, use `$derived` or `$effect` instead\nhttps://svelte.dev/e/legacy_reactive_statement_invalid"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  },
  {
    "testName": "snippets",
    "provider": "OpenRouter",
    "modelId": "liquid/lfm-2.5-2.6b:free",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { title } from './title.svelte';\n  \n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<template>\n  <ul>\n    {#each books as book}\n      <li data-testid=\"book-item\">\n        <title title={book} />\n      </li>\n    {/each}\n  </ul>\n</template>",
        "success": false,
        "errors": [
          "Failed to resolve import \"./title.svelte\" from \"tmp/samples/openrouter/snippets_sample0_ab06953b54ea308b/Component.svelte\". Does the file exist?"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  function title(bookTitle) {\n    return {\n      type: \"snippet\",\n      params: [bookTitle],\n      render: () => (\n        <span data-testid=\"book-title\">{bookTitle}</span>\n      )\n    };\n  }\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {#if title(book)}\n        <span data-testid=\"book-title\">{book}</span>\n      </li>\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample1_7766edd359bfb836/Component.svelte:15:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Hardcoded array of 3 book titles\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  // Snippet function that takes a book title and returns a component\n  function title(bookTitle) {\n    return {\n      render: ({ children }) => (\n        <span data-testid=\"book-title\">{bookTitle}</span>\n      )\n    };\n  }\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {#if title(book)}\n        {#render title(book)}\n          <!-- span is rendered here with data-testid=\"book-title\" -->\n        {/if}\n      </li>\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample2_f9ece7a098ae6643/Component.svelte:15:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Hardcoded array of 3 book titles\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  // Snippet function that takes a book title and returns a component definition\n  export let title = () => ({\n    render: ({ children }) => (\n      <span data-testid=\"book-title\">{children}</span>\n    )\n  });\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {#if title()}\n        {#render title()}\n          <!-- The title snippet renders the span with data-testid=\"book-title\" -->\n        {/if}\n      </li>\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample3_8a8a42fefd0fa17b/Component.svelte:14:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Hardcoded array of 3 book titles\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  // Snippet function that takes a book title and returns a component\n  function title(bookTitle) {\n    return {\n      render: () => (\n        <span data-testid=\"book-title\">{bookTitle}</span>\n      )\n    };\n  }\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {#if title(book)}\n        <span data-testid=\"book-title\">{book}</span>\n      {/if}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample4_8d184f34cd6ee46c/Component.svelte:15:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Hardcoded array of book titles\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  // Snippet function that takes a book title and returns a snippet configuration\n  function title(bookTitle) {\n    return {\n      type: \"snippet\",\n      params: [bookTitle],\n      render: () => `\n        <span data-testid=\"book-title\">${bookTitle}</span>\n      `\n    };\n  }\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {#if title(book)}\n        <span data-testid=\"book-title\">{book}</span>\n      </li>\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample5_bac5dfaec8e05074/Component.svelte:28:6 `</li>` attempted to close an element that was not open\nhttps://svelte.dev/e/element_invalid_closing_tag"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Hardcoded array of 3 book titles\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {$title(book)}\n    </li>\n  {/each}\n</ul>\n\n<!-- Snippet component that takes a book title and renders it -->\nexport let title = {\n  get() {\n    return {\n      render({ children }) {\n        return (\n          <span data-testid=\"book-title\">{this.title}</span>\n        );\n      }\n    };\n  }\n};",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample6_356d09d2a24d0d87/Component.svelte:22:8 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<template>\n  <ul>\n    {#each books as book}\n      <li data-testid=\"book-item\">\n        <slot name=\"title\" />\n      </li>\n    {% endeach %}\n  </ul>\n</template>\n\n<script>\n  export let title;\n\n  function title() {\n    return {\n      type: \"snippet\",\n      params: [title],\n      render: () => `\n        <span data-testid=\"book-title\">${title}</span>\n      `\n    };\n  }\n</script>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample7_ad1ea81210371250/Component.svelte:17:5 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Hardcoded array of 3 book titles\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n  \n  // Snippet function that takes a book title and returns a span element\n  function title(title) {\n    return `<span data-testid=\"book-title\">{title}</span>`;\n  }\n</script>\n\n<main>\n  <ul>\n    {$for (book of books)}\n      ${#each books as book}\n        ${@render title book}\n      {/each}\n    </ul>\n  </main>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample8_6da9fea67c325cd8/Component.svelte:19:16 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Hardcoded array of book titles\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {#if title(book)}\n        <span data-testid=\"book-title\">{book}</span>\n      {/if}\n    </li>\n  {/each}\n</ul>\n\n<script>\n  function title(bookTitle) {\n    return {\n      type: \"component\",\n      name: \"title\",\n      params: { bookTitle },\n      render({ children }) {\n        return <span data-testid=\"book-title\">{bookTitle}</span>;\n      }\n    };\n  }\n</script>",
        "success": false,
        "errors": [
          "svelte-bench/tmp/samples/openrouter/snippets_sample9_9d1716436181dcbb/Component.svelte:29:15 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-08-14T22:56:15.012Z"
  }
]