[
  {
    "testName": "counter",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 4,
    "pass1": 0.4,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  import { onMount } from 'svelte';\n\n  $state let count = 0;\n\n  function decrement() {\n    count--;\n  }\n\n  function increment() {\n    count++;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/counter_sample1_83b8c512fcd5f116/Component.svelte:6:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let $state = '';\n  let $effect = '';\n\n  $state count = 0;\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n\n  $effect(() => {\n    console.log('count changed');\n  });\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/counter_sample2_f91a9eef6e04eb8e/Component.svelte:7:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<!-- Counter.svelte -->\n<script>\n  $state count: number = 0;\n  \n  function decrement() {\n    count--;\n  }\n  \n  function increment() {\n    count++;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/counter_sample3_852d5c69cf9ff331/Component.svelte:5:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n\t$state count = 0;\n\n\tfunction increment() {\n\t\tcount += 1;\n\t}\n\n\tfunction decrement() {\n\t\tcount -= 1;\n\t}\n</script>\n\n<div class=\"counter\">\n\t<button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n\t<span data-testid=\"count-value\">{count}</span>\n\t<button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/counter_sample4_c2fe650b3d83ac9d/Component.svelte:4:8 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  $state count = 0;\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n</script>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    justify-content: center;\n    gap: 1rem;\n    border: 1px solid #ccc;\n    padding: 1rem;\n    border-radius: 8px;\n  }\n  \n  button {\n    width: 2rem;\n    height: 2rem;\n    font-size: 1.5rem;\n  }\n</style>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/counter_sample5_f9e6c004daa96c72/Component.svelte:4:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n  \n  function decrement() {\n    count--;\n  }\n  \n  function increment() {\n    count++;\n  }\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" on:click={increment}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n\n  function increment() {\n    count++;\n  }\n\n  function decrement() {\n    count--;\n  }\n</script>\n\n<div class=\"counter\">\n  <button on:click={decrement} data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button on:click={increment} data-testid=\"increment-button\">+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n\n  function increment() {\n    count += 1;\n  }\n\n  function decrement() {\n    count -= 1;\n  }\n</script>\n\n<div class=\"counter\">\n  <button on:click={decrement} data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button on:click={increment} data-testid=\"increment-button\">+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<Counter>\n\t<script lang=\"ts\">\n\t\timport { onMount } from 'svelte';\n\n\t\tlet count = $state(0);\n\n\t\tfunction decrement() {\n\t\t\tcount--;\n\t\t}\n\n\t\tfunction increment() {\n\t\t\tcount++;\n\t\t}\n\t</script>\n\n\t<div class=\"counter\">\n\t\t<button data-testid=\"decrement-button\" on:click={decrement}>-</button>\n\t\t<span data-testid=\"count-value\">{count}</span>\n\t\t<button data-testid=\"increment-button\" on:click={increment}>+</button>\n\t</div>\n\n\t<style>\n\t\t.counter {\n\t\t\tdisplay: flex;\n\t\t\talign-items: center;\n\t\t\tjustify-content: center;\n\t\t\tgap: 1rem;\n\t\t}\n\n\t\tspan {\n\t\t\tfont-size: 1.5rem;\n\t\t\tfont-weight: bold;\n\t\t}\n\n\t\tbutton {\n\t\t\tpadding: 0.5rem 1rem;\n\t\t\tborder: none;\n\t\t\tbackground-color: #007bff;\n\t\t\tcolor: white;\n\t\t\tborder-radius: 0.25rem;\n\t\t\tcursor: pointer;\n\t\t}\n\t</style>\n</Counter>",
        "success": false,
        "errors": [
          "Counter is not defined\n\n\tin <unknown>\n",
          "Counter is not defined\n\n\tin <unknown>\n",
          "Counter is not defined\n\n\tin <unknown>\n",
          "Counter is not defined\n\n\tin <unknown>\n"
        ]
      }
    ],
    "timestamp": "2025-12-08T17:35:43.936Z"
  },
  {
    "testName": "derived",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 5,
    "pass1": 0.5,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { onMount } from 'svelte';\n\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  \n  // Using $derived to create a computed value\n  let doubled = $derived(number * 2);\n  \n  // Function to increment the number\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { tick } from 'svelte';\n\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n\n  const increment = () => number++;\n\n  $effect(() => {\n    tick();\n    console.log('Number:', number, 'Doubled:', doubled);\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n    import { Tick } from 'svelte/motion';\n\n    let { number = 5 } = $state();\n\n    const doubled = $derived(number * 2);\n\n    function increment() {\n        number += 1;\n    }\n</script>\n\n<div>\n    <p data-testid=\"number-value\">Number: {number}</p>\n    <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n    <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "window.matchMedia is not a function"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { derived, state } from 'svelte/store';\n\n  let number = state(5);\n  let doubled = derived(number, $number => $number * 2);\n\n  function increment() {\n    number.update($n => $n + 1);\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "(0 , state) is not a function\n\n\tin <unknown>\n",
          "(0 , state) is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<svelte:head>\n    <script>\n        export let number = 5;\n    </script>\n\n    <script lang=\"ts\">\n        import { onMount, tick } from 'svelte';\n\n        let number = $state(5);\n        let doubled = $derived(number * 2);\n\n        function increment() {\n            number++;\n        }\n\n        onMount(async () => {\n            await tick();\n            number = $state.snapshot(props.number);\n        });\n    </script>\n\n    <div>\n        <p data-testid=\"number-value\">Number: {number}</p>\n        <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n        <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n    </div>\n</svelte:head>",
        "success": false,
        "errors": [
          "number is not defined\n\n\tin <unknown>\n\tin Component.svelte\n",
          "number is not defined\n\n\tin <unknown>\n\tin Component.svelte\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { number = $state(5) } = $state({});\n\n  let doubled = $derived(number * 2);\n\n  function increment() {\n    number++;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/derived_sample6_23f5c24682cb7050/Component.svelte:4:17 `$state(...)` can only be used as a variable declaration initializer, a class field declaration, or the first assignment to a class field at the top level of the constructor.\nhttps://svelte.dev/e/state_invalid_placement"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $derived } from 'svelte/store';\n  \n  let number = $state(5);\n  let doubled = $derived(number * 2);\n  \n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/derived_sample7_778d62a147527f53/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { onMount } from 'svelte';\n\n  let count = $state(5);\n  let doubled = $derived(count * 2);\n\n  function increment() {\n    count += 1;\n  }\n\n  $effect(() => {\n    console.log('Count changed:', count);\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {count}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  },
  {
    "testName": "derived-by",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 4,
    "pass1": 0.4,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { onMount } from 'svelte';\n\n  let text = $state('');\n  let wordCount = $derived.by(() => text.split(/\\s+/).filter(word => word.length > 0).length);\n  let charCount = $derived.by(() => text.length);\n  let isLongText = $derived.by(() => wordCount > 15);\n\n  function clearText() {\n    text = '';\n  }\n\n  onMount(() => {\n    // Initial setup if needed\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLongText ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={$text}\n    on:input={handleInput}\n  />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLongText ? 'Long text' : 'Short text'}</p>\n  </div>\n\n  $state text = '';\n  \n  $derived.by words = $text.split(/\\s+/).filter(Boolean);\n  \n  $derived.by wordCount = words.length;\n  \n  $derived.by charCount = $text.length;\n  \n  $derived.by isLongText = wordCount > 15;\n  \n  function handleInput(event: InputEvent) {\n    $text = (event.target as HTMLInputElement).value;\n  }\n  \n  function clearText() {\n    $text = '';\n  }\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/derived-by_sample1_69d3f0ebf2a4fe48/Component.svelte:28:26 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const stats = $derived.by({\n    wordCount: text.trim().split(/\\s+/).length,\n    charCount: text.length,\n    isLong: text.trim().split(/\\s+/).filter(Boolean).length > 15\n  });\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n  />\n  <button\n    data-testid=\"clear-button\"\n    on:click={() => text = ''}\n  >\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "fn is not a function\n\n\tin <unknown>\n",
          "fn is not a function\n\n\tin <unknown>\n",
          "fn is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const derived = $derived.by({\n    wordCount: text.trim() === '' ? 0 : text.trim().split(/\\s+/).length,\n    charCount: text.length,\n    isLong: text.trim() !== '' ? text.trim().split(/\\s+/).length > 15 : false\n  });\n\n  function clearText() {\n    text = '';\n  }\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n  />\n  <button on:click={clearText} data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$derived.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$derived.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$derived.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/derived-by_sample3_8937ed3a85d60cae/Component.svelte:25:39 `$derived.wordCount` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n<script>\n  let text = $state('')\n  \n  let wordCount = $derived.by(() => {\n    const words = text.trim().split(/\\s+/).filter(word => word.length > 0)\n    return words.length\n  })\n  \n  let charCount = $derived.by(() => {\n    return text.length\n  })\n  \n  let isLongText = $derived.by(() => {\n    return wordCount > 15\n  })\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    type=\"text\" \n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" on:click={() => text = ''}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {isLongText ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<svelte:component this=\"{TextDerived}\" let:data-testid>\n\n<script lang=\"ts\">\n  let text = $state('');\n  \n  const wordCount = $derived.by(() => text.split(/\\s+/).filter(Boolean).length);\n  const charCount = $derived.by(() => text.length);\n  const isLong = $derived.by(() => wordCount > 15);\n\n  function clearText() {\n    text = '';\n  }\n\n  $effect(() => {\n    console.log('Text updated:', text);\n  });\n</script>\n\n<div>\n  <input \n    data-testid=\"text-input\" \n    bind:value={text} \n    type=\"text\" \n    placeholder=\"Enter text here\" \n  />\n  <button \n    data-testid=\"clear-button\" \n    on:click={clearText}\n  >\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>\n\n</svelte:component>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/derived-by_sample5_2c2f731091a0f159/Component.svelte:24:16 Can only bind to state or props\nhttps://svelte.dev/e/bind_invalid_value"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { tick } from 'svelte';\n  import { getContext } from 'svelte';\n\n  let text = $state(\"\"); // Reactive text input\n\n  // Derived calculations using $derived.by\n  let wordCount = $derived.by(() => {\n    return text.trim() ? text.trim().split(/\\s+/).length : 0;\n  });\n\n  let charCount = $derived.by(() => {\n    return text.length;\n  });\n\n  let isLongText = $derived.by(() => {\n    const words = text.trim() ? text.trim().split(/\\s+/) : [];\n    return words.length > 15;\n  });\n\n  function clearText() {\n    text = \"\";\n  }\n\n  async function handleKeyPress(event) {\n    if (event.key === 'Enter') {\n      // Execute updates when user presses Enter\n      await tick();\n      console.log(\"Text entered:\", text);\n      console.log(\"Word count:\", wordCount);\n      console.log(\"Character count:\", charCount);\n      console.log(\"Is long text:\", isLongText);\n    }\n  }\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} on:keypress={handleKeyPress} />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLongText ? \"Long text\" : \"Short text\"}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <input data-testid=\"text-input\" bind:value={$text} type=\"text\" />\n  <button data-testid=\"clear-button\" on:click={() => $text = ''} >Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {$wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {$charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {$isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/derived-by_sample7_0570410bb85f937d/Component.svelte:4:46 `$text` is an illegal variable name. To reference a global variable called `$text`, use `globalThis.$text`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { tick } from 'svelte';\n\n  let text = $state('');\n  let wordCount = $derived.by(() => text.split(/\\s+/).filter(Boolean).length);\n  let charCount = $derived.by(() => text.length);\n  let isLong = $derived.by(() => wordCount > 15);\n  let lengthStatus = $derived.by(() => isLong ? 'Long text' : 'Short text');\n\n  async function clearText() {\n    text = '';\n    await tick();\n    text = '';\n  }\n\n  function handleInputChange(event) {\n    text = event.target.value;\n  }\n\n  function handleClearClick() {\n    clearText();\n  }\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    on:input={handleInputChange}\n  />\n  <button data-testid=\"clear-button\" on:click={handleClearClick}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {lengthStatus}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<svelte:options immutable />\n\n<script>\n  let text = $state('');\n\n  const wordCount = $derived.by(() => {\n    return text.trim() ? text.trim().split(/\\s+/).length : 0;\n  });\n\n  const charCount = $derived.by(() => text ? text.length : 0);\n\n  const isLongText = $derived.by(() => wordCount > 15);\n\n  function clearText() {\n    text = '';\n  }\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    bind:value={text}\n    on:input={event => text = event.target.value}\n  />\n  <button data-testid=\"clear-button\" on:click={clearText}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {isLongText ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/derived-by_sample9_57863000c30488f8/Component.svelte:3:0 A component can only have one `<svelte:options>` element\nhttps://svelte.dev/e/svelte_meta_duplicate"
        ]
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  },
  {
    "testName": "each",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 2,
    "pass1": 0.20000000000000007,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    characters = [...characters, 'George'];\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character (character)}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>Add George</button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click=\"{() => addCharacter('George')}\">Add George</button>\n</div>\n\n<script>\n  $state characters = [\"Jerry\", \"Elaine\", \"Kramer\"]\n\n  function addCharacter(character) {\n    $characters = [...$characters, character]\n  }\n</script>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/each_sample2_cad759b7d43d1675/Component.svelte:13:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each $characters as character}\n    <li data-testid=\"character\">{character}</li>\n    {:else}\n    <li data-testid=\"character\">No characters</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click=\"{addGeorge}\">Add George</button>\n</div>\n\n<script>\n  const seinfeldCharacters = [\"Jerry\", \"Elaine\", \"Kramer\"];\n  let $characters = $state([...seinfeldCharacters]);\n\n  function addGeorge() {\n    $characters = [...$characters, \"George\"];\n  }\n</script>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/each_sample3_270736d35b906e22/Component.svelte:16:6 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n    let characters = [\"Jerry\", \"Elaine\", \"Kramer\"];\n\n    function addGeorge() {\n        characters = [...characters, \"George\"];\n    }\n</script>\n\n<div>\n    <ul data-testid=\"characters-list\">\n        {#each characters as character}\n            <li data-testid=\"character\">{character}</li>\n        {/each}\n    </ul>\n    <button data-testid=\"add-george-button\" on:click={addGeorge}>\n        Add George\n    </button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n  \n  function addGeorge() {\n    characters = [...characters, 'George'];\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state(['Jerry', 'Elaine', 'Kramer']);\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button on:click={addGeorge} data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = [\"Jerry\", \"Elaine\", \"Kramer\"];\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n    <li data-testid=\"character\">{character}</li>\n    {:else}\n    <li data-testid=\"character\">No characters found</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n\n  function addGeorge() {\n    characters.push(\"George\");\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button on:click={addGeorge} data-testid=\"add-george-button\">Add George</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = ['Jerry', 'Elaine', 'Kramer'];\n\n  function addGeorge() {\n    characters.push('George');\n  }\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" on:click={addGeorge}>\n    Add George\n  </button>\n</div>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  },
  {
    "testName": "effect",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { tick } from 'svelte';\n\n  let $number = 5;\n  \n  $effect(() => {\n    let timer;\n    \n    // Debounce the effect using tick\n    $: {\n      clearTimeout(timer);\n      timer = setTimeout(() => {\n        tick().then(() => {\n          doubled = $number * 2;\n        });\n      }, 0);\n    }\n  });\n  \n  function increment() {\n    $number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/effect_sample1_904e30fea5cc4b93/Component.svelte:6:6 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Initialize state with $state\n  let $number = 5;\n  \n  // Use $effect to calculate doubled value\n  $effect(() => {\n    const doubled = $number * 2;\n    return doubled;\n  });\n  \n  // Increment function\n  function increment() {\n    $number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {$number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$effect(() => $number * 2)}</p>\n  <button on:click={increment} data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/effect_sample2_ed5686e62417e8dc/Component.svelte:5:6 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number += 1}>\n    Increment\n  </button>\n</div>\n\n<script>\n  import {state, effect} from 'svelte/store';\n\n  let number = 5;\n  const doubled = $state<number | null>(null);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/effect_sample3_0bf00adffa1cff11/Component.svelte:18:4 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { onMount } from 'svelte';\n  let number = 5;\n\n  $effect(() => {\n    let doubled = number * 2;\n  });\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {$doubled ?? 0}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/effect_sample4_c024bf27a58e905f/Component.svelte:18:43 `$doubled` is an illegal variable name. To reference a global variable called `$doubled`, use `globalThis.$doubled`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n\n  let doubled = $effect(() => number * 2);\n\n  function increment() {\n    number += 1;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/effect_sample5_6b0ddf99c7b8f122/Component.svelte:6:16 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $effect, $state } from 'svelte/reactivity';\n\n  let number = $state(5);\n  let doubled;\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/effect_sample6_ab80a2dfd2d78ff9/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { tick } from 'svelte';\n\n  let number = $state(5);\n  let doubled = $effect(() => {\n    tick().then(() => {});\n    return number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/effect_sample7_5ecb68cbd038b4a7/Component.svelte:7:16 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true}/>\n\n<script>\n  let number = $state(5);\n  let doubled = $effect(() => number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={() => number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/effect_sample8_686943bcfacc1994/Component.svelte:5:16 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { onMount } from 'svelte';\n  import { tick } from 'svelte';\n\n  let number = $state(5);\n  let doubled;\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n\n  function increment() {\n    number++;\n  }\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 10\u001b[39m\nReceived:\n\u001b[31m  Doubled:\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 12\u001b[39m\nReceived:\n\u001b[31m  Doubled: 10\u001b[39m"
        ]
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  },
  {
    "testName": "hello-world",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 6,
    "pass1": 0.5999999999999999,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No additional script needed for this simple component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 10px;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<svelte:component this={HelloWorld}>\n    <style>\n        .greeting {\n            color: blue;\n            text-align: center;\n            margin: 20px\n        }\n    </style>\n    <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</svelte:component>",
        "success": false,
        "errors": [
          "HelloWorld is not defined\n\n\tin <unknown>\n\tin Component.svelte\n",
          "HelloWorld is not defined\n\n\tin <unknown>\n\tin Component.svelte\n"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<svelte:head>\n<style>\n.greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n    margin-top: 5rem;\n}\n</style>\n</svelte:head>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<svelte:window on:load={() => { document.documentElement.style.fontSize = '18px'; }}>\n<script>\n  // No additional JavaScript needed for this simple component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 2rem;\n    font-family: Arial, sans-serif;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</svelte:window>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/hello-world_sample3_4c35e8eeb76117d2/Component.svelte:3:85 <svelte:window> cannot have children\nhttps://svelte.dev/e/svelte_meta_invalid_content"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // No external dependencies needed for this simple component\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Define the HelloWorld component\n  const HelloWorld = () => {\n    return (\n      <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n    );\n  };\n</script>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 20px;\n  }\n</style>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/hello-world_sample6_16fbb3c23d04f1c6/Component.svelte:7:6 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n    font-size: 1.2rem;\n    font-family: Arial, sans-serif;\n  }\n</style>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<svelte:component this={HelloWorld}>\n\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n  <style>\n    .greeting {\n      color: blue;\n      text-align: center;\n      margin: 20px;\n    }\n  </style>\n\n</svelte:component>",
        "success": false,
        "errors": [
          "HelloWorld is not defined\n\n\tin <unknown>\n\tin Component.svelte\n",
          "HelloWorld is not defined\n\n\tin <unknown>\n\tin Component.svelte\n"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<svelte:head>\n    <style>\n        .greeting {\n            color: blue;\n            text-align: center;\n            margin: 20px;\n        }\n    </style>\n</svelte:head>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  },
  {
    "testName": "inspect",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 3,
    "pass1": 0.30000000000000004,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { tick, tick_after_each } from 'svelte';\n\n  let text = $state('Hello world');\n\n  $inspect(text);\n  $inspect(text).with((value) => console.log('Input changed to:', value));\n  \n  $effect(() => {\n    $inspect.trace();\n    console.log('Effect running with text:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  import { onMount, tick } from 'svelte';\n\n  let text = $state('Hello world');\n\n  $inspect(text).with(() => {\n    console.log('Input changed:', text);\n  });\n\n  $inspect.trace(text);\n\n  $effect(() => {\n    console.log('Input changed in effect:', text);\n  });\n\n  $inspect(() => text.length);\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample1_12cd5bf7e6ea869f/Component.svelte:12:2 `$inspect.trace(...)` must be the first statement of a function body\nhttps://svelte.dev/e/inspect_trace_invalid_placement"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { inspect } from 'svelte/internal'\n    \n  let text = $state('Hello world')\n  let count = $state(0)\n    \n  // Basic inspection\n  $inspect(text);\n  \n  // Custom callback\n  $inspect(text).with(newValue => console.log(`Text updated: ${newValue}`));\n  \n  // Tracing inside effect\n  $effect(() => {\n    let traceId = $inspect.trace().with(value => console.log(`Tracing: ${value}`, { type: 'store' }))\n    return () => traceId()\n  });\n  \n  // Derived state for character count\n  $effect(() => {\n    count = text.length\n  })\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$inspect(text).with((value) => value)} />\n  <p data-testid=\"text-value\">Current text: \"{$inspect(text).with((value) => value)}</p>\n  <p data-testid=\"char-count\">Character count: {$inspect(count).with((value) => value)}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample2_526b6f2929471c69/Component.svelte:4:2 Imports of `svelte/internal/*` are forbidden. It contains private runtime code which is subject to change without notice. If you're importing from `svelte/internal/*` to work around a limitation of Svelte, please open an issue at https://github.com/sveltejs/svelte and explain your use case\nhttps://svelte.dev/e/import_svelte_internal_forbidden"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state(\"Hello world\");\n\n  $inspect;\n  $inspect(text).with(value => console.log(`Text changed to: ${value}`));\n  $inspect.trace();\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample3_bfbd01f1c224d834/Component.svelte:6:2 Cannot use rune without parentheses\nhttps://svelte.dev/e/rune_missing_parentheses"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n\n  $inspect(text);\n  $inspect(text).with((value) => {\n    console.log(`Input changed to: ${value}`);\n  });\n\n  $effect(() => {\n    console.trace('Input changed');\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let $state = (initialValue) => {\n    const subscribers = new Set;\n    let value = initialValue;\n\n    return {\n      get value() {\n        return value;\n      },\n      set value(newValue) {\n        value = newValue;\n        subscribers.forEach(sub => sub());\n      },\n      subscribe(callback) {\n        subscribers.add(callback);\n        callback();\n        return () => subscribers.delete(callback);\n      }\n    };\n  };\n\n  let $inspect = (item, options = {}) => {\n    const subscribers = new Set;\n    let value = item;\n\n    const withCallback = (callback) => {\n      return { with: () => callback(value) };\n    };\n\n    return {\n      get value() {\n        return value;\n      },\n      set value(newValue) {\n        value = newValue;\n        subscribers.forEach(sub => sub());\n      },\n      with: withCallback,\n      subscribe(callback) {\n        subscribers.add(callback);\n        callback();\n        return () => subscribers.delete(callback);\n      },\n      trace() {\n        console.log('Tracing:', value, options);\n      }\n    };\n  };\n\n  let text = $inspect($state('Hello world'));\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text.value} />\n  {#each $inspect(text, {}).with.with(text => [\n    {key: 'text-value', value: text}\n  ]) as item (item.key)}\n    <p data-testid={item.key}>Current text: \"{item.value}\"</p>\n  {/each}\n  {#each $inspect(text.length, {type: 'char-count'}).with.with(length => [\n    {key: 'char-count', value: length}\n  ]) as item (item.key)}\n    <p data-testid={item.key}>Character count: {item.value}</p>\n  {/each}\n  $effect(() => {\n    text.trace();\n  });\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/inspect_sample5_95bbc50ada6b7ecf/Component.svelte:68:16 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={$state.text} />\n  <p data-testid=\"text-value\">Current text: \"{$state.text}\"</p>\n  <p data-testid=\"char-count\">Character count: {$state.text.length}</p>\n</div>\n\n<script>\n  import {$state, $inspect} from 'svelte/internal';\n\n  let text = $state('Hello world');\n\n  $inspect(text);\n  \n  $inspect(text).with(({newValue, oldValue}) => {\n    console.log(`Text changed from \"${oldValue}\" to \"${newValue}\"`);\n  });\n  \n  $inspect(text).trace();\n\n  $effect(() => {\n    console.log('Text changed:', $state.text);\n  });\n</script>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/inspect_sample6_418cda4246bae4b5/Component.svelte:10:10 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n\n  $inspect(text);\n  $inspect(text)\n    .with((value) => console.log(`Input changed to: \"${value}\"`));\n\n  $effect(() => {\n    $inspect.trace()\n    console.log('Effect runs when input changes');\n  });\n\n  $derived.charCount = text.length;\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {$derived.charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample7_e865938e25bf2fca/Component.svelte:15:2 `$derived.charCount` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { tick } from 'svelte';\n\n  let text = $state(\"Hello world\");\n\n  $inspect(text);\n\n  $inspect(text).with(() => {\n    console.log(\"Input changed to:\", text);\n  });\n\n  $inspect.trace(\"text\", () => console.log('Tracing text change:', text));\n\n  $effect(() => {\n    console.log(\"Effect running - current text:\", text);\n  });\n\n  $effect(() => {\n    $inspect(text).with(() => {\n      console.log(\"Characters:\", text.length);\n    });\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/openrouter/inspect_sample8_513fc5cc96a9dc97/Component.svelte:14:2 `$inspect.trace` must be called with zero or one arguments\nhttps://svelte.dev/e/rune_invalid_arguments_length"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n\n  $inspect(text).with((value) => {\n    console.log('Input updated:', value);\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Input value:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  },
  {
    "testName": "props",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { props } from 'svelte/internal';\n\n  let $props = props({\n    name: 'World',\n    count: 0,\n    showDetails: false\n  });\n\n  let count = $props.count;\n\n  function increment() {\n    count += 1;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" on:click={increment}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {$props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample0_00737c31d205e0fc/Component.svelte:6:6 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<svelte:component this={PropsDemo}>\n  <div>\n    <h1 data-testid=\"name-display\">{helloName() + ', ' + $props.name}</h1>\n    <p data-testid=\"count-display\">Count: {$count}</p>\n    <button data-testid=\"increment-button\" on:click={handleIncrement}>\n      Increment\n    </button>\n\n    {#if $props.showDetails}\n      <div data-testid=\"details\">\n        <p>Name is {$props.name}</p>\n        <p>Count is {$count}</p>\n        <p>ShowDetails is {$props.showDetails}</p>\n      </div>\n    {/if}\n  </div>\n\n  <script lang=\"ts\">\n    export let helloName = ($props.name || 'World') + '!';\n    let count = $props.count;\n    $state let $count = count;\n\n    function handleIncrement() {\n      $count++;\n    }\n  </script>\n</svelte:component>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample1_e186a909d02ecfde/Component.svelte:6:43 `$count` is an illegal variable name. To reference a global variable called `$count`, use `globalThis.$count`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { props } from 'svelte-perfect';\n\n  export const $props = {\n    name: 'name',\n    count: 'count',\n    showDetails: 'showDetails',\n  };\n\n  const { name = 'World', count = 0, showDetails = false } = $props;\n\n  let state = $state({\n    count: count,\n  });\n\n  function incrementCount() {\n    state.count += 1;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {state.count}</p>\n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {state.count}</p>\n      <p>ShowDetails is true</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample2_011a8648b18f99a3/Component.svelte:6:15 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { onMount } from 'svelte';\n\n  let { name = 'World', count = 0, showDetails = false } = $props;\n\n  $state countValue = count;\n\n  function incrementCount() {\n    countValue++;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {countValue}</p>\n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {name}</p>\n    <p>Count is {countValue}</p>\n    <p>ShowDetails is {showDetails}</p>\n  </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample3_2307d0fddfc83c26/Component.svelte:8:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  $props({\n    name: { type: String, default: 'World' },\n    count: { type: Number, default: 0 },\n    showDetails: { type: Boolean, default: false }\n  })\n\n  let { count: $count } = $props\n  let name = $props.name\n  $state let localCount = $count\n\n  function increment() {\n    localCount += 1\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {localCount}</p>\n\n  <button data-testid=\"increment-button\" on:click={increment}>\n    Increment\n  </button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {localCount}</p>\n      <p>ShowDetails is {$props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample4_e81c38a5e380b633/Component.svelte:12:9 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { props } from 'svelte/internal'\n  \n  let { name = 'World', count = 0, showDetails = false } = $props\n\n  return {\n    props: {\n      name: String,\n      count: Number,\n      showDetails: Boolean\n    }\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={() => $state.count++}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample5_51c29831d75c3247/Component.svelte:8:2 'return' outside of function\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { props } from \"smui\";\n  \n  export let $props = {\n    name: props.string(\"World\"),\n    count: props.number(0),\n    showDetails: props.boolean(false)\n  };\n  \n  let $state = {\n    count: $props.count\n  };\n  \n  function incrementCount() {\n    $state.count += 1;\n  }\n  \n  $effect(() => {\n    $state.count = $props.count;\n  });\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {$props.name}!</h1>\n  <p data-testid=\"count-display\">Count: {$state.count}</p>\n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if $props.showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {$props.name}</p>\n      <p>Count is {$state.count}</p>\n      <p>ShowDetails is {$props.showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample6_72aa33cf6025815e/Component.svelte:6:13 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<svelte:component this={$props().name} />\n<script>\n  let { name = \"World\", count = 0, showDetails = false } = $props();\n  let $count = count;\n\n  function incrementCount() {\n    $count += 1;\n  }\n</script>\n\n\n<h1 data-testid=\"name-display\">Hello, {$name}!</h1>\n<p data-testid=\"count-display\">Count: {$count}</p>\n<button data-testid=\"increment-button\" on:click={incrementCount}>\n  Increment\n</button>\n\n{#if showDetails}\n  <div data-testid=\"details\">\n    <p>Name is {$name}</p>\n    <p>Count is {$count}</p>\n    <p>ShowDetails is {showDetails ? 'true' : 'false'}</p>\n  </div>\n{/if}",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample7_a2237b317351b20d/Component.svelte:6:6 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let $props = {\n    name: 'World',\n    count: 0,\n    showDetails: false\n  };\n\n  let $state count = $props.count;\n\n  function incrementCount() {\n    count += 1;\n  }\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{`Hello, ${$props.name}!`}</h1>\n  <p data-testid=\"count-display\">{`Count: ${count}`}</p>\n  \n  <button data-testid=\"increment-button\" on:click={incrementCount}>Increment</button>\n\n  {#if $props.showDetails}\n  <div data-testid=\"details\">\n    <p>{`Name is ${$props.name}`}</p>\n    <p>{`Count is ${count}`}</p>\n    <p>{`ShowDetails is ${$props.showDetails}`}</p>\n  </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/props_sample8_3eac6fb648b94428/Component.svelte:10:13 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<svelte:component this={PropsDemo}>\n\t<script>\n\t\timport { PropsDemo } from './PropsDemo.svelte';\n\t</script>\n</svelte:component>",
        "success": false,
        "errors": [
          "PropsDemo is not defined\n\n\tin <unknown>\n\tin Component.svelte\n",
          "PropsDemo is not defined\n\n\tin <unknown>\n\tin Component.svelte\n",
          "PropsDemo is not defined\n\n\tin <unknown>\n\tin Component.svelte\n",
          "PropsDemo is not defined\n\n\tin <unknown>\n\tin Component.svelte\n"
        ]
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  },
  {
    "testName": "snippets",
    "provider": "OpenRouter",
    "modelId": "essentialai/rnj-1-instruct",
    "numSamples": 10,
    "numCorrect": 2,
    "pass1": 0.20000000000000007,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  const title = (bookTitle) => {\n    return `<span data-testid=\"book-title\">${bookTitle}</span>`;\n  };\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  function title(title) {\n    return $render`<span data-testid=\"book-title\">{title}</span>`;\n  }\n\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">{$render(title, book)}</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/snippets_sample1_a28659added74b26/Component.svelte:5:11 `$render` is an illegal variable name. To reference a global variable called `$render`, use `globalThis.$render`\nhttps://svelte.dev/e/global_reference_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  // Array of hardcoded book titles\n  let books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  // Snippet to display a book title in a span with data-testid\n  let title = (book) => `\n    <span data-testid=\"book-title\">{book}</span>\n  `;\n</script>\n\n<ul>\n  {#each books as book (book)}\n    <li data-testid=\"book-item\">\n      {book => { \n        return title(book);\n      }}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m(book) =&gt; {\n\t\t\t\treturn title(book);\n\t\t\t}\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m(book) =&gt; {\n\t\t\t\treturn title(book);\n\t\t\t}\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m(book) =&gt; {\n\t\t\t\treturn title(book);\n\t\t\t}\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m(book) =&gt; {\n\t\t\t\treturn title(book);\n\t\t\t}\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m(book) =&gt; {\n\t\t\t\treturn title(book);\n\t\t\t}\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m(book) =&gt; {\n\t\t\t\treturn title(book);\n\t\t\t}\u001b[0m\n      \u001b[36m</li>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const title = (book) => (\n    `<span data-testid=\"book-title\">${book}</span>`\n  );\n\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"book-title\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<ul>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[36m<li\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"book-item\"\u001b[39m\n      \u001b[36m/>\u001b[39m\n      \u001b[0m\u001b[0m\n    \u001b[36m</ul>\u001b[39m\n    \u001b[0m\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).toBeInTheDocument()\u001b[22m\n\n\u001b[31mreceived\u001b[39m value must be an HTMLElement or an SVGElement.\nReceived has type:  Null\nReceived has value: \u001b[31mnull\u001b[39m"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const Title = (title) => (\n    <span data-testid=\"book-title\">{title}</span>\n  );\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render Title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/snippets_sample4_c7d89f1ba36c63e6/Component.svelte:11:4 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const bookTitles = [\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ];\n\n  const title = (book) => (\n    <span data-testid=\"book-title\">\n      {book}\n    </span>\n  );\n</script>\n\n<ul>\n  {#each bookTitles as title}\n    <li data-testid=\"book-item\">\n      {@render title(title)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/snippets_sample5_eab7df5854869845/Component.svelte:11:4 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  {$render snippet}\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {$render snippet book}\n    </li>\n  {:else}\n    No books available\n  {/each}\n</ul>\n\nsnippet :: (book) => {\n  <span data-testid=\"book-title\">{book}</span>;\n}",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/snippets_sample6_1bca5c69af43b7d2/Component.svelte:6:11 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n\n  function Title({ title }) {\n    return `<span data-testid=\"book-title\">{title}</span>`;\n  }\n</script>\n\n<ul>\n  {@render books.map((book) => <li data-testid=\"book-item\">{Title({ title: book })}</li>)}\n</ul>",
        "success": false,
        "errors": [
          "/svelte-bench/tmp/samples/openrouter/snippets_sample7_5809d2e421ceac4a/Component.svelte:12:31 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<ul>\n  <li data-testid=\"book-item\"><span data-testid=\"book-title\">The Lord of the Rings</span></li>\n  <li data-testid=\"book-item\"><span data-testid=\"book-title\">To Kill a Mockingbird</span></li>\n  <li data-testid=\"book-item\"><span data-testid=\"book-title\">1984</span></li>\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = [\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ];\n\n  const title = (bookTitle) => `\n    <span data-testid=\"book-title\">${bookTitle}</span>\n  `;\n</script>\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@html title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2025-12-08T17:35:43.937Z"
  }
]