[
  {
    "testName": "counter",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count--}\n  >-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count++}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count -= 1}\n  >-</button>\n  <span data-testid=\"count-value\" aria-live=\"polite\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count += 1}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n\n  .counter button {\n    padding: 0.5rem 1rem;\n    font-size: 1rem;\n    cursor: pointer;\n  }\n\n  .counter span {\n    min-width: 2rem;\n    text-align: center;\n    font-variant-numeric: tabular-nums;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count--}\n  >-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count++}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" onclick={() => count--} aria-label=\"Decrement\" type=\"button\">-</button>\n  <span data-testid=\"count-value\" aria-live=\"polite\">{count}</span>\n  <button data-testid=\"increment-button\" onclick={() => count++} aria-label=\"Increment\" type=\"button\">+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n\n  .counter button {\n    padding: 0.5rem 1rem;\n    font-size: 1rem;\n    cursor: pointer;\n  }\n\n  .counter span {\n    min-width: 2rem;\n    text-align: center;\n    font-variant-numeric: tabular-nums;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count--}\n  >-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count++}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count -= 1}\n  >-</button>\n  <span data-testid=\"count-value\" aria-live=\"polite\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count += 1}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n\n  .counter button {\n    padding: 0.5rem 1rem;\n    font-size: 1rem;\n    cursor: pointer;\n  }\n\n  .counter span {\n    min-width: 2ch;\n    text-align: center;\n    font-variant-numeric: tabular-nums;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count--}\n  >-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count++}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count -= 1}\n  >-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count += 1}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: inline-flex;\n    align-items: center;\n    gap: 1rem;\n  }\n\n  .counter button {\n    padding: 0.5rem 1rem;\n    font-size: 1rem;\n    cursor: pointer;\n  }\n\n  .counter span {\n    min-width: 2ch;\n    text-align: center;\n    font-variant-numeric: tabular-nums;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count--}\n  >-</button>\n  <span data-testid=\"count-value\" aria-live=\"polite\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count++}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n\n  .counter button {\n    padding: 0.5rem 1rem;\n    font-size: 1rem;\n    cursor: pointer;\n  }\n\n  .counter span {\n    min-width: 2ch;\n    text-align: center;\n    font-variant-numeric: tabular-nums;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button\n    type=\"button\"\n    data-testid=\"decrement-button\"\n    aria-label=\"Decrement\"\n    onclick={() => count--}\n  >-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    aria-label=\"Increment\"\n    onclick={() => count++}\n  >+</button>\n</div>\n\n<style>\n  .counter {\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "derived",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button type=\"button\" data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $derived(number * 2);\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "derived-by",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n\n    return {\n      wordCount,\n      charCount: text.length,\n      isLong: wordCount > 15\n    };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" aria-label=\"Text\" bind:value={text} />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  let stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n    const charCount = text.length;\n\n    return {\n      wordCount,\n      charCount,\n      isLong: wordCount > 15\n    };\n  });\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    aria-label=\"Text to analyze\"\n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {stats.isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  let stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n    const charCount = text.length;\n\n    return {\n      wordCount,\n      charCount,\n      isLong: wordCount > 15\n    };\n  });\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    aria-label=\"Text to analyze\"\n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {stats.isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  let stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n\n    return {\n      wordCount,\n      charCount: text.length,\n      isLong: wordCount > 15\n    };\n  });\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    aria-label=\"Text to analyze\"\n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {stats.isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  let stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n    const charCount = text.length;\n    const isLong = wordCount > 15;\n\n    return { wordCount, charCount, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} aria-label=\"Text\" />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n\n    return {\n      wordCount,\n      charCount: text.length,\n      isLong: wordCount > 15\n    };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" aria-label=\"Text\" bind:value={text} />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  let stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n\n    return {\n      wordCount,\n      charCount: text.length,\n      isLong: wordCount > 15\n    };\n  });\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    aria-label=\"Text to analyze\"\n    bind:value={text}\n  />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">\n      Status: {stats.isLong ? 'Long text' : 'Short text'}\n    </p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  let stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n    const charCount = text.length;\n    const isLong = wordCount > 15;\n\n    return { wordCount, charCount, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} aria-label=\"Text\" />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  const stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n    const charCount = text.length;\n    const isLong = wordCount > 15;\n\n    return { wordCount, charCount, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} aria-label=\"Text\" />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('');\n\n  let stats = $derived.by(() => {\n    const trimmedText = text.trim();\n    const wordCount = trimmedText ? trimmedText.split(/\\s+/).length : 0;\n    const charCount = text.length;\n    const isLong = wordCount > 15;\n\n    return { wordCount, charCount, isLong };\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} aria-label=\"Text\" />\n  <button data-testid=\"clear-button\" type=\"button\" onclick={() => (text = '')}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">Words: {stats.wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {stats.charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {stats.isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "each",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    type=\"button\"\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    type=\"button\"\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    type=\"button\"\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    type=\"button\"\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let characters = $state([\"Jerry\", \"Elaine\", \"Kramer\"]);\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as character}\n      <li data-testid=\"character\">{character}</li>\n    {/each}\n  </ul>\n  <button\n    data-testid=\"add-george-button\"\n    onclick={() => characters.push(\"George\")}\n  >\n    Add George\n  </button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "effect",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(10);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(10);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(10);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <label>\n    Enter a number:\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(10);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <label>\n    Number\n    <input type=\"number\" bind:value={number} />\n  </label>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => number += 1}\n  >\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $state(0);\n\n  $effect(() => {\n    doubled = number * 2;\n  });\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>\n    Increment\n  </button>\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "hello-world",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  main {\n    display: grid;\n    min-height: 100vh;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 0.5rem;\n  }\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  :global(body) {\n    margin: 0;\n  }\n\n  main {\n    min-height: 100vh;\n    display: grid;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 0.5rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  :global(body) {\n    margin: 0;\n  }\n\n  main {\n    display: grid;\n    min-height: 100vh;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 0.5rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  main {\n    display: grid;\n    min-height: 100svh;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 0.5rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  :global(body) {\n    margin: 0;\n  }\n\n  main {\n    display: grid;\n    min-height: 100svh;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 1rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  :global(body) {\n    margin: 0;\n  }\n\n  main {\n    display: grid;\n    min-height: 100vh;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 0.5rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  :global(body) {\n    margin: 0;\n  }\n\n  main {\n    display: grid;\n    min-height: 100vh;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 0.5rem;\n  }\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<main>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</main>\n\n<style>\n  :global(body) {\n    margin: 0;\n  }\n\n  main {\n    display: grid;\n    min-height: 100svh;\n    place-items: center;\n  }\n\n  .greeting {\n    color: blue;\n    text-align: center;\n    margin: 0.5rem;\n  }\n</style>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "inspect",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 3,
    "pass1": 0.30000000000000004,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  let characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    if (type === 'update') {\n      console.log('Text updated:', value);\n    }\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Text effect:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  let charCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    if (type === 'update') {\n      console.log('Text updated:', value);\n    }\n  });\n\n  $effect(() => {\n    $inspect.trace('Text input effect');\n    console.log('Current input:', text);\n  });\n</script>\n\n<div>\n  <input\n    data-testid=\"text-input\"\n    type=\"text\"\n    aria-label=\"Text\"\n    bind:value={text}\n  />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {charCount}</p>\n</div>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nCurrent input: Hello wor…' to contain 'init'",
          "expected 'Hello world\\nCurrent input: Hello wor…' to contain 'init'",
          "expected 'Hello world\\nCurrent input: Hello wor…' to contain 'init'"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  const characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    if (type === 'update') {\n      console.log('Text updated:', value);\n    }\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Text effect:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  let characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    console.log(`Text ${type}:`, value);\n  });\n\n  $effect(() => {\n    $inspect.trace('Text change effect');\n    console.log('Current text:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  let characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    if (type === 'update') {\n      console.log('Text updated:', value);\n    }\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Text changed:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" aria-label=\"Text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nText changed: Hello worl…' to contain 'init'",
          "expected 'Hello world\\nText changed: Hello worl…' to contain 'init'",
          "expected 'Hello world\\nText changed: Hello worl…' to contain 'init'"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  let characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    if (type === 'update') {\n      console.log('Text updated:', value);\n    }\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Text effect:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  const characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    console.log(`Text ${type}:`, value);\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Input changed:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  let characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    if (type === 'update') {\n      console.log('Text updated:', value);\n    }\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Text effect:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  const characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    console.log(`Text ${type}:`, value);\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Input changed:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  let characterCount = $derived(text.length);\n\n  $inspect(text);\n\n  $inspect(text).with((type, value) => {\n    if (type === 'update') {\n      console.log('Text updated:', value);\n    }\n  });\n\n  $effect(() => {\n    $inspect.trace();\n    console.log('Text effect:', text);\n  });\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {characterCount}</p>\n</div>",
        "success": false,
        "errors": [
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'",
          "expected 'Hello world\\nText effect: Hello world…' to contain 'init'"
        ]
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "props",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let { name = \"World\", count = 0, showDetails = false } = $props();\n\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    data-testid=\"increment-button\"\n    type=\"button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  type Props = {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  };\n\n  let { name = \"World\", count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  interface Props {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  }\n\n  let { name = \"World\", count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  type Props = {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  };\n\n  let { name = \"World\", count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  type Props = {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  };\n\n  let { name = 'World', count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    data-testid=\"increment-button\"\n    type=\"button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  type Props = {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  };\n\n  let { name = \"World\", count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  type Props = {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  };\n\n  let { name = \"World\", count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    data-testid=\"increment-button\"\n    type=\"button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  type Props = {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  };\n\n  let { name = \"World\", count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  type Props = {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  };\n\n  let { name = 'World', count = 0, showDetails = false }: Props = $props();\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button data-testid=\"increment-button\" onclick={() => currentCount += 1}>\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let {\n    name = \"World\",\n    count = 0,\n    showDetails = false\n  }: {\n    name?: string;\n    count?: number;\n    showDetails?: boolean;\n  } = $props();\n\n  let currentCount = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">Hello, {name}!</h1>\n  <p data-testid=\"count-display\">Count: {currentCount}</p>\n  <button\n    type=\"button\"\n    data-testid=\"increment-button\"\n    onclick={() => currentCount += 1}\n  >\n    Increment\n  </button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {currentCount}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  },
  {
    "testName": "snippets",
    "provider": "OpenAI",
    "modelId": "gpt-6-astra",
    "numSamples": 10,
    "numCorrect": 10,
    "pass1": 1,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ]);\n</script>\n\n{#snippet title(bookTitle: string)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = $state([\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ]);\n</script>\n\n{#snippet title(bookTitle)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ]);\n</script>\n\n{#snippet title(bookTitle)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book (book)}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ]);\n</script>\n\n{#snippet title(bookTitle)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book (book)}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ]);\n</script>\n\n{#snippet title(bookTitle: string)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let books = $state([\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ]);\n</script>\n\n{#snippet title(bookTitle: string)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = $state([\n    'The Lord of the Rings',\n    'To Kill a Mockingbird',\n    '1984'\n  ]);\n</script>\n\n{#snippet title(bookTitle)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book (book)}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ]);\n</script>\n\n{#snippet title(bookTitle: string)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script lang=\"ts\">\n  let books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ]);\n</script>\n\n{#snippet title(bookTitle: string)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let books = $state([\n    \"The Lord of the Rings\",\n    \"To Kill a Mockingbird\",\n    \"1984\"\n  ]);\n</script>\n\n{#snippet title(bookTitle)}\n  <span data-testid=\"book-title\">{bookTitle}</span>\n{/snippet}\n\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-07T21:05:57.536Z"
  }
]