[
  {
    "testName": "counter",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 4,
    "pass1": 0.4,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n</script>\n<button data-testid=\"decrement-button\" onclick={() => count -= 1}>-</button>\n<span data-testid=\"count-value\">{count}</span>\n<button data-testid=\"increment-button\" onclick={() => count += 1}>+</button>\n<style>\n.counter {\n  /* Add your styles here */\n}\n</style>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n</script>\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" onclick={() => count -= 1}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" onclick={() => count += 1}>+</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n</script>\n<button data-testid=\"decrement-button\" onclick={() => count -= 1}>-</button>\n<span data-testid=\"count-value\">{count}</span>\n<button data-testid=\"increment-button\" onclick={() => count += 1}>+</button>\nThis component implements a simple counter with increment and decrement functionality, using Svelte's built-in reactive state management.",
        "success": true,
        "errors": []
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n</script>\n<div class=\"counter\">\n  <button onclick={() => count -= 1}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button onclick={() => count += 1}>+</button>\n</div>\nThis component creates a reactive count, increments it, and decrements it, with visibility through the count value element.",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"counter\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m-\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<span\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"count-value\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m0\u001b[0m\n      \u001b[36m</span>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m+\u001b[0m\n      \u001b[36m</button>\u001b[39m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m This component creates a reactive count, increments it, and decrements it, with visibility through the count value element.\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"increment-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"counter\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m-\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<span\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"count-value\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m0\u001b[0m\n      \u001b[36m</span>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m+\u001b[0m\n      \u001b[36m</button>\u001b[39m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m This component creates a reactive count, increments it, and decrements it, with visibility through the count value element.\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"counter\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m-\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<span\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"count-value\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m0\u001b[0m\n      \u001b[36m</span>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m+\u001b[0m\n      \u001b[36m</button>\u001b[39m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m This component creates a reactive count, increments it, and decrements it, with visibility through the count value element.\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<div\u001b[39m\n      \u001b[33mclass\u001b[39m=\u001b[32m\"counter\"\u001b[39m\n    \u001b[36m>\u001b[39m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m-\u001b[0m\n      \u001b[36m</button>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<span\u001b[39m\n        \u001b[33mdata-testid\u001b[39m=\u001b[32m\"count-value\"\u001b[39m\n      \u001b[36m>\u001b[39m\n        \u001b[0m0\u001b[0m\n      \u001b[36m</span>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<button>\u001b[39m\n        \u001b[0m+\u001b[0m\n      \u001b[36m</button>\u001b[39m\n    \u001b[36m</div>\u001b[39m\n    \u001b[0m This component creates a reactive count, increments it, and decrements it, with visibility through the count value element.\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n</script>\n\n<button data-testid=\"decrement-button\" onClick={() => count -= 1}>\n  -\n</button>\n<span data-testid=\"count-value\">{count}</span>\n<button data-testid=\"increment-button\" onClick={() => count += 1}>\n  +\n</button>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<!--\n  Counter Component\n  Displays a count with increment and decrement buttons.\n-->\n<script>\n  import { $state } from 'svelte';\n</script>\n<button data-testid=\"decrement-button\">-</button>\n<span data-testid=\"count-value\">{count}</span>\n<button data-testid=\"increment-button\">+</button>\n<style>\n  .counter {\n    display: flex;\n    gap: 8px;\n  }\n</style>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:8:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<template>\n  <div class=\"counter\">\n    <button data-testid=\"decrement-button\">-</button>\n    <span data-testid=\"count-value\">0</span>\n    <button data-testid=\"increment-button\">+</button>\n  </div>\n</template>\n\n<script>\nlet count = $state(0);\n</script>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"count-value\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"increment-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\">-</button>\n  <span data-testid=\"count-value\">0</span>\n  <button data-testid=\"increment-button\">+</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  -1\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  2\u001b[39m\nReceived:\n\u001b[31m  0\u001b[39m"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { onMount } from 'svelte';\nimport { writable } from 'svelte/store';\n\nconst countStore = writable(0);\n\nonMount(() => {\n  countStore = writable(0);\n});\n\nfunction decrement() {\n  countStore.set(countStore.value - 1);\n}\n\nfunction increment() {\n  countStore.set(countStore.value + 1);\n}\n</script>\n\n<button onclick={decrement}>-</button>\n<span>{countStore.value}</span>\n<button onclick={increment}>+</button>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:10:2 Cannot assign to constant\nhttps://svelte.dev/e/constant_assignment"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet count = $state(0);\n</script>\n<div class=\"counter\">\n  <button data-testid=\"decrement-button\" onclick={() => count -= 1}>-</button>\n  <span data-testid=\"count-value\">{count}</span>\n  <button data-testid=\"increment-button\" onclick={() => count += 1}>+</button>\n</div>\nThis Svelte component uses Svelte's `$state` for reactive data and implements a basic counter with increment and decrement functionality.",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "derived",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 2,
    "pass1": 0.20000000000000007,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived } from 'svelte';\nlet number = $state(5);\nlet doubled = $derived(number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $derived(() => number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 10\u001b[39m\nReceived:\n\u001b[31m  Doubled: () => get(number) * 2\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 12\u001b[39m\nReceived:\n\u001b[31m  Doubled: () => get(number) * 2\u001b[39m"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $derived(() => number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number += 1}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 10\u001b[39m\nReceived:\n\u001b[31m  Doubled: () => get(number) * 2\u001b[39m",
          "\u001b[2mexpect(\u001b[22m\u001b[31melement\u001b[39m\u001b[2m).toHaveTextContent()\u001b[22m\n\nExpected element to have text content:\n\u001b[32m  Doubled: 12\u001b[39m\nReceived:\n\u001b[31m  Doubled: () => get(number) * 2\u001b[39m"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $derived, $state } from 'svelte';\nlet number = $state(5);\nlet doubled = $derived(() => number * 2);\n</script>\n<div>\n<p data-testid=\"number-value\">Number: {number}</p>\n<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n<button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $derived(number * 2);\n\nfunction increment() {\n  number += 1;\n}\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={increment}>Increment</button>\n</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $derived(number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>\nNote: This component uses `$derived` to automatically update the doubled value whenever the number changes.",
        "success": true,
        "errors": []
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived } from 'svelte';\nlet number = $state(5);\nlet doubled = $derived(() => number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived } from 'svelte';\nlet number = $state(5);\nlet doubled = $derived(() => number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<svelte:fragment>\n  <div>\n    <p data-testid=\"number-value\">Number: {number}</p>\n    <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n    <button data-testid=\"increment-button\" onclick={() => number += 1}>Increment</button>\n  </div>\n</svelte:fragment>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:3:0 `<svelte:fragment>` must be the direct child of a component\nhttps://svelte.dev/e/svelte_fragment_invalid_placement"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived } from 'svelte';\nlet number = $state(5);\nlet doubled = $derived(() => number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "derived-by",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived.by } from 'svelte';\nlet text = $state('');\nlet wordCount = $derived.by(() => text.split(' ').length);\nlet charCount = $derived.by(() => text.length);\nlet isLong = $derived.by(() => wordCount > 15);\n</script>\n<div>\n  <input type=\"text\" bind:value={text} />\n  <button onclick={() => text = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">{wordCount}</p>\n    <p data-testid=\"char-count\">{charCount}</p>\n    <p data-testid=\"length-indicator\">{isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:25 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { derived, $state } from 'svelte';\n\nlet text = $state('');\n\nconst countWords = () => text.split(/\\s+/).length;\nconst countChars = () => text.length;\nconst isLong = () => countWords() > 15;\n\n$derived(by: () => ({\n  wordCount: countWords(),\n  charCount: countChars(),\n  isLong: isLong()\n}));\n</script>\n\n<div>\n  <input data-testid=\"text-input\" bind:value={text} />\n  <button data-testid=\"clear-button\" onClick={() => text = ''}>\n    Clear\n  </button>\n  <div>\n    <p data-testid=\"word-count\">{wordCount}</p>\n    <p data-testid=\"char-count\">{charCount}</p>\n    <p data-testid=\"length-indicator\">{isLong() ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:11 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived } from 'svelte';\nlet { value, set } = $state('');\n$derived.by(() => {\n  const wordCount = value.split(/\\s+/).length;\n  const charCount = value.length;\n  const isLong = wordCount > 15;\n  return {\n    wordCount,\n    charCount,\n    isLong\n  };\n});\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={value} />\n  <button data-testid=\"clear-button\" onclick={() => value = ''}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\" class={isLong ? 'long-text' : 'short-text'}>Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived } from 'svelte';\nconst text = $state('');\nconst wordCount = $derived.by(() => text.trim().split(/\\s+/).length);\nconst charCount = $derived.by(() => text.length);\nconst isLong = $derived.by(() => wordCount > 15);\n</script>\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\">Clear</button>\n  <div>\n    <p data-testid=\"word-count\">{wordCount}</p>\n    <p data-testid=\"char-count\">{charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { svelte, $state, $derived, $effect, $effect.by } from 'svelte';\n\n  let text = $state('');\n\n  $effect.by(() => {\n    const wordCount = text.split(' ').length;\n    const charCount = text.length;\n    const isLong = wordCount > 15;\n\n    return {\n      wordCount,\n      charCount,\n      isLong\n    };\n  });\n</script>\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n<button data-testid=\"clear-button\" onClick={() => text = ''}>\n  Clear\n</button>\n<div>\n  <p data-testid=\"word-count\">Words: {wordCount}</p>\n  <p data-testid=\"char-count\">Characters: {charCount}</p>\n  <p data-testid=\"length-indicator\">{isLong ? 'Long text' : 'Short text'}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:46 Identifier '$effect' has already been declared\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived.by } from 'svelte';\nlet text = $state('');\n</script>\n\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\n<div>\n  <p data-testid=\"word-count\">{text.split(/\\s+/).length}</p>\n  <p data-testid=\"char-count\">{text.length}</p>\n  <p data-testid=\"length-indicator\">\n    {text.split(/\\s+/).length > 15 ? 'Long text' : 'Short text'}\n  </p>\n  <button data-testid=\"clear-button\" onclick={() => text = ''}>Clear</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:25 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<template>\n  <div>\n    <input type=\"text\" bind:value={text} />\n    <button onclick={() => text = ''}>Clear</button>\n    <p data-testid=\"word-count\">{words}</p>\n    <p data-testid=\"char-count\">{chars}</p>\n    <p data-testid=\"length-indicator\">{length > 15 ? 'Long' : 'Short'}</p>\n  </div>\n</template>\n\n<script>\nlet text = $state('');\nlet words = $derived.by(() => text.split(/\\s+/).length);\nlet chars = $derived.by(() => text.length);\nlet length = $derived.by(() => words > 15);\n</script>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"word-count\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"text-input\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"text-input\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived.by } from 'svelte';\nlet text = $state('');\nlet wordCount = $derived.by(() => text.split(' ').length);\nlet charCount = $derived.by(() => text.length);\nlet isLong = $derived.by(() => wordCount > 15);\n</script>\n\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n<button data-testid=\"clear-button\" onclick={() => text = ''}>\n  Clear\n</button>\n<div>\n  <p data-testid=\"word-count\">{wordCount}</p>\n  <p data-testid=\"char-count\">{charCount}</p>\n  <p data-testid=\"length-indicator\">{isLong ? 'Long text' : 'Short text'}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:25 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived.by, $effect } from 'svelte';\nimport { writable } from 'svelte/store';\n\nlet text = $state('');\n\n$effect(() => {\n  wordCount = $derived.by(() => {\n    const words = text.split(/\\s+/);\n    return words.length;\n  });\n  charCount = $derived.by(() => {\n    return text.length;\n  });\n  isLong = $derived.by(() => {\n    return text.length > 15;\n  });\n});\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <button data-testid=\"clear-button\" onclick={() => (text = '')}>Clear</button>\n  <div>\n    <p data-testid=\"word-count\">Words: {wordCount}</p>\n    <p data-testid=\"char-count\">Characters: {charCount}</p>\n    <p data-testid=\"length-indicator\">Status: {isLong ? 'Long text' : 'Short text'}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:25 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $derived } from 'svelte';\nlet text = $state('');\n\n$derived.by(() => {\n  const words = text.split(' ').length;\n  const chars = text.length;\n  const isLong = words > 15;\n  return { words, chars, isLong };\n});\n</script>\n\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n\n<div>\n  <p data-testid=\"word-count\">\n    Words: {derived.words}\n  </p>\n  <p data-testid=\"char-count\">\n    Characters: {derived.chars}\n  </p>\n  <p data-testid=\"length-indicator\">\n    Status: {derived.isLong ? 'Long text' : 'Short text'}\n  </p>\n</div>\n\n<button data-testid=\"clear-button\" onclick={() => text = ''}>\n  Clear\n</button>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "each",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet characters = ['Jerry', 'Elaine', 'Kramer'];\nlet addGeorge = () => {\n  characters.push('George');\n};\n</script>\n\n<ul data-testid=\"characters-list\">\n{#each characters as character}\n<li data-testid=\"character\">{character}</li>\n{/each}\n</ul>\n\n<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet characters = ['Jerry', 'Elaine', 'Kramer'];\n</script>\n<ul data-testid=\"characters-list\">\n{#each characters as character}\n<li data-testid=\"character\">{character}</li>\n{/each}\n<button data-testid=\"add-george-button\" onclick={() => characters.push('George')}>\nAdd George\n</button>\n</ul>",
        "success": false,
        "errors": [
          "expected 3 to be 4 // Object.is equality"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { onMount } from 'svelte';\nlet characters = ['Jerry', 'Elaine', 'Kramer'];\n\nonMount(() => {\n  addGeorge();\n});\n\nfunction addGeorge() {\n  characters.push('George');\n  renderList();\n}\n</script>\n\n{#each characters as character}\n<li data-testid=\"character\">{character}</li>\n{/each}\n\n<button data-testid=\"add-george-button\" onclick={() => addGeorge()}>\n  Add George\n</button>",
        "success": false,
        "errors": [
          "renderList is not defined\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n",
          "renderList is not defined\n\n\tin $effect\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { writable, on } from 'svelte';\n  let count = writable(0);\n  on(count, (number) => console.log(`count is ${number}`));\n</script>\n\n{#each count as number}\n<p>{number}</p>\n{/each}\n<button onclick={() => count.update((n) => n + 1)}>increment</button>",
        "success": false,
        "errors": [
          "writable is not a function\n\n\tin <unknown>\n",
          "writable is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet characters = ['Jerry', 'Elaine', 'Kramer'];\nlet addGeorge = () => {\n  characters.push('George');\n  updateCharacterList();\n};\n</script>\n\n<ul data-testid=\"characters-list\">\n{#each characters as character}\n<li data-testid=\"character\">{character}</li>\n{/each}\n</ul>\n<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>\n\n{#if !characters.length}\n<div data-testid=\"no-george\">No Georges yet!</div>\n{/if}\n\n{#if characters.includes('George')}\n<div data-testid=\"george\">George is one of us!</div>\n{/if}",
        "success": false,
        "errors": [
          "Cannot convert object to primitive value"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n---\n---crier",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"character\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m---\n---crier\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"character\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m---\n---crier\u001b[0m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { onMount } from 'svelte';\nonMount(async () => {\n  const response = await fetch('/path/to/data.json');\n  const data = await response.json();\n  data.forEach((item) => {\n    render(item);\n  });\n});\n</script>",
        "success": false,
        "errors": [
          "Cannot convert object to primitive value"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet characters = $state(['Jerry', 'Elaine', 'Kramer']);\nfunction addGeorge() {\n  characters.push('George');\n}\n</script>\n<ul data-testid=\"characters-list\">\n{#each characters as character}\n<li data-testid=\"character\">{character}</li>\n{/each}\n</ul>\n<button data-testid=\"add-george-button\" onclick={addGeorge}>Add George</button>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet characters = ['Jerry', 'Elaine', 'Kramer'];\nlet georgeCount = 0;\n\nfunction addGeorge() {\n  characters.push('George');\n  georgeCount += 1;\n}\n</script>\n\n<div>\n  <ul data-testid=\"characters-list\">\n    {#each characters as char}\n    <li data-testid=\"character\">{char}</li>\n    {/each}\n  </ul>\n  <button data-testid=\"add-george-button\" onclick={addGeorge}>Add George {georgeCount}</button>\n</div>\nThis solution meets the task requirements:\n- Uses `{#each}` block with character array\n- Adds a button that increments georgeCount and pushes 'George' to array\n- Ensures each list item displays just the character name\n- Provides comprehensive data-testid attributes for testing",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:22:14 Expected whitespace\nhttps://svelte.dev/e/expected_whitespace"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\nexport const App = {\n  data() {\n    return {\n      characters: [\"Jerry\", \"Elaine\", \"Kramer\"]\n    };\n  },\n  method: function (button) {\n    this.characters.push(\"George\");\n  },\n  render() {\n    return {\n      components: {\n        \"ul:characters-list\": { each: this.characters, template: (item) => item }\n      }\n    };\n  }\n};",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 Expected token }\nhttps://svelte.dev/e/expected_token"
        ]
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "effect",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $state, $effect } from 'svelte';\nlet number = $state(5);\nlet doubled = $effect(() => number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<template>\n  <div>\n    <p data-testid=\"number-value\">Number: {number}</p>\n    <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n    <button data-testid=\"increment-button\" onclick={() => incrementNumber()}>Increment</button>\n  </div>\n</template>\n\n<script>\nexport let number = $state(5);\n\nconst doubled = $effect(() => {\n  return number * 2;\n});\n\nconst incrementNumber = () => {\n  number += 1;\n};\n</script>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:12:0 Cannot export state from a module if it is reassigned. Either export a function returning the state value or only mutate the state value's properties\nhttps://svelte.dev/e/state_invalid_export"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\">Increment</button>\n</div>",
        "success": false,
        "errors": [
          "number is not defined\n\n\tin <unknown>\n",
          "number is not defined\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $effect(() => number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:14 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let number = $state(5);\n  let doubled = $effect(() => number * 2);\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:16 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<template>\n  <div>\n    <p data-testid=\"number-value\">Number: {number}</p>\n    <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n    <button data-testid=\"increment-button\">Increment</button>\n  </div>\n</template>\n<script>\nlet number = $state(5);\nlet doubled = $effect(() => number * 2);\n</script>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:12:14 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $effect } from 'svelte';\nlet number = $state(5);\n\n$effect(() => {\n  doubled = number * 2;\n});\n</script>\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $effect(() => number * 2);\n</script>\n\n<div>\n<p data-testid=\"number-value\">Number: {number}</p>\n<p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n<button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:14 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $state, $effect } from 'svelte';\n  \n  const number = $state(5);\n  const doubled = $effect(() => number * 2);\n  \n  $effect(() => doubled = number * 2);\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:11 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet number = $state(5);\nlet doubled = $effect(() => {\n  return number * 2;\n});\n</script>\n\n<div>\n  <p data-testid=\"number-value\">Number: {number}</p>\n  <p data-testid=\"doubled-value\">Doubled: {doubled}</p>\n  <button data-testid=\"increment-button\" onclick={() => number++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:5:14 `$effect()` can only be used as an expression statement\nhttps://svelte.dev/e/effect_invalid_placement"
        ]
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "hello-world",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 5,
    "pass1": 0.5,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { onMount } from 'svelte';\n</script>\n<style>\n.greeting {\n  color: blue;\n  margin: 1rem;\n  display: flex;\n  justify-content: center;\n}\n</style>\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n<script>\nonMount(() => {\n  // Add any necessary setup or cleanup here\n});\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<template>\n  <div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n</template>\n<style>\n.greeting {\n  color: blue;\n  margin: 2rem;\n}\n</style>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { onMount } from 'svelte';\n</script>\n<style>\n.greeting {\n  color: blue;\n  margin: 2rem;\n}\n</style>\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n<script>\nonMount(() => {\n  // Custom setup code here\n});\n</script>\nNote: The `onMount` hook is included for completeness, but in a real-world scenario, you might not need it.",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<!--\n  This is the HelloWorld component\n-->\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<details>\n<summary>Sample Solution</summary>\n<code>\n<div class=\"greeting\" style=\"display: flex; justify-content: center; margin: 1rem;\">\nHello, World!\n</div>\n</code>\n</details>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<details>\u001b[39m\n      \u001b[36m<summary>\u001b[39m\n        \u001b[0mSample Solution\u001b[0m\n      \u001b[36m</summary>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<code>\u001b[39m\n        \u001b[36m<div\u001b[39m\n          \u001b[33mclass\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n          \u001b[33mstyle\u001b[39m=\u001b[32m\"display: flex; justify-content: center; margin: 1rem;\"\u001b[39m\n        \u001b[36m>\u001b[39m\n          \u001b[0mHello, World!\u001b[0m\n        \u001b[36m</div>\u001b[39m\n      \u001b[36m</code>\u001b[39m\n    \u001b[36m</details>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"greeting\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<details>\u001b[39m\n      \u001b[36m<summary>\u001b[39m\n        \u001b[0mSample Solution\u001b[0m\n      \u001b[36m</summary>\u001b[39m\n      \u001b[0m \u001b[0m\n      \u001b[36m<code>\u001b[39m\n        \u001b[36m<div\u001b[39m\n          \u001b[33mclass\u001b[39m=\u001b[32m\"greeting\"\u001b[39m\n          \u001b[33mstyle\u001b[39m=\u001b[32m\"display: flex; justify-content: center; margin: 1rem;\"\u001b[39m\n        \u001b[36m>\u001b[39m\n          \u001b[0mHello, World!\u001b[0m\n        \u001b[36m</div>\u001b[39m\n      \u001b[36m</code>\u001b[39m\n    \u001b[36m</details>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<!--\nThis is an empty Svelte component that imports the necessary Svelte runtime module and creates a div with the required data-testid and class styling.\n-->\n\n<script>\nimport { createApp } from 'svelte';\nconst app = createApp();\n</script>\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n\n<style>\n.greeting {\n  color: blue;\n  text-align: center;\n  margin: 1rem;\n}\n</style>",
        "success": false,
        "errors": [
          "createApp is not a function\n\n\tin <unknown>\n",
          "createApp is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\" style=\"color: blue; margin: 2rem;\">Hello, World!</div>",
        "success": true,
        "errors": []
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<component>\n<!-- Your component code goes here -->\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n<style>\n.greeting {\n  color: blue;\n  margin: 1rem;\n  text-align: center;\n}\n</style>\n</component>",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { onMount } from 'svelte';\nonMount(() => {\n  console.log('component mounted');\n});\n</script>\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>\n<style>\n.greeting {\n  color: blue;\n  margin: 0.5em;\n}\n</style>",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<div data-testid=\"greeting\" class=\"greeting\">Hello, World!</div>",
        "success": true,
        "errors": []
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "inspect",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 0,
    "pass1": 0,
    "pass10": 0,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { inspect, trace } from 'svelte';\nlet text = $state('Hello world');\n</script>\n<input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>",
        "success": false,
        "errors": [
          "expected \"log\" to be called at least once",
          "expected \"log\" to be called at least once",
          "expected \"log\" to be called at least once"
        ],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $inspect } from 'svelte';\nlet text = $state('Hello world');\n$inspect(text).with(() => {\n  const trace = $inspect.trace();\n  trace.on((type, ...args) => {\n    switch (type) {\n      case 'change':\n        console.log(`input changed from ${args[0]} to ${args[1]}`);\n        break;\n      case 'update':\n        console.log(`text value changed from ${args[0]} to ${args[1]}`);\n        break;\n    }\n  });\n});\n</script>\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { inspect } from 'svelte/inspect';\nlet text = $state('Hello world');\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n  <pre data-testid=\"char-count\" />\n</div>\n\n<script>\n$inspect(text).with((changes, diff) => {\n  const count = changes.reduce((total, change) => {\n    total += change.length;\n    return total;\n  }, 0);\n  $inspect.trace(`updated ${text.length} characters`);\n  return count;\n});\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:15:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet text = $state('Hello world');\n$inspect(text).with((val) => {\n  const count = text.length;\n  $effect(() => {\n    $state.text = text;\n  });\n});\n</script>\n<input type=\"text\" bind:value={text} />\n<p>Current text: \"{text}\"</p>\n<p>Character count: {text.length}</p>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:8:4 `$state.text` is not a valid rune\nhttps://svelte.dev/e/rune_invalid_name"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $inspect } from 'svelte';\n</script>\n\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>\n<script>\nlet text = $state('Hello world');\n$inspect(text).with((change) => {\n  // Custom update handling\n  text = change.newValue;\n});\n\n$inspect(text).trace((change) => {\n  // Trace log when text changes\n  $state('Character count updated to: ' + change.newValue.length);\n});\n</script>\nExplanation:\n1. The component creates a `$state` instance for the input text\n2. Uses `$inspect` to log and track changes to the text\n3. Implements a custom callback for tracking updates\n4. Uses `$inspect.trace()` to log character count updates\n5. Displays the input and character count with data-testid attributes",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:12:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { inspect } from 'svelte';\nlet text = $state('Hello world');\nlet count = $state(text.length);\n$effect(() => {\n  count = text.length;\n});\n\n$inspect(text).with((value) => {\n  count = value.length;\n});\n$inspect.trace(() => {\n  count = text.length;\n});\n</script>\n<div>\n  <input data-testid=\"text-input\" type=\"text\" bind:value={text} />\n  <p data-testid=\"text-value\">Current text: \"{text}\"</p>\n  <p data-testid=\"char-count\">Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n",
          "$inspect(...) is not a function\n\n\tin <unknown>\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<!-- svelte:component { data: { $inspect: { value: text, trace: () => console.log(`Updated from ${text} to ${text}`) }} } -->",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"text-value\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"text-input\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"text-input\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"text-input\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { inspect } from 'svelte/inspect';\nlet text = $state('Hello world');\n</script>\n<textarea data-testid=\"text-input\" bind:value={text}>\n</textarea>\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>\n<script>\n$inspect(text).with((prev, curr) => {\n  console.log(`Updated from ${prev} to ${curr}`);\n});\n</script>\n<script>\n$inspect.trace(() => {\n  console.log(`Input changed to \"${text}\"`);\n});\n</script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:11:0 A component can have a single top-level `<script>` element and/or a single top-level `<script module>` element\nhttps://svelte.dev/e/script_duplicate"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\n  let text = $state('Hello world');\n  $inspect(text).with(() => {\n    console.log(`updated to '${text}'`);\n  });\n</script>\n<textarea data-testid=\"text-input\" bind:value={text} rows={8}>\n  Current text: {text}\n</textarea>\n<p data-testid=\"text-value\">Current text: \"{text}\"</p>\n<p data-testid=\"char-count\">Character count: {text.length}</p>",
        "success": false,
        "errors": [
          "expected false to be true // Object.is equality",
          "expected false to be true // Object.is equality"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { inspect } from 'svelte/inspect';\nlet text = $state('Hello world');\n\n$inspect(text).with(() => {\n  const charCount = text.length;\n  $inspect.trace(() => console.log(`Text changed to '${text}'; has ${charCount} characters`));\n});\n</script>\n\n<div>\n  <input type=\"text\" bind:value={text} />\n  <p>{text}</p>\n  <p>Character count: {text.length}</p>\n</div>",
        "success": false,
        "errors": [
          "\"./inspect\" is not exported under the conditions [\"browser\", \"node\", \"development\", \"svelte\", \"svelte\", \"svelte\", \"import\"] from package /Users/k/Documents/GitHub/svelte-bench/node_modules/svelte (see exports field in /Users/k/Documents/GitHub/svelte-bench/node_modules/svelte/package.json)"
        ]
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "props",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 1,
    "pass1": 0.09999999999999998,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet { name = 'World', count = 0, showDetails = false } = $props();\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" onclick={() => count++}>Increment</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<template>\n  <div>\n    <h1 data-testid=\"name-display\">{name}</h1>\n    <p data-testid=\"count-display\">Count: {count}</p>\n    <button data-testid=\"increment-button\" onclick={() => count++}>Increment</button>\n    {#if showDetails}\n      <div data-testid=\"details\">\n        <p>Name is {name}</p>\n        <p>Count is {count}</p>\n        <p>ShowDetails is {showDetails}</p>\n      </div>\n    {/if}\n  </div>\n</template>\n\n<script>\nlet { name = 'World', count = 0, showDetails = false } = $props();\n</script>",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"name-display\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"name-display\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"increment-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"details\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div />\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[36m<template />\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet { name = 'World', count = 0, showDetails = false } = $props();\nlet count = $state(count);\n</script>\n\n<div>\n  <h1>{name}</h1>\n  <p>Count: {count}</p>\n  <button onclick={() => count++}>Increment</button>\n\n  {#if showDetails}\n    <div>\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:5:4 Identifier 'count' has already been declared\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\n  import { $props } from 'svelte';\n  import { $state } from 'svelte';\n  import { increment } from 'svelte/transition';\n\n  let { name = 'World', count = 0, showDetails = false } = $props();\n  let count = $state(count);\n</script>\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" use:increment>{count + 1}</button>\n\n  {#if showDetails}\n    <div data-testid=\"details\">\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:9:6 Identifier 'count' has already been declared\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<template>\n  <div>\n    <h1 data-testid=\"name-display\">{{ name }}</h1>\n    <p data-testid=\"count-display\">Count: {{ count }}</p>\n    <button data-testid=\"increment-button\" onclick={() => count++}>Increment</button>\n    <div data-testid=\"details\" if:showDetails>\n      <p>Name is {{ name }}</p>\n      <p>Count is {{ count }}</p>\n      <p>ShowDetails is {{ showDetails }}</p>\n    </div>\n  </div>\n</template>\n\n<script>\n  export let name = 'World';\n  export let count = $state(0);\n  export let showDetails = $state(false);\n</script>",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:17:2 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $props } from 'svelte';\nlet {\n  name = 'World',\n  count = 0,\n  showDetails = false,\n  $state: countState = $state(0)\n} = $props();\n</script>\n\n<div>\n  <h1>{name}</h1>\n  <p>Count: {count}</p>\n  <button onclick={() => count++}>Increment</button>\n  <div {showDetails}>\n    <p>Name is {name}</p>\n    <p>Count is {count}</p>\n    <p>ShowDetails is {showDetails}</p>\n  </div>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $props } from 'svelte';\nlet {\n  name = 'World',\n  count = 0,\n  showDetails = false\n} = $props();\n</script>\n\n{#if showDetails}\n<div data-testid=\"details\">\n  <p>Name is {name}</p>\n  <p>Count is {count}</p>\n</div>\n{/if}\n\n<div>\n  <h1 data-testid=\"name-display\">{name}</h1>\n  <p data-testid=\"count-display\">Count: {count}</p>\n  <button data-testid=\"increment-button\" onclick={() => count++}>Increment</button>\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\nHere's the Svelte component code that demonstrates the `$props` rune:\n\n<script>\nexport let name = 'World';\nexport let count = 0;\nexport let showDetails = false;\n</script>\n\n<h1 data-testid=\"name-display\">{name}</h1>\n<p data-testid=\"count-display\">Count: {count}</p>\n<button data-testid=\"increment-button\" onclick={() => count++}>Increment</button>\n\n{#if showDetails}\n<div data-testid=\"details\">\n  <p>Name is {name}</p>\n  <p>Count is {count}</p>\n  <p>ShowDetails is {showDetails}</p>\n</div>\n{/if}\nThis component accepts the required props, provides a reactive count, and shows/hides details based on the `showDetails` prop.",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:6:0 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { $props, $state } from 'svelte';\nlet { name = 'World', count = 0, showDetails = false } = $props();\nlet details = $state('');\n\nconst increment = () => {\n  count++;\n  if (showDetails) {\n    details = `Name is ${name}, Count is ${count}, ShowDetails is ${showDetails}`;\n  }\n};\n\n$effect(() => {\n  if (showDetails) {\n    details = `Name is ${name}, Count is ${count}, ShowDetails is ${showDetails}`;\n  }\n});\n</script>\n\n<div>\n  <h1>{name}</h1>\n  <p>Count: {count}</p>\n  <button onclick={increment}>Increment</button>\n\n  {#if showDetails}\n    <div>\n      <p>Name is {name}</p>\n      <p>Count is {count}</p>\n      <p>ShowDetails is {showDetails}</p>\n    </div>\n  {:else}\n    <div style=\"display: none;\"></div>\n  {/if}\n</div>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:4:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid"
        ]
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\n<!--\nThis is a placeholder comment to demonstrate the placeholder syntax for component documentation.\nYou can remove this comment once you have created the actual component documentation.\n-->",
        "success": false,
        "errors": [
          "Unable to find an element by: [data-testid=\"name-display\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"name-display\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"increment-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m",
          "Unable to find an element by: [data-testid=\"details\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m<body>\u001b[39m\n  \u001b[36m<div />\u001b[39m\n  \u001b[36m<div>\u001b[39m\n    \u001b[0m\u001b[0m\n  \u001b[36m</div>\u001b[39m\n\u001b[36m</body>\u001b[39m"
        ]
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  },
  {
    "testName": "snippets",
    "provider": "Ollama",
    "modelId": "hf.co/naren-1219/SvelteMind:Q4_K_M",
    "numSamples": 10,
    "numCorrect": 3,
    "pass1": 0.30000000000000004,
    "pass10": 1,
    "context": {
      "used": false,
      "content": ""
    },
    "samples": [
      {
        "index": 0,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet books = ['The Lord of the Rings', 'To Kill a Mockingbird', '1984'];\n</script>\n\n<ul>\n  {#each books as book}\n  <li data-testid=\"book-item\">\n    {@render title(book)}\n  </li>\n  {/each}\n</ul>\n\n{#snippet title(title)}\n<span data-testid=\"book-title\">{title}</span>\n{/snippet}",
        "success": true,
        "errors": [],
        "temperature": 0
      },
      {
        "index": 1,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { title } from './snippets.svelte';\n</script>\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "Failed to resolve import \"./snippets.svelte\" from \"tmp/samples/ollama/Component.svelte\". Does the file exist?"
        ]
      },
      {
        "index": 2,
        "code": "<svelte:options runes={true} />\n\n<script>\nexport let bookTitles = ['The Lord of the Rings', 'To Kill a Mockingbird', '1984'];\n</script>\n\n{#each bookTitles as title}\n<li data-testid=\"book-item\">\n{@render title({ title })}\n</li>\n{/each}",
        "success": false,
        "errors": [
          "tmp/samples/ollama/Component.svelte:4:0 Cannot use `export let` in runes mode — use `$props()` instead\nhttps://svelte.dev/e/legacy_export_invalid"
        ]
      },
      {
        "index": 3,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet books = ['The Lord of the Rings', 'To Kill a Mockingbird', '1984'];\n</script>\n<ul>\n{@render books.map((book) => (\n  <li data-testid=\"book-item\">\n    <span data-testid=\"book-title\">{book}</span>\n  </li>\n))}\n</ul>\nNote: The snippet `title` is not explicitly used in this example, but it would be defined elsewhere in the component.",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:8:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      },
      {
        "index": 4,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet books = [\n  \"The Lord of the Rings\",\n  \"To Kill a Mockingbird\",\n  \"1984\"\n];\n</script>\n\n<ul>\n  {#each books as book}\n  <li data-testid=\"book-item\">\n    {@render title(book)}\n  </li>\n  {:else}\n  <li>Oops!</li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "title is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "title is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "title is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 5,
        "code": "<svelte:options runes={true} />\n\n<script>\n  const books = ['The Lord of the Rings', 'To Kill a Mockingbird', '1984'];\n</script>\n<ul>\n  {#each books as book}\n    <li data-testid=\"book-item\">\n      {@render title(book)}\n    </li>\n  {/each}\n</ul>",
        "success": false,
        "errors": [
          "title is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "title is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n",
          "title is not defined\n\n\tin <unknown>\n\tin Component.svelte\n\tin undefined\n"
        ]
      },
      {
        "index": 6,
        "code": "<svelte:options runes={true} />\n\n<snippet name=\"title\" description=\"Displays book titles\">\n  <span data-testid=\"book-title\">{this}</span>\n</snippet>\n<template>\n  <ul>\n    {@render title(\"The Lord of the Rings\")}\n    {@render title(\"To Kill a Mockingbird\")}\n    {@render title(\"1984\")}\n  </ul>\n</template>\n<@script>\n  let books = [\"The Lord of the Rings\", \"To Kill a Mockingbird\", \"1984\"];\n</@script>",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:1 Expected a valid element or component name. Components must have a valid variable name or dot notation expression\nhttps://svelte.dev/e/tag_invalid_name"
        ]
      },
      {
        "index": 7,
        "code": "<svelte:options runes={true} />\n\n<script>\nlet books = [\n  'The Lord of the Rings',\n  'To Kill a Mockingbird',\n  '1984'\n];\n</script>\n\n<ul>\n  {#each books as book}\n  <li data-testid=\"book-item\">\n    {@render title(book)}\n  </li>\n  {/each}\n</ul>\n\n{#snippet title(title)}\n  <span data-testid=\"book-title\">{title}</span>\n{/snippet}\n## Key Points:\n- The `books` array is hardcoded inside the component\n- The `title` snippet is defined to render a book title\n- The snippet is used to render each book title in the list",
        "success": true,
        "errors": []
      },
      {
        "index": 8,
        "code": "<svelte:options runes={true} />\n\n<script>\nimport { snippet } from 'svelte';\nlet books = [\n  \"The Lord of the Rings\",\n  \"To Kill a Mockingbird\",\n  \"1984\"\n];\n</script>\n<ul>\n  {#each books as book}\n  <li data-testid=\"book-item\">\n    {@render title(book)}\n  </li>\n  {/each}\n</ul>\n{#snippet title(title)}\n  <span data-testid=\"book-title\">{title}</span>\n{/snippet}",
        "success": true,
        "errors": []
      },
      {
        "index": 9,
        "code": "<svelte:options runes={true} />\n\nexport let books = [\n  'The Lord of the Rings',\n  'To Kill a Mockingbird',\n  '1984'\n];\n\nexport const title = (title: string) => {\n  return `<span data-testid=\"book-title\">${title}</span>`;\n};\n\nlet bookItems = books.map((book) => {\n  return `<li data-testid=\"book-item\">{@render title(book)}</li>`;\n});\n\nlet bookList = `<!-- Use {@render} with a snippet to dynamically render content --> <ul>${bookItems.join('')}</ul>`;\n\nexport default bookList;",
        "success": false,
        "errors": [
          "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:10:2 Unexpected token\nhttps://svelte.dev/e/js_parse_error"
        ]
      }
    ],
    "timestamp": "2026-09-21T10:36:15.118Z",
    "tps": 34.93,
    "tpsDetails": {
      "testName": "counter",
      "evalCount": 67,
      "evalDurationNs": 1918071000,
      "promptEvalCount": 279,
      "promptEvalDurationNs": 807997000,
      "promptTps": 345.3,
      "loadDurationNs": 1576784,
      "measuredAt": "2026-09-21T13:26:34.488Z"
    }
  }
]