[ { "testName": "counter", "provider": "Ollama", "modelId": "hf.co/naren-1219/SvelteMind:F16", "numSamples": 10, "numCorrect": 1, "pass1": 0.09999999999999998, "pass10": 1, "context": { "used": false, "content": "" }, "samples": [ { "index": 0, "code": "\n\n\n

\n \n {count}\n \n

", "success": false, "errors": [ "Unable to find an element by: [data-testid=\"count-value\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[36m\u001b[39m\n \u001b[36m\u001b[39m\n \u001b[0m \u001b[0m\n \u001b[36m\u001b[39m\n \u001b[0m0\u001b[0m\n \u001b[36m\u001b[39m\n \u001b[0m \u001b[0m\n \u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[0m\u001b[0m\n \u001b[36m\u001b[39m\n\u001b[36m\u001b[39m", "Unable to find an element by: [data-testid=\"increment-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[0m\u001b[0m\n \u001b[36m\u001b[39m\n\u001b[36m\u001b[39m", "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[0m\u001b[0m\n \u001b[36m\u001b[39m\n\u001b[36m\u001b[39m" ], "temperature": 0 }, { "index": 1, "code": "\n\n\n

\n \n {count}\n \n

", "success": false, "errors": [ "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[0m\u001b[0m\n \u001b[36m\u001b[39m\n\u001b[36m\u001b[39m" ] }, { "index": 2, "code": "\n\n\n

\n \n {count}\n \n

", "success": false, "errors": [ "$state(...) is not a function\n\n\tin \n", "$state(...) is not a function\n\n\tin \n", "$state(...) is not a function\n\n\tin \n", "$state(...) is not a function\n\n\tin \n" ] }, { "index": 3, "code": "\n\n\n

\n \n {count}\n \n

\n", "success": false, "errors": [ "Unable to find an element by: [data-testid=\"count-value\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[0m\u001b[0m\n \u001b[36m\u001b[39m\n\u001b[36m\u001b[39m" ] }, { "index": 4, "code": "\n\n\n

\n \n {count}\n \n

", "success": false, "errors": [ "Unable to find an element by: [data-testid=\"count-value\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[0m\u001b[0m\n \u001b[36m\u001b[39m\n\u001b[36m\u001b[39m" ] }, { "index": 5, "code": "\n\n\n

\n \n {count}\n \n

", "success": false, "errors": [ "Unable to find an element by: [data-testid=\"decrement-button\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[0m\u001b[0m\n \u001b[36m\u001b[39m\n\u001b[36m\u001b[39m" ] }, { "index": 6, "code": "\n\n\n\n{count}\n\nThis component:\n- Uses `$$state` for reactive state management\n- Provides increment and decrement functionality\n- Displays current count using `` element\n- Uses specific test IDs for component testing", "success": false, "errors": [ "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:13:32 `` was left open\nhttps://svelte.dev/e/element_unclosed" ] }, { "index": 7, "code": "\n\n\n

\n \n {count}\n \n

\n", "success": false, "errors": [ "$state(...) is not a function\n\n\tin \n", "$state(...) is not a function\n\n\tin \n", "$state(...) is not a function\n\n\tin \n", "$state(...) is not a function\n\n\tin \n" ] }, { "index": 8, "code": "\n\n\n

\n \n {count}\n \n

", "success": true, "errors": [] }, { "index": 9, "code": "\n\nHere's the complete Counter component:\n\n\n{count}\n\n", "success": false, "errors": [ "/Users/k/Documents/GitHub/svelte-bench/tmp/samples/ollama/Component.svelte:5:9 The $ prefix is reserved, and cannot be used for variables and imports\nhttps://svelte.dev/e/dollar_prefix_invalid" ] } ], "timestamp": "2026-09-21T10:30:20.732Z", "tps": 11.85, "tpsDetails": { "testName": "counter", "evalCount": 63, "evalDurationNs": 5315653000, "promptEvalCount": 279, "promptEvalDurationNs": 1200825000, "promptTps": 232.34, "loadDurationNs": 2400767, "measuredAt": "2026-09-21T13:26:29.397Z" } }, { "testName": "derived", "provider": "Ollama", "modelId": "hf.co/naren-1219/SvelteMind:F16", "numSamples": 10, "numCorrect": 6, "pass1": 0.5999999999999999, "pass10": 1, "context": { "used": false, "content": "" }, "samples": [ { "index": 0, "code": "\n\n\n

Number: {number}

Doubled: {doubled}

\n \n

", "success": true, "errors": [], "temperature": 0 }, { "index": 1, "code": "\n\n\n

Number: {number}

Doubled: {doubled}

\n \n

", "success": true, "errors": [] }, { "index": 2, "code": "\n\n\n

Number: {number}

Doubled: {doubled}

\n \n

", "success": true, "errors": [] }, { "index": 3, "code": "\n\n\n", "success": false, "errors": [ "Unable to find an element by: [data-testid=\"number-value\"]\n\nIgnored nodes: comments, script, style\n\u001b[36m\u001b[39m\n \u001b[36m

\u001b[39m\n \u001b[36m