diff --git a/compiler/src/check.rs b/compiler/src/check.rs index 780b01aa..7697f960 100644 --- a/compiler/src/check.rs +++ b/compiler/src/check.rs @@ -338,6 +338,25 @@ fn snapshot_reads<'a>(n: Node<'a>, caught: bool, locals: &mut HashSet, o snapshot_reads(c, caught, locals, out); } } +/// AC-3a: the submit buttons of a form's markup (a ` + } + loading on(filter) fallback {

loading

} { +

{rows.length}

+ } + } + + + + { rename("form") }}> + + +
{ rename("form2") }}> + +
+} +"#; + let got: Vec = codes(src, false).iter().filter(|(_, c)| c == "GUESS_WHILE_LOADING").map(|(l, _)| *l).collect(); + // `out` and the second form: `detail` asks about `sel`, which `rename` reads; `rows` asks about `filter`, + // which the click never reads; `in` sits in detail's region, `off` and the first form are disabled on it + assert_eq!(got, vec![27, 32], "{:?}", codes(src, false)); + } + /// 08 CN-3d: in a client-only build, a reader the first load surely waits on needs a `loading` above it. #[test] fn first_load_unheld_in_client_only_builds() { diff --git a/eval/PREREG-desk-12.md b/eval/PREREG-desk-12.md new file mode 100644 index 00000000..6085b981 --- /dev/null +++ b/eval/PREREG-desk-12.md @@ -0,0 +1,75 @@ +# Round 12: round 11 again with five arms and twice the frontier runs + +Pre-registration, committed before any round 12 agent runs. Status: FROZEN (Ryan, 2026-11-11 "Go"). + +## Why + +Round 11 (`results/desk-2026-11-11`) replicated round 10's small-tier result. + +- **Small tier:** .rx had 3.8 defects per app and React 9.7. H11a holds, at p = 0.047. +- **Frontier tier:** every arm was between 2 and 3. With three runs per arm, H11c ("no worse than React") could not be shown: the interval was [−2.3, 2.3]. +- **XState:** a side check the same day (`results/xstate-2026-11-11`, not pre-registered) put small XState at 7.0, between .rx and React and below Foldkit's 13.2. + +Round 12 asks whether these hold with fresh agents. It adds XState as a fifth arm and doubles the frontier runs, so the frontier comparison has a chance to resolve. + +Ryan, 2026-11-11: replication now, maintenance next. Maintenance moves to round 13 and runs on round 12's apps. + +## Task and arms (as round 11 unless stated) + +**Task:** `tasks/desk9` with `brief-plain.md`, unchanged. Graded by `hidden.mjs` (77) and `hidden-overlap.mjs` (38), unchanged. + +**Arms:** +- **`rx`** with `--inline-docs` (`language.md`, `async.md`, `reference.md`). +- **`foldkit`** with `--inline-docs`. +- **`react`**. +- **`vanilla`**. +- **`xstate`**, new: React 19 with xstate 5.32.6 and @xstate/react 6.1.0, with no TanStack Query. It uses `--inline-docs`, which here means Stately's own v5 cheatsheet and @xstate/react's README in the prompt. Every Stately v5 docs page is readable in `docs/xstate`, from statelyai/docs at 7ee0c60. The checker is the same as React's. + +**Language and guide changes since round 11's freeze:** +- `guess` replaces `optimistic` (#47). +- Errors are cleared with `onFlush` (#47). +- A guess over an equation is allowed (#48). +- Destructuring works (#49). + +The guide follows all four. `rxc` is built from the commit that holds this file once it is frozen. Round 11's .rx desks use `optimistic`, so the gate below grades them with round 11's `rxc`, which isolates the Solid change. + +**Guide addition** (principle only, no task code): `async.md` says how to guess a prop. You guess over the child's own equation of it, never `props.x`. + +**Solid:** `solid-next` at `next` bcef1041, built from source. It holds the fixes for both cases of solidjs/solid#4762: cf32a50b (a keyed projection of a projection losing a row on a switch) and #4769 (two guesses, one landing after the switch, now hold together as D-33 says). + +**Harness:** subagents, as in round 11. + +## Runs, grading + +**Runs:** 6 frontier and 6 small per arm, 60 in all, in one interleaved batch. Each run gets at most 15 `./check`/`./test` calls. + +**Grading:** `harness/grade.sh`, with round 11's definitions of correct and cost. Only defects are counted. A fuzz hole counts unless it is traced to a legal hold, and each exclusion is listed with its trace. + +## Hypotheses and tests + +All tests use seed 12. + +- **H12a** (primary, replication of H11a): small .rx runs have fewer defects than small React runs. One-sided permutation test on mean defects, 100,000 relabellings. +- **H12b**: three comparisons, Holm-corrected together with H12a: + - small .rx has fewer defects than small XState; + - small .rx has fewer defects than small plain; + - small .rx has fewer defects than small Foldkit. +- **H12c**: frontier .rx runs have no more defects than frontier React runs. Difference in means with a 90% bootstrap interval, 10,000 resamples. The claim holds if the upper end is under 1 defect. +- **H12d**: small XState runs have fewer defects than small Foldkit runs. Same test as H12a, reported apart from the .rx family. This is the PROCESS question: state machines against Effect/TEA. +- **Descriptive:** cost per correct app, by arm and tier. +- **Reported alongside:** + - the change from round 11 for each arm and tier; + - defects by kind; + - turns, cost per run and lines. + +The direction is the claim being tested. A null or reversed result is reported as plainly. + +## Gate before freezing + +Round 11's nine .rx desks are regraded on the pinned Solid, with round 11's `rxc`. Any desk that gains a hole is traced. A Solid regression is filed upstream, then fixed or recorded, before the round starts. + +**Gate result** (2026-11-11, on `next` bcef1041's source, round 11's `rxc`): 26 defects across the nine desks, against 29 in round 11. Only rx-2 changed, from 5 to 2: its three fuzz holes (seeds 1, 3, 18, solidjs/solid#4762) are gone, and its two overlap failures, which are its own (a pending source in a component that remounts), remain. The other eight desks grade exactly as in round 11. No desk gained a hole, so nothing needed tracing. + +## Cost + +About $67 at round 11's prices: 6 × ($2.05 + $1.38 + $1.27 + $3.63 + $2.30) for the frontier runs, plus about $3.30 for the small ones. diff --git a/eval/harness/build.mjs b/eval/harness/build.mjs index 769bfdde..e39698d8 100644 --- a/eval/harness/build.mjs +++ b/eval/harness/build.mjs @@ -1,6 +1,6 @@ // Builds one app (a run directory or a reference) into `/.build/dist` with Vite, for the tests to drive. // rx arm: `rxc compile src/main.rx` → JavaScript over Solid rc.19, mounted with the .rx runtime (ref/compile/rt-web.ts) -// react arm: `src/App.tsx` (default export) mounted with createRoot under StrictMode, as Vite's React template does +// react and xstate arms: `src/App.tsx` (default export) mounted with createRoot under StrictMode, as Vite's React template does // solid arm: `src/App.tsx` (default export) compiled by @solidjs/compiler (Solid 2's JSX compiler) and mounted with // @solidjs/web's `render` // vanilla arm: `src/main.ts` default-exports `mount(root: HTMLElement)`, called once with the `#app` container @@ -57,7 +57,7 @@ export async function build(dir, arm = armOf(dir)) { writeFileSync(path.join(work, "entry.js"), entry); writeFileSync(path.join(work, "index.html"), `app
\n`); const vite = await import(pathToFileURL(path.join(evalDir, "node_modules/vite/dist/node/index.js")).href); - const plugins = arm === "react" ? [(await import(pathToFileURL(path.join(evalDir, "node_modules/@vitejs/plugin-react/dist/index.js")).href)).default()] + const plugins = arm === "react" || arm === "xstate" ? [(await import(pathToFileURL(path.join(evalDir, "node_modules/@vitejs/plugin-react/dist/index.js")).href)).default()] : arm === "solid" ? [await solidPlugin()] : []; try { await vite.build({ diff --git a/eval/harness/check.mjs b/eval/harness/check.mjs index 6de099fa..45328c32 100644 --- a/eval/harness/check.mjs +++ b/eval/harness/check.mjs @@ -1,6 +1,6 @@ // The arm's compiler feedback, as the agent runs it. // rx: rxc check src/main.rx -// react: tsc --noEmit -p . and eslint (react-hooks rules) +// react, xstate: tsc --noEmit -p . and eslint (react-hooks rules) // solid: tsc --noEmit -p . and eslint (eslint-plugin-solid's v2 config) // vanilla: tsc --noEmit -p . (strict) // foldkit: tsc --noEmit -p . with Foldkit's template tsconfig (its lint, oxlint + effect-tsgo, is not on npm) diff --git a/eval/harness/cost.mjs b/eval/harness/cost.mjs index 29bda216..1e6865f2 100644 --- a/eval/harness/cost.mjs +++ b/eval/harness/cost.mjs @@ -45,7 +45,7 @@ export function permutation(a, b, n = 100000, seed = 5) { if (import.meta.url === pathToFileURL(process.argv[1]).href) { const [list, dir, out] = process.argv.slice(2); const runs = readFileSync(list, "utf8").trim().split("\n").map(l => l.trim().split(/\s+/)).map(([run, id]) => { - const [, task, arm, n] = run.match(/^(\w+)-(rx|react|solid|foldkit|vanilla)-(\d+)$/); + const [, task, arm, n] = run.match(/^(\w+)-(rx|react|xstate|solid|foldkit|vanilla)-(\d+)$/); return { run, task, arm, n: Number(n), ...usage(path.join(dir, `agent-${id}.jsonl`)) }; }); for (const r of runs) console.log(`${r.run.padEnd(16)} billed ${Math.round(r.billed).toString().padStart(7)} calls ${r.calls} out ${r.output} cacheRead ${r.cacheRead} cacheWrite ${r.cacheWrite}`); diff --git a/eval/harness/prompt-arms.json b/eval/harness/prompt-arms.json index b176568a..2f18e30e 100644 --- a/eval/harness/prompt-arms.json +++ b/eval/harness/prompt-arms.json @@ -5,6 +5,18 @@ "API_TYPES": "`src/api.d.ts`", "CHECKER": "the TypeScript compiler (strict) and ESLint with eslint-plugin-react-hooks' recommended rules (including the React Compiler's)" }, + "xstate": { + "DOCS_RULE": "", + "LANG_RULE": "React 19 with TypeScript and XState v5: `src/App.tsx` must default-export the root component (it is mounted with `createRoot` under `StrictMode`). You may add files under `src/`. Installed packages you may use: react, react-dom, xstate 5.32 and @xstate/react 6.1 (XState v5, which differs from v4). Model the app's state and its server requests as XState machines and actors (`setup`, `createMachine`, `fromPromise`, `useMachine`/`useActor`/`useSelector`), as an XState codebase would.", + "API_TYPES": "`src/api.d.ts`", + "CHECKER": "the TypeScript compiler (strict) and ESLint with eslint-plugin-react-hooks' recommended rules (including the React Compiler's)" + }, + "xstate-inline": { + "DOCS_RULE": ", except that you may read XState's documentation in {{DIR}}/docs/xstate (Stately's v5 docs pages)", + "LANG_RULE": "React 19 with TypeScript and XState v5: `src/App.tsx` must default-export the root component (it is mounted with `createRoot` under `StrictMode`). You may add files under `src/`. Installed packages you may use: react, react-dom, xstate 5.32 and @xstate/react 6.1 (XState v5, which differs from v4). Model the app's state and its server requests as XState machines and actors (`setup`, `createMachine`, `fromPromise`, `useMachine`/`useActor`/`useSelector`), as an XState codebase would. The two pages at the end of this message are XState v5's own cheatsheet and @xstate/react's README; the rest of Stately's v5 docs are in `docs/xstate` (for example `xstate-react.mdx`, `promise-actors.mdx`, `invoke.mdx`, `actors.mdx`).", + "API_TYPES": "`src/api.d.ts`", + "CHECKER": "the TypeScript compiler (strict) and ESLint with eslint-plugin-react-hooks' recommended rules (including the React Compiler's)" + }, "rx": { "DOCS_RULE": ", except that you may read the language documentation in {{DIR}}/docs", "LANG_RULE": "ReactiveScript (`.rx`), a new language that compiles to Solid 2.0. Nothing about it is online. Read `docs/guide/async.md` first: one page on reading from and writing to a server. Its specification is in `docs/spec/` (the concrete syntax is `docs/spec/07-surface.md`; modules and imports are `docs/spec/09-modules.md`). `src/main.rx` must be the program's entry, `export default component App() { … }`; you may add `.rx` modules under `src/`. Import the backend from \"./api\": loaders with `import pure { … } from \"./api\"`, functions that write with `import { … } from \"./api\"`. The prelude gives `import pure { len, str, map, filter, sum, every, indexOf } from \"./prelude\"`. Give an element a test id with `data-testid=\"…\"`.", diff --git a/eval/harness/scaffold.mjs b/eval/harness/scaffold.mjs index 1d741264..ffd3211e 100644 --- a/eval/harness/scaffold.mjs +++ b/eval/harness/scaffold.mjs @@ -12,15 +12,20 @@ // For foldkit, the docs are Foldkit's own FOLDKIT.md (create-foldkit-app's template) at the end of the prompt, and the // Foldkit repository at the installed release's tag readable in docs/foldkit, as Foldkit tells agents to vendor it // (examples, skills, AGENTS.md, the framework source); FOLDKIT_REPO names the checkout (default ../../foldkit). -// Arms: rx, react, solid (Solid 2 TSX, round 5), foldkit (round 6), vanilla (plain TypeScript and DOM, no framework). -// Usage: node harness/scaffold.mjs [--reference] [--brief ] [--base ] [--inline-docs] -import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, symlinkSync, writeFileSync } from "node:fs"; +// Arms: rx, react, solid (Solid 2 TSX, round 5), foldkit (round 6), vanilla (plain TypeScript and DOM, no framework), +// xstate (React with XState v5 machines, no TanStack Query; 2026-11-11). +// Usage: node harness/scaffold.mjs [--reference] [--brief ] [--base ] [--inline-docs] +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; const here = path.dirname(fileURLToPath(import.meta.url)); const evalDir = path.resolve(here, ".."); const foldkitRepo = process.env.FOLDKIT_REPO ?? path.resolve(evalDir, "../../foldkit"); +// For xstate, the docs are Stately's own (github.com/statelyai/docs, content/docs, the XState v5 pages as .mdx): the +// cheatsheet and @xstate/react's README at the end of the prompt, every page readable in docs/xstate. STATELY_DOCS names the +// checkout (default ../../stately-docs). +const statelyDocs = process.env.STATELY_DOCS ?? path.resolve(evalDir, "../../stately-docs"); export function scaffold(task, arm, dir, { reference = false, brief = null, base = null, inline = false } = {}) { const t = path.join(evalDir, "tasks", task); @@ -39,8 +44,8 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base }, null, 2) + "\n"); cpSync(path.join(t, "visible.mjs"), path.join(dir, "tests/visible.mjs")); cpSync(path.join(t, "fixtures.mjs"), path.join(dir, "tests/fixtures.mjs")); - if (arm === "react" || arm === "solid" || arm === "vanilla") { - const jsx = arm === "react" ? { jsx: "react-jsx" } : arm === "solid" ? { jsx: "preserve", jsxImportSource: "@solidjs/web" } : {}; + if (arm === "react" || arm === "xstate" || arm === "solid" || arm === "vanilla") { + const jsx = arm === "react" || arm === "xstate" ? { jsx: "react-jsx" } : arm === "solid" ? { jsx: "preserve", jsxImportSource: "@solidjs/web" } : {}; writeFileSync(path.join(dir, "tsconfig.json"), JSON.stringify({ compilerOptions: { target: "ES2022", module: "ESNext", moduleResolution: "bundler", ...jsx, strict: true, noEmit: true, allowJs: true, checkJs: false, skipLibCheck: true, lib: ["ES2022", "DOM", "DOM.Iterable"] }, include: ["src"], @@ -58,6 +63,10 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base cpSync(path.join(foldkitRepo, p), path.join(dir, "docs/foldkit", p), { recursive: true }); if (!existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules")); } + if (arm === "xstate") { + cpSync(path.join(statelyDocs, "content/docs"), path.join(dir, "docs/xstate"), { recursive: true, filter: f => !/[\\/]assets([\\/]|$)/.test(f) && (statSync(f).isDirectory() || /\.(mdx?|json)$/.test(f)) }); + if (!existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules")); + } // Solid's own docs ship in its package: the agent can read them there before its first build if (arm === "solid" && !existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules")); if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true }); @@ -81,6 +90,7 @@ export function prompt(arm, dir, { plain = false, change = false, inline = false for (let i = 0; i < 2; i++) for (const k of [...Object.keys(fill), "DIR"]) text = text.replaceAll(`{{${k}}}`, k === "DIR" ? dir : fill[k]); if (docs && arm === "rx") for (const f of ["language.md", "async.md", "reference.md"]) text += `\n---\n\n${readFileSync(path.join(evalDir, "../guide", f), "utf8")}`; if (docs && arm === "foldkit") text += `\n---\n\n${readFileSync(path.join(foldkitRepo, "packages/create-foldkit-app/templates/base/FOLDKIT.md"), "utf8")}`; + if (docs && arm === "xstate") text += `\n---\n\n${readFileSync(path.join(statelyDocs, "content/docs/cheatsheet.mdx"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}\n---\n\n${readFileSync(path.join(evalDir, "node_modules/@xstate/react/README.md"), "utf8")}`; if (docs && arm === "solid") text += `\n---\n\n${readFileSync(path.join(evalDir, "node_modules/solid-js/skills/solid-2.0/SKILL.md"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}`; return text; } diff --git a/eval/package.json b/eval/package.json index 9992f9b0..4dd7abd3 100644 --- a/eval/package.json +++ b/eval/package.json @@ -12,6 +12,7 @@ "@types/react": "19.2.2", "@types/react-dom": "19.2.1", "@vitejs/plugin-react": "5.0.4", + "@xstate/react": "6.1.0", "effect": "^4.1.0", "eslint": "9.37.0", "eslint-plugin-react-hooks": "7.0.0", @@ -23,6 +24,7 @@ "solid-js": "2.0.0-rc.21", "typescript": "5.9.3", "typescript-eslint": "8.46.0", - "vite": "7.1.9" + "vite": "7.1.9", + "xstate": "5.32.6" } } diff --git a/eval/reference/chain1/rx/issues.rx b/eval/reference/chain1/rx/issues.rx index 13838f04..cb92c79e 100644 --- a/eval/reference/chain1/rx/issues.rx +++ b/eval/reference/chain1/rx/issues.rx @@ -105,7 +105,7 @@ export component IssuesPage() {

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/chain2/rx/issues.rx b/eval/reference/chain2/rx/issues.rx index a400f55d..bf8d04dd 100644 --- a/eval/reference/chain2/rx/issues.rx +++ b/eval/reference/chain2/rx/issues.rx @@ -117,7 +117,7 @@ export component IssuesPage(props: { onOpenCount: (n: number) => void }) {

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/chain3/rx/issues.rx b/eval/reference/chain3/rx/issues.rx index a19469ef..960d5b2a 100644 --- a/eval/reference/chain3/rx/issues.rx +++ b/eval/reference/chain3/rx/issues.rx @@ -119,7 +119,7 @@ export component IssuesPage(props: { onOpenCount: (n: number) => void; onOpenHel

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/chain4/rx/issues.rx b/eval/reference/chain4/rx/issues.rx index 97122ac4..8f934f9b 100644 --- a/eval/reference/chain4/rx/issues.rx +++ b/eval/reference/chain4/rx/issues.rx @@ -146,7 +146,7 @@ export component IssuesPage(props: { onOpenCount: (n: number) => void; onOpenHel

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/comments/rx/main.rx b/eval/reference/comments/rx/main.rx index d9f2f0d5..43fab7ed 100644 --- a/eval/reference/comments/rx/main.rx +++ b/eval/reference/comments/rx/main.rx @@ -94,7 +94,7 @@ export default component App() { }} { commentInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/desk/rx/issues.rx b/eval/reference/desk/rx/issues.rx index 7cd02652..350276cf 100644 --- a/eval/reference/desk/rx/issues.rx +++ b/eval/reference/desk/rx/issues.rx @@ -104,7 +104,7 @@ export component IssuesPage() {

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/labels/rx/issues.rx b/eval/reference/labels/rx/issues.rx index 13838f04..cb92c79e 100644 --- a/eval/reference/labels/rx/issues.rx +++ b/eval/reference/labels/rx/issues.rx @@ -105,7 +105,7 @@ export component IssuesPage() {

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/related/rx/issues.rx b/eval/reference/related/rx/issues.rx index db6ad3f6..154c7a00 100644 --- a/eval/reference/related/rx/issues.rx +++ b/eval/reference/related/rx/issues.rx @@ -106,7 +106,7 @@ export component IssuesPage(props: { onOpenHelp: (query: string, id: number) =>

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/reference/rename/rx/main.rx b/eval/reference/rename/rx/main.rx index 55ce3ff3..28000cde 100644 --- a/eval/reference/rename/rx/main.rx +++ b/eval/reference/rename/rx/main.rx @@ -89,7 +89,7 @@ export default component App() {

{assignError}

}} { titleInput := e.target.value }} /> - + {if renameError != "" {

rename error: {renameError}

}} diff --git a/eval/results/desk12-2026-11-11/README.md b/eval/results/desk12-2026-11-11/README.md new file mode 100644 index 00000000..e7659f4e --- /dev/null +++ b/eval/results/desk12-2026-11-11/README.md @@ -0,0 +1,160 @@ +# Round 12 results: round 11 again, five arms, six runs per tier (2026-11-11) + +**Pre-registration:** `eval/PREREG-desk-12.md`, frozen at 82c6e42. + +**Runs:** 60, which is 5 arms × (6 frontier + 6 small), in one interleaved batch. Desks 1–6 are the frontier tier (Opus), desks 7–12 the small tier (Haiku). + +**Toolchain:** +- Solid `next` bcef1041 (source identical to 0739445a), built from source. +- rxc from 82c6e42. + +**Grading:** `harness/grade.sh`, with hidden 77, overlap 38 and 20 fuzz seeds. +- Defects are failed hidden tests, plus failed overlap tests, plus fuzz seeds that found a hole. +- Only defects are counted. +- No fuzz hole was traced to a legal hold, so nothing is excluded. Two holes are Solid bugs (filed), and they count against .rx, as pre-registered. + +## Defects per app + +| arm | frontier mean | frontier per desk | small mean | small per desk | small median | +|---|---|---|---|---|---| +| .rx (inline docs) | 3.2 | 1, 10, 4, 0, 2, 2 | **6.0** | 3, 5, 9, 2, 15, 2 | 4 | +| React | 1.8 | 3, 0, 1, 3, 2, 2 | 11.3 | 16, 12, 6, 8, 11, 15 | 11.5 | +| plain TypeScript | 2.7 | 3, 2, 4, 3, 2, 2 | 12.3 | 7, 18, 13, 12, 14, 10 | 12.5 | +| Foldkit (inline docs) | 2.8 | 3, 2, 4, 3, 2, 3 | 13.2 | 15, 12, 13, 12, 16, 11 | 12.5 | +| XState (inline docs) | 2.0 | 3, 2, 1, 1, 2, 3 | 13.0 | 18, 19, 18, 1, 5, 17 | 17.5 | + +## Tests + +All tests use seed 12. Full output is in `stats.txt`. + +| test | result | verdict | +|---|---|---| +| **H12a** (primary): small .rx < small React | p = 0.046, Holm 0.091 | **Not supported** | +| **H12b**: small .rx < small Foldkit | p = 0.011, Holm 0.043 | **Supported** | +| **H12b**: small .rx < small plain | p = 0.023, Holm 0.068 | **Not supported** | +| **H12b**: small .rx < small XState | p = 0.056, Holm 0.091 | **Not supported** | +| **H12c**: frontier .rx no worse than frontier React | difference +1.33, 90% interval [−0.83, 3.83] | **Not shown.** The upper end is not under 1. | +| **H12d**: small XState < small Foldkit | p = 0.50 | **Null** | + +H12a and H12b are Holm-corrected together, as pre-registered. H12a's raw p is about the same as round 11's (0.047). The difference is that round 12 pre-registered three more comparisons in the same family, so the correction is stricter. + +## Change from round 11 + +| arm, tier | round 11 | round 12 | +|---|---|---| +| .rx frontier | 2.0 | 3.2 | +| .rx small | 3.8 | 6.0 | +| React frontier | 2.0 | 1.8 | +| React small | 9.7 | 11.3 | +| plain frontier | 3.0 | 2.7 | +| plain small | 14.5 | 12.3 | +| Foldkit frontier | 2.7 | 2.8 | +| Foldkit small | 13.2 | 13.2 | +| XState frontier | 2.0 (side check) | 2.0 | +| XState small | 7.0 (side check) | 13.0 | + +## Correct apps and cost + +**Correct apps:** two have no defect, frontier `rx-4` and frontier `react-2`. + +**Cost per correct app:** +- frontier .rx: $14.29 (six runs over one correct app); +- frontier React: $7.85. + +Every other cell has no correct app. + +Cost per run is billed tokens (`cost.json`) at round 10's rates (`usd.json`): + +| arm | frontier $/run | small $/run | frontier calls | small calls | lines (frontier / small) | +|---|---|---|---|---|---| +| .rx | 2.38 | 0.095 | 25.2 | 17.5 | 431 / 370 | +| React | 1.31 | 0.064 | 14.5 | 11.2 | 857 / 736 | +| plain | 1.19 | 0.073 | 12.3 | 8.2 | 918 / 1000 | +| Foldkit | 3.29 | 0.169 | 35.7 | 41.5 | 2355 / 2113 | +| XState | 2.49 | 0.137 | 26.5 | 26.5 | 1388 / 1296 | + +Lines are non-blank source lines, without the given `api` and `flights` files. + +## Defects by kind + +- **Overlap tests:** the four non-.rx small arms average 8.3 to 10.8 failed overlap tests per app; small .rx averages 2.5. "Comments / labels that load while an add is out keep it" still fails in most desks of every arm, including rx-2, rx-5 and rx-6. +- **Fuzz holes:** .rx holes, by kind: + +| kind | holes | desks | +|---|---|---| +| detail title | 6 | rx-2, rx-3, rx-9, rx-10 | +| closed row shown | 4 | rx-1, rx-2, rx-3, rx-7 | +| tab count out of step with the rows | 4 | rx-3 (×2), rx-10, rx-12 | +| row assignee | 4 | all rx-11 | +| dropped guess | 2 | rx-7, rx-11 | +| row title | 1 | rx-2 | +| labels at rest | 1 | rx-11 | + +- **Small-tier outliers:** XState is bimodal (1, 5 against 17–19). In the high desks, most defects are overlap: late loads drop changes still in flight. + +### .rx traces + +- **rx-1, seed 15 ("closed row shown"): a Solid bug, filed as solidjs/solid#4786.** + - The app guesses two closes on a `createOptimistic` list of hidden ids and an assign on the optimistic issues store. + - When both closes have landed, both close guesses are gone, although `assign:4` and both refetches are still out. + - It passes without the assign guess. It also fails on round 11's Solid (6751472f). + - Repro: `repro/two-close-guesses.spec.tsx`. +- **rx-2, seed 13 ("closed row shown"): a Solid bug, filed as solidjs/solid#4787.** + - The app chooses between two keyed lists with `filter == "open" ? openIssues : closedIssues`. + - When it switches back to a list whose load failed, the refetch never clears the error. + - With an accepted guess on the other list, the open view shows the closed list's rows, at rest. It also fails on 6751472f. + - Repro: `repro/switch-after-failed-list.spec.tsx`. +- **rx-7, seed 12 ("closed row shown"): an agent mistake.** + - After its await, each close writes `closed := [...closed, r.id]`. With two closes out, the second write loses the first close's id. + - The guide's functional form avoids this. AC-1e only covers a read made before the await, so no check fired. +- **Detail title (rx-2 seeds 5 and 8, rx-9 seed 15, rx-10 seed 15, replayed): agent mistakes, the same kind as round 11's.** Two shapes: + - The rename is made while the detail is still loading. A guess on a value that was never shown is void (D-34), so the heading shows the old title through the whole rename. + - The app puts no guess on the heading at all (rx-2). + - rx-3 seed 15 and rx-9 seed 11 have the same trail shape but were not replayed. +- **Tab count (rx-3 ×2, rx-10, rx-12): counted.** Each app passes the count up from an `effect` inside the list's loading region. While a row guess is out, the tab shows the count from before the guess. This is a tear between two views, not a hold. +- **Not classified:** + - rx-7 seed 8: a close declined on #4 also drops the assign guess on #4. Both guesses are functional guesses on one source, so this may be H-11. + - rx-11 (row assignee ×4, a dropped rename guess, doubled labels): the desk fails 9 hidden and overlap tests on its own rename and save logic, so these are most likely its own. + - rx-3 seed 5. + +**Without the two Solid bugs**, frontier .rx is 3.0 instead of 3.2 (rx-1 0, rx-2 9), and small .rx is unchanged at 6.0. The tests above do not move. + +## Deviations + +- **Temporary visible tests, restored byte for byte (checked):** rx-1, rx-2, rx-3, rx-5, rx-6, xstate-1, xstate-2, react-11, react-12. +- **Scratch files written to `/tmp` and then deleted:** react-7, foldkit-9. +- **foldkit-9 posted one stray message to the project thread.** It has been edited to say so. +- **Ran `tsc` outside the check/test budget:** foldkit-7, foldkit-12. +- **Listed file names in `eval/harness` (read no contents):** foldkit-7. +- **Scratch scripts or simulations inside the run dir, deleted:** xstate-7, xstate-9, xstate-11, xstate-12. +- **rx-9 ended at 29 of 30 visible tests:** a related hit never opens the help page. It is graded as it stands. +- **A machine restart interrupted grading twice.** The interrupted desks were regraded from the start, and no result mixes two runs. +- None of these affects grading. + +## Recurring .rx traps reported by the agents (guide candidates, not checks) + +- **A filter switch held by a row guess (rx-2, rx-3, rx-5):** with one list whose question reads the filter, a guess on it holds the filter switch. All three split the list per filter. +- **A hold at mount (rx-4, rx-8, rx-9, rx-12):** `pending?` or a mount-time read outside a loading region holds the tab switch. rx-9 never solved it. + +## Sensitivity (not pre-registered) + +- **Without the worst small desk** (rx-11, 15 defects), small .rx averages 4.2, and every Holm comparison clears 0.05 (React p = 0.009). +- **A rank test**, which one outlier cannot dominate, gives small .rx < React p = 0.036. +- **Without the worst frontier desk** (rx-2, 10), frontier .rx averages 1.8, level with React. +- **Spread:** the standard deviation of small-tier defects is 7.9 for XState, 5.1 for .rx, 3.9 for React and 1.9 for Foldkit. Frontier is 3.6 for .rx and 1.2 for React. + +## Reading + +- **Small tier:** .rx still has the fewest defects, about half of React's, plain's, XState's and Foldkit's. With the pre-registered Holm family, only the Foldkit comparison clears 0.05, so the primary hypothesis is not supported this round. +- **Frontier tier:** .rx is the worst arm, at 3.2, though its median is 2. One desk (rx-2, 10 defects) accounts for much of that, and two .rx holes are Solid bugs (#4786, #4787). React is best at 1.8. +- **XState:** level with React at the frontier. In the small tier it lost the side check's advantage (7.0 → 13.0) and is now level with Foldkit and plain. +- **Cost:** frontier .rx costs $2.38 per run, against $1.31 for React. That is up from round 11's $2.05. + +## Files + +- `summary.txt`: per desk. +- `graded/`: `grade.sh` output. +- `stats.py`, `stats.txt`. +- `cost.json`, `usd.json`. +- `agents-12.txt`: desk to transcript. +- `repro/`: the plain-Solid repros. diff --git a/eval/results/desk12-2026-11-11/agents-12.txt b/eval/results/desk12-2026-11-11/agents-12.txt new file mode 100644 index 00000000..2e7363c7 --- /dev/null +++ b/eval/results/desk12-2026-11-11/agents-12.txt @@ -0,0 +1,60 @@ +desk12-rx-1 a2adcc860413e93be +desk12-foldkit-1 a139926c988c12fa7 +desk12-react-1 a34fae9b65175a351 +desk12-vanilla-1 a3fe146269a48f327 +desk12-xstate-1 a4e38081558642822 +desk12-rx-7 a1a2fec31d1b8a77e +desk12-foldkit-7 a1cada09a3a45236b +desk12-react-7 a7bbe53d54a2bce3e +desk12-vanilla-7 ab8e8448de3a44a7a +desk12-xstate-7 aea72cfdb2062a71b +desk12-rx-2 aeb6d74755925b637 +desk12-foldkit-2 af7c64b4b83886410 +desk12-react-2 a0055201cb80640ed +desk12-vanilla-2 a7c7382a8117a3304 +desk12-xstate-2 ae0c06da1dfdb5c25 +desk12-rx-8 ac2a3ecdef7d9eb7e +desk12-foldkit-8 a118273811805976b +desk12-react-8 abe15c6315c2be8c3 +desk12-vanilla-8 a58410988f54ff148 +desk12-xstate-8 a94f397ea79b75bbf +desk12-rx-3 a0559f280b6399d43 +desk12-foldkit-3 aeaaaf05d03a789fc +desk12-react-3 aa321e6cbbd3be969 +desk12-vanilla-3 aac54043cc1125c25 +desk12-xstate-3 affb8ad029601946b +desk12-rx-9 aa329eef1001465b3 +desk12-foldkit-9 ac2c7d376ad5ad545 +desk12-react-9 ae521de90f5b7f35a +desk12-vanilla-9 a25ab8e6647b18e52 +desk12-xstate-9 a5da6342624907ee8 +desk12-rx-4 ac3350f0df75b6aeb +desk12-foldkit-4 a62b40df39af2a356 +desk12-react-4 afa956dff110d1ef4 +desk12-vanilla-4 ad4c9fb68b3c66e28 +desk12-xstate-4 a50c61cf3e926fedb +desk12-rx-10 a668866d052f45159 +desk12-foldkit-10 a8a531db059da8512 +desk12-react-10 a18751ce352dbcefe +desk12-vanilla-10 a14177a5f6843636a +desk12-xstate-10 a5c46d6a332c2a3cc +desk12-rx-5 a4bda814948c8cef4 +desk12-foldkit-5 a072930b5ee5d5ddd +desk12-react-5 aa26e94820fbcd504 +desk12-vanilla-5 a6ab112dab302338b +desk12-xstate-5 a5c96cc781d53458d +desk12-rx-11 ad4f28e198cbf742c +desk12-foldkit-11 a8d95ebe8b22826d3 +desk12-react-11 a77cf07d01b52dacc +desk12-vanilla-11 aa70ee82b32e4b49e +desk12-xstate-11 a86ccff47c7117834 +desk12-rx-6 add471b87949b59da +desk12-foldkit-6 a527785c5df3f9938 +desk12-react-6 a243b640faeee847d +desk12-vanilla-6 ae38f3b1526ae7d8d +desk12-xstate-6 a9bf2a63511c7fac6 +desk12-rx-12 a68f68bd9daa5f0ca +desk12-foldkit-12 a3dbb3a0ece54aff9 +desk12-react-12 a0dd5f8ce8884fa34 +desk12-vanilla-12 a7f4dd1341822892d +desk12-xstate-12 abd05c774a847eff7 diff --git a/eval/results/desk12-2026-11-11/cost.json b/eval/results/desk12-2026-11-11/cost.json new file mode 100644 index 00000000..3b1a5155 --- /dev/null +++ b/eval/results/desk12-2026-11-11/cost.json @@ -0,0 +1,733 @@ +{ + "weights": { + "input": 1, + "cacheWrite": 1.25, + "cacheRead": 0.1, + "output": 5 + }, + "runs": [ + { + "run": "desk12-rx-1", + "task": "desk12", + "arm": "rx", + "n": 1, + "input": 36, + "cacheWrite": 111782, + "cacheRead": 1535886, + "output": 32102, + "calls": 18, + "billed": 453862.1 + }, + { + "run": "desk12-foldkit-1", + "task": "desk12", + "arm": "foldkit", + "n": 1, + "input": 66, + "cacheWrite": 145625, + "cacheRead": 3126364, + "output": 62092, + "calls": 33, + "billed": 805193.65 + }, + { + "run": "desk12-react-1", + "task": "desk12", + "arm": "react", + "n": 1, + "input": 32, + "cacheWrite": 81477, + "cacheRead": 987557, + "output": 28800, + "calls": 16, + "billed": 344633.95 + }, + { + "run": "desk12-vanilla-1", + "task": "desk12", + "arm": "vanilla", + "n": 1, + "input": 30, + "cacheWrite": 88321, + "cacheRead": 971513, + "output": 35635, + "calls": 15, + "billed": 385757.55 + }, + { + "run": "desk12-xstate-1", + "task": "desk12", + "arm": "xstate", + "n": 1, + "input": 54, + "cacheWrite": 130052, + "cacheRead": 2609007, + "output": 61966, + "calls": 27, + "billed": 733349.7 + }, + { + "run": "desk12-rx-7", + "task": "desk12", + "arm": "rx", + "n": 7, + "input": 28, + "cacheWrite": 204874, + "cacheRead": 1424741, + "output": 98603, + "calls": 14, + "billed": 891609.6 + }, + { + "run": "desk12-foldkit-7", + "task": "desk12", + "arm": "foldkit", + "n": 7, + "input": 86, + "cacheWrite": 279518, + "cacheRead": 6220729, + "output": 163192, + "calls": 43, + "billed": 1787516.4 + }, + { + "run": "desk12-react-7", + "task": "desk12", + "arm": "react", + "n": 7, + "input": 30, + "cacheWrite": 126438, + "cacheRead": 1289461, + "output": 72552, + "calls": 15, + "billed": 649783.6 + }, + { + "run": "desk12-vanilla-7", + "task": "desk12", + "arm": "vanilla", + "n": 7, + "input": 18, + "cacheWrite": 182266, + "cacheRead": 754373, + "output": 106896, + "calls": 9, + "billed": 837767.8 + }, + { + "run": "desk12-xstate-7", + "task": "desk12", + "arm": "xstate", + "n": 7, + "input": 94, + "cacheWrite": 246043, + "cacheRead": 8196570, + "output": 156921, + "calls": 47, + "billed": 1911909.75 + }, + { + "run": "desk12-rx-2", + "task": "desk12", + "arm": "rx", + "n": 2, + "input": 60, + "cacheWrite": 129992, + "cacheRead": 2956711, + "output": 44047, + "calls": 30, + "billed": 678456.1000000001 + }, + { + "run": "desk12-foldkit-2", + "task": "desk12", + "arm": "foldkit", + "n": 2, + "input": 66, + "cacheWrite": 144902, + "cacheRead": 2983317, + "output": 63995, + "calls": 33, + "billed": 799500.2 + }, + { + "run": "desk12-react-2", + "task": "desk12", + "arm": "react", + "n": 2, + "input": 30, + "cacheWrite": 86281, + "cacheRead": 949412, + "output": 32199, + "calls": 15, + "billed": 363817.45 + }, + { + "run": "desk12-vanilla-2", + "task": "desk12", + "arm": "vanilla", + "n": 2, + "input": 26, + "cacheWrite": 81660, + "cacheRead": 767702, + "output": 28248, + "calls": 13, + "billed": 320111.2 + }, + { + "run": "desk12-xstate-2", + "task": "desk12", + "arm": "xstate", + "n": 2, + "input": 54, + "cacheWrite": 121006, + "cacheRead": 2403725, + "output": 51972, + "calls": 27, + "billed": 651544 + }, + { + "run": "desk12-rx-8", + "task": "desk12", + "arm": "rx", + "n": 8, + "input": 46, + "cacheWrite": 198415, + "cacheRead": 2987901, + "output": 126008, + "calls": 23, + "billed": 1176894.85 + }, + { + "run": "desk12-foldkit-8", + "task": "desk12", + "arm": "foldkit", + "n": 8, + "input": 124, + "cacheWrite": 295555, + "cacheRead": 10333210, + "output": 184349, + "calls": 62, + "billed": 2324633.75 + }, + { + "run": "desk12-react-8", + "task": "desk12", + "arm": "react", + "n": 8, + "input": 22, + "cacheWrite": 123959, + "cacheRead": 759288, + "output": 68764, + "calls": 11, + "billed": 574719.55 + }, + { + "run": "desk12-vanilla-8", + "task": "desk12", + "arm": "vanilla", + "n": 8, + "input": 14, + "cacheWrite": 185108, + "cacheRead": 470624, + "output": 107368, + "calls": 7, + "billed": 815301.4 + }, + { + "run": "desk12-xstate-8", + "task": "desk12", + "arm": "xstate", + "n": 8, + "input": 28, + "cacheWrite": 156652, + "cacheRead": 1366231, + "output": 95697, + "calls": 14, + "billed": 810951.1 + }, + { + "run": "desk12-rx-3", + "task": "desk12", + "arm": "rx", + "n": 3, + "input": 68, + "cacheWrite": 122334, + "cacheRead": 3781160, + "output": 52688, + "calls": 34, + "billed": 794541.5 + }, + { + "run": "desk12-foldkit-3", + "task": "desk12", + "arm": "foldkit", + "n": 3, + "input": 64, + "cacheWrite": 122311, + "cacheRead": 3119895, + "output": 57228, + "calls": 32, + "billed": 751082.25 + }, + { + "run": "desk12-react-3", + "task": "desk12", + "arm": "react", + "n": 3, + "input": 28, + "cacheWrite": 57513, + "cacheRead": 904710, + "output": 31995, + "calls": 14, + "billed": 322365.25 + }, + { + "run": "desk12-vanilla-3", + "task": "desk12", + "arm": "vanilla", + "n": 3, + "input": 18, + "cacheWrite": 52189, + "cacheRead": 504720, + "output": 27538, + "calls": 9, + "billed": 253416.25 + }, + { + "run": "desk12-xstate-3", + "task": "desk12", + "arm": "xstate", + "n": 3, + "input": 46, + "cacheWrite": 90390, + "cacheRead": 1973842, + "output": 53571, + "calls": 23, + "billed": 578272.7 + }, + { + "run": "desk12-rx-9", + "task": "desk12", + "arm": "rx", + "n": 9, + "input": 40, + "cacheWrite": 161078, + "cacheRead": 2307802, + "output": 113888, + "calls": 20, + "billed": 1001607.7 + }, + { + "run": "desk12-foldkit-9", + "task": "desk12", + "arm": "foldkit", + "n": 9, + "input": 68, + "cacheWrite": 208075, + "cacheRead": 4867095, + "output": 142588, + "calls": 34, + "billed": 1459811.25 + }, + { + "run": "desk12-react-9", + "task": "desk12", + "arm": "react", + "n": 9, + "input": 18, + "cacheWrite": 129682, + "cacheRead": 535259, + "output": 79881, + "calls": 9, + "billed": 615051.4 + }, + { + "run": "desk12-vanilla-9", + "task": "desk12", + "arm": "vanilla", + "n": 9, + "input": 18, + "cacheWrite": 144080, + "cacheRead": 745247, + "output": 96320, + "calls": 9, + "billed": 736242.7 + }, + { + "run": "desk12-xstate-9", + "task": "desk12", + "arm": "xstate", + "n": 9, + "input": 64, + "cacheWrite": 306521, + "cacheRead": 5685124, + "output": 175356, + "calls": 32, + "billed": 1828507.65 + }, + { + "run": "desk12-rx-4", + "task": "desk12", + "arm": "rx", + "n": 4, + "input": 34, + "cacheWrite": 72351, + "cacheRead": 1319097, + "output": 33516, + "calls": 17, + "billed": 389962.45 + }, + { + "run": "desk12-foldkit-4", + "task": "desk12", + "arm": "foldkit", + "n": 4, + "input": 70, + "cacheWrite": 125165, + "cacheRead": 3537625, + "output": 59319, + "calls": 35, + "billed": 806883.75 + }, + { + "run": "desk12-react-4", + "task": "desk12", + "arm": "react", + "n": 4, + "input": 30, + "cacheWrite": 57752, + "cacheRead": 955274, + "output": 31609, + "calls": 15, + "billed": 325792.4 + }, + { + "run": "desk12-vanilla-4", + "task": "desk12", + "arm": "vanilla", + "n": 4, + "input": 22, + "cacheWrite": 48707, + "cacheRead": 615320, + "output": 24211, + "calls": 11, + "billed": 243492.75 + }, + { + "run": "desk12-xstate-4", + "task": "desk12", + "arm": "xstate", + "n": 4, + "input": 56, + "cacheWrite": 87552, + "cacheRead": 2488186, + "output": 47315, + "calls": 28, + "billed": 594889.6 + }, + { + "run": "desk12-rx-10", + "task": "desk12", + "arm": "rx", + "n": 10, + "input": 32, + "cacheWrite": 133347, + "cacheRead": 1768004, + "output": 93753, + "calls": 16, + "billed": 812281.15 + }, + { + "run": "desk12-foldkit-10", + "task": "desk12", + "arm": "foldkit", + "n": 10, + "input": 78, + "cacheWrite": 212658, + "cacheRead": 4832273, + "output": 155865, + "calls": 39, + "billed": 1528452.8 + }, + { + "run": "desk12-react-10", + "task": "desk12", + "arm": "react", + "n": 10, + "input": 14, + "cacheWrite": 136140, + "cacheRead": 456511, + "output": 84795, + "calls": 7, + "billed": 639815.1 + }, + { + "run": "desk12-vanilla-10", + "task": "desk12", + "arm": "vanilla", + "n": 10, + "input": 14, + "cacheWrite": 104782, + "cacheRead": 469192, + "output": 79922, + "calls": 7, + "billed": 577520.7 + }, + { + "run": "desk12-xstate-10", + "task": "desk12", + "arm": "xstate", + "n": 10, + "input": 36, + "cacheWrite": 181392, + "cacheRead": 2214381, + "output": 139523, + "calls": 18, + "billed": 1145829.1 + }, + { + "run": "desk12-rx-5", + "task": "desk12", + "arm": "rx", + "n": 5, + "input": 58, + "cacheWrite": 118846, + "cacheRead": 3080349, + "output": 54445, + "calls": 29, + "billed": 728875.4 + }, + { + "run": "desk12-foldkit-5", + "task": "desk12", + "arm": "foldkit", + "n": 5, + "input": 84, + "cacheWrite": 130847, + "cacheRead": 4285215, + "output": 61954, + "calls": 42, + "billed": 901934.25 + }, + { + "run": "desk12-react-5", + "task": "desk12", + "arm": "react", + "n": 5, + "input": 28, + "cacheWrite": 54699, + "cacheRead": 878192, + "output": 29518, + "calls": 14, + "billed": 303810.95 + }, + { + "run": "desk12-vanilla-5", + "task": "desk12", + "arm": "vanilla", + "n": 5, + "input": 24, + "cacheWrite": 52353, + "cacheRead": 704634, + "output": 26828, + "calls": 12, + "billed": 270068.65 + }, + { + "run": "desk12-xstate-5", + "task": "desk12", + "arm": "xstate", + "n": 5, + "input": 50, + "cacheWrite": 81147, + "cacheRead": 2073044, + "output": 46338, + "calls": 25, + "billed": 540478.15 + }, + { + "run": "desk12-rx-11", + "task": "desk12", + "arm": "rx", + "n": 11, + "input": 30, + "cacheWrite": 194289, + "cacheRead": 1736541, + "output": 110711, + "calls": 15, + "billed": 970100.35 + }, + { + "run": "desk12-foldkit-11", + "task": "desk12", + "arm": "foldkit", + "n": 11, + "input": 60, + "cacheWrite": 235043, + "cacheRead": 4645895, + "output": 177656, + "calls": 30, + "billed": 1646733.25 + }, + { + "run": "desk12-react-11", + "task": "desk12", + "arm": "react", + "n": 11, + "input": 24, + "cacheWrite": 116341, + "cacheRead": 1092237, + "output": 88636, + "calls": 12, + "billed": 697853.95 + }, + { + "run": "desk12-vanilla-11", + "task": "desk12", + "arm": "vanilla", + "n": 11, + "input": 14, + "cacheWrite": 135642, + "cacheRead": 456472, + "output": 86738, + "calls": 7, + "billed": 648903.7 + }, + { + "run": "desk12-xstate-11", + "task": "desk12", + "arm": "xstate", + "n": 11, + "input": 46, + "cacheWrite": 234320, + "cacheRead": 3015494, + "output": 143895, + "calls": 23, + "billed": 1313970.4 + }, + { + "run": "desk12-rx-6", + "task": "desk12", + "arm": "rx", + "n": 6, + "input": 46, + "cacheWrite": 98331, + "cacheRead": 2108039, + "output": 38578, + "calls": 23, + "billed": 526653.65 + }, + { + "run": "desk12-foldkit-6", + "task": "desk12", + "arm": "foldkit", + "n": 6, + "input": 78, + "cacheWrite": 136075, + "cacheRead": 4162496, + "output": 56274, + "calls": 39, + "billed": 867791.3500000001 + }, + { + "run": "desk12-react-6", + "task": "desk12", + "arm": "react", + "n": 6, + "input": 26, + "cacheWrite": 55731, + "cacheRead": 804321, + "output": 30157, + "calls": 13, + "billed": 300906.85 + }, + { + "run": "desk12-vanilla-6", + "task": "desk12", + "arm": "vanilla", + "n": 6, + "input": 28, + "cacheWrite": 55362, + "cacheRead": 874689, + "output": 30548, + "calls": 14, + "billed": 309439.4 + }, + { + "run": "desk12-xstate-6", + "task": "desk12", + "arm": "xstate", + "n": 6, + "input": 58, + "cacheWrite": 93187, + "cacheRead": 2605904, + "output": 50524, + "calls": 29, + "billed": 629752.15 + }, + { + "run": "desk12-rx-12", + "task": "desk12", + "arm": "rx", + "n": 12, + "input": 34, + "cacheWrite": 136862, + "cacheRead": 1693903, + "output": 97148, + "calls": 17, + "billed": 826241.8 + }, + { + "run": "desk12-foldkit-12", + "task": "desk12", + "arm": "foldkit", + "n": 12, + "input": 82, + "cacheWrite": 187366, + "cacheRead": 5284252, + "output": 131529, + "calls": 41, + "billed": 1420359.7000000002 + }, + { + "run": "desk12-react-12", + "task": "desk12", + "arm": "react", + "n": 12, + "input": 26, + "cacheWrite": 113232, + "cacheRead": 1027119, + "output": 85810, + "calls": 13, + "billed": 673327.9 + }, + { + "run": "desk12-vanilla-12", + "task": "desk12", + "arm": "vanilla", + "n": 12, + "input": 20, + "cacheWrite": 143638, + "cacheRead": 888763, + "output": 95474, + "calls": 10, + "billed": 745813.8 + }, + { + "run": "desk12-xstate-12", + "task": "desk12", + "arm": "xstate", + "n": 12, + "input": 50, + "cacheWrite": 177895, + "cacheRead": 3455736, + "output": 132553, + "calls": 25, + "billed": 1230757.35 + } + ], + "diff": -286600.6916666669, + "ratio": 1.59175503905721, + "p": 0.998450015499845 +} diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-1.txt new file mode 100644 index 00000000..8c345411 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-1: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-10.txt new file mode 100644 index 00000000..b4b7eeaa --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-10.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk12-foldkit-10: 2/20 seeds found a hole + seed 3 step 35: count: tab Issues (2), 1 open rows shown + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 1 ; close ; add-label ui ; user guest ; tab help ; tab issues ; land removeLabel:4,bug ; fail as + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-11.txt new file mode 100644 index 00000000..c1ef612a --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-11.txt @@ -0,0 +1,26 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-foldkit-11: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 39: two views: row "#4 Typo in footer", detail "#4 Bravofooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-12.txt new file mode 100644 index 00000000..953973c7 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-12.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk12-foldkit-12: 2/20 seeds found a hole + seed 9 step 40: row assignee: #4 shows null, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 33: detail title: "retry#3 Crash on savestack trace attachedassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-2.txt new file mode 100644 index 00000000..3a55c7bf --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-3.txt new file mode 100644 index 00000000..bc52abee --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-3.txt @@ -0,0 +1,11 @@ +== hidden +✗ base: settings: saving an invalid name asks nothing +76/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-3: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-4.txt new file mode 100644 index 00000000..d8014d67 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-4.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-4: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-5.txt new file mode 100644 index 00000000..8937e039 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-6.txt new file mode 100644 index 00000000..37fa968d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-6.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-6: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-7.txt new file mode 100644 index 00000000..a6595775 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-7.txt @@ -0,0 +1,24 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-foldkit-7: 3/20 seeds found a hole + seed 3 step 35: count: tab Issues (3), 2 open rows shown + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land addLabel:4,bug ; row-assign 0 ; remove-label 1 ; close ; add-label ui ; user guest ; tab help ; tab issues ; land assign:3,bob ; fail close + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-8.txt new file mode 100644 index 00000000..dddd6d21 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-8.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-foldkit-8: 1/20 seeds found a hole + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-9.txt new file mode 100644 index 00000000..b639133d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-9.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-foldkit-9: 2/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-1.txt new file mode 100644 index 00000000..f022505d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-1: 1/20 seeds found a hole + seed 16 step 37: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-10.txt new file mode 100644 index 00000000..3c381073 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-10.txt @@ -0,0 +1,16 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +32/38 passed +== fuzz +desk12-react-10: 2/20 seeds found a hole + seed 2 step 37: at rest: comments: #2 shows [], server [guest: fixed? | guest: fixed?] + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-11.txt new file mode 100644 index 00000000..b5f6dbe5 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-11.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-react-11: 3/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 3 step 39: at rest: labels: #4 shows [docs,p1], server [docs,bug,p1] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail search:Typo in footer ; land fetchLabels:4 ; row-close 2 ; row-close 0 ; land addLabel:4,bug ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land assign:3,bob ; fai + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-12.txt new file mode 100644 index 00000000..cf975039 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-12.txt @@ -0,0 +1,25 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-react-12: 4/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-2.txt new file mode 100644 index 00000000..5ac0761f --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-2.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-react-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-3.txt new file mode 100644 index 00000000..a0ef373e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-react-3: 1/20 seeds found a hole + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-4.txt new file mode 100644 index 00000000..a113d2b5 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-4.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-4: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-5.txt new file mode 100644 index 00000000..bbcb3194 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-6.txt new file mode 100644 index 00000000..fde3606a --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-6.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-6: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-7.txt new file mode 100644 index 00000000..105b7c74 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-7.txt @@ -0,0 +1,26 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-react-7: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 12 step 33: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Typo in footer ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land search:Alpha ; fail assign:4,guest ; assign ; land closeIssue + seed 15 step 37: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-8.txt new file mode 100644 index 00000000..69dcd6c3 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-8.txt @@ -0,0 +1,21 @@ +== hidden +✗ rows: row errors and detail errors leave each other alone +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-react-8: 3/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [docs,p1], server [docs] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; row-close 0 ; fail search:Typo in footer ; land fetchLabels:4 ; row-close 1 ; row-close 0 ; land closeIssue:4 ; close ; add-label p1 ; remove-label 0 ; remove-label 0 ; filter + seed 12 step 44: row assignee: #1 shows guest, server null + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; rename Alpha ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchComments:1 ; land fetchComments:4 ; row-close 0 ; rename Alpha ; (move failed: locator.click: Timeout 2000ms exceeded.) ; fail cl + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-9.txt new file mode 100644 index 00000000..dee0c45e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-9.txt @@ -0,0 +1,13 @@ +== hidden +77/77 passed +== overlap +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +33/38 passed +== fuzz +desk12-react-9: 1/20 seeds found a hole + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-1.txt new file mode 100644 index 00000000..0e5dc065 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-1.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-1: 1/20 seeds found a hole + seed 15 step 31: closed row shown: #1 is closed on the server + land fetchIssue:0 ; user bob ; filter closed ; land fetchComments:0 ; land fetchIssues:open ; fail fetchLabels:0 ; focus ; filter open ; fail fetchIssues:closed ; user guest ; land fetchIssues:open ; row-close 1 ; land closeIssue:3 ; fail fetchIssues:open ; focus ; tab help ; tab issues ; focus ; land fetchLabels:0 ; land fetchComments:0 ; land fetchIssue:0 ; land fetchIssues:open ; tab help ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-10.txt new file mode 100644 index 00000000..2802bf1e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-10.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-10: 2/20 seeds found a hole + seed 1 step 17: count: tab Issues (2), 3 open rows shown + user bob ; focus ; land fetchIssues:open ; row-assign 2 ; row-close 0 ; row-assign 0 ; select #3 ; rename Alpha ; land assign:3,bob ; rename Alpha ; row-assign 1 ; land renameIssue:3,Alpha ; focus ; land fetchLabels:3 ; land fetchIssue:3 ; land fetchIssues:open ; fail closeIssue:1 + seed 15 step 24: detail title: "retry#2 Old buglong goneassign to guest rename closepostadd ", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-11.txt new file mode 100644 index 00000000..f02f9680 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-11.txt @@ -0,0 +1,27 @@ +== hidden +✗ base: an accepted post stays after switching issues and back +✗ base: settings: a failed save reverts the saved line, shows the error and keeps the draft +✗ base: settings: edits made while a save is in flight survive its failure +✗ base: settings: a failed save after a successful one reverts to the last save +73/77 passed +== overlap +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +33/38 passed +== fuzz +desk12-rx-11: 6/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchLabels:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchComments:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 3 step 37: row assignee: #3 shows bob, server null + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; post ping ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; row-close 0 ; fail search:Typo in footer ; land fetchComments:4 ; row-close 1 ; row-assign 0 ; land closeIssue:1 ; close ; row-assign 0 ; row-close 0 ; post fixed? ; filter closed ; post ping ; assign ; land closeIssue:3 ; fail assign:3 + seed 9 step 40: row assignee: #4 shows guest, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 18 step 36: row assignee: #3 shows null, server bob + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-assign 0 ; row-close 1 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta ; user bob ; row-assign 0 ; row-close 1 ; rename Delta ; assign ; row-assign 0 ; land fetchComments:4 ; rename Alpha ; rename Alpha ; rename Delta ; row-close 0 ; focus ; close + seed 19 step 17: dropped guess: #3 title: showed "Delta" while it was in flight, now "Alpha" + land fetchIssues:open ; user guest ; row-assign 1 ; row-close 2 ; land assign:3,guest ; fail closeIssue:4 ; select #3 ; post fixed? ; land fetchLabels:3 ; rename Alpha ; rename Delta ; row-assign 0 ; land assign:1,guest ; land fetchIssue:3 ; row-assign 1 ; land addComment:3,guest,fixed? ; land renameIssue:3,Alpha + seed 20 step 38: at rest: labels: #2 shows [p1,p1], server [p1] + filter closed ; filter open ; user guest ; filter closed ; user ann ; land fetchIssues:closed ; land fetchIssues:open ; select #2 ; land fetchLabels:2 ; filter open ; focus ; land fetchComments:2 ; land fetchIssue:2 ; fail fetchIssues:open ; close ; post ping ; add-label p1 ; land search:Old bug ; close ; land closeIssue:2 ; remove-label 1 ; close ; user ann ; assign ; land closeIssue:2 ; clos diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-12.txt new file mode 100644 index 00000000..961a6ab8 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-12.txt @@ -0,0 +1,9 @@ +== hidden +77/77 passed +== overlap +✗ a label added on one issue does not show on another +37/38 passed +== fuzz +desk12-rx-12: 1/20 seeds found a hole + seed 1 step 11: count: tab Issues (2), 1 open rows shown + user bob ; focus ; land fetchIssues:open ; row-assign 2 ; row-close 0 ; row-assign 0 ; select #3 ; row-assign 0 ; land assign:3,bob ; row-assign 0 ; row-close 1 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-2.txt new file mode 100644 index 00000000..1b74e847 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-2.txt @@ -0,0 +1,20 @@ +== hidden +✗ base: an accepted assignment survives switching filters and back +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +33/38 passed +== fuzz +desk12-rx-2: 4/20 seeds found a hole + seed 5 step 23: detail title: "#3 Crash on savestack trace attachedrelated help failed: dec", server "Delta" + focus ; fail fetchIssues:closed ; tab help ; tab issues ; land fetchIssues:closed ; user bob ; land fetchIssues:open ; user guest ; row-assign 0 ; row-assign 0 ; land assign:1,guest ; land fetchIssues:open ; row-assign 1 ; row-assign 1 ; land assign:3,guest ; row-close 2 ; select #3 ; rename Delta ; select #3 ; land fetchIssue:3 ; land closeIssue:4 ; fail search:Crash on save ; land renameIssu + seed 8 step 22: detail title: "#4 Typo in footerfooter says 2019related help failed: declin", server "Alpha" + land fetchIssues:closed ; tab help ; tab issues ; fail fetchIssues:closed ; user bob ; land fetchIssues:open ; row-close 0 ; land closeIssue:1 ; user guest ; land fetchIssues:open ; row-close 0 ; select #4 ; land fetchComments:4 ; row-close 0 ; rename Alpha ; land fetchIssue:4 ; rename Alpha ; user guest ; fail search:Typo in footer ; close ; land fetchLabels:4 ; land renameIssue:4,Alpha + seed 13 step 8: closed row shown: #2 is closed on the server + filter closed ; fail fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; filter open ; land assign:2,guest ; land fetchIssues:closed ; land fetchIssues:open + seed 16 step 42: row title: #1 shows "Login fails", server "Bravo" + user guest ; fail fetchIssues:open ; tab help ; land fetchIssues:closed ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #4 ; land fetchComments:4 ; close ; post me too ; fail addComment:4,guest,me too ; row-assign 1 ; row-assign 1 ; close ; post ping ; assign ; land addComment:4,guest,ping ; fail assign:3,guest ; fail assign:4,guest ; select #1 ; rename Bravo ; assign ; fail close diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-3.txt new file mode 100644 index 00000000..e46bb31f --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-3.txt @@ -0,0 +1,14 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-3: 4/20 seeds found a hole + seed 5 step 22: closed row shown: #3 is closed on the server + focus ; fail fetchIssues:open ; tab help ; tab issues ; focus ; land fetchIssues:open ; row-close 0 ; row-assign 1 ; land assign:4,guest ; land closeIssue:1 ; land fetchIssues:open ; row-close 1 ; land closeIssue:4 ; land fetchIssues:open ; row-assign 0 ; land assign:3,guest ; row-close 0 ; select #3 ; rename Delta ; select #3 ; land fetchIssue:3 ; land closeIssue:3 + seed 7 step 11: count: tab Issues (0), 1 open rows shown + land fetchIssues:open ; row-close 1 ; row-assign 0 ; row-close 0 ; land closeIssue:1 ; fail assign:1,guest ; row-assign 0 ; fail assign:4,guest ; fail closeIssue:3 ; user ann ; row-close 0 + seed 15 step 38: detail title: "#2 Bravolong gonesaving filescrash reportsretry assign to gu", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 + seed 18 step 19: count: tab Issues (1), 2 open rows shown + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-4.txt new file mode 100644 index 00000000..96c29b64 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-4.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-4: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-5.txt new file mode 100644 index 00000000..90180674 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-rx-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-6.txt new file mode 100644 index 00000000..17615345 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-6.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-rx-6: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-7.txt new file mode 100644 index 00000000..657323bb --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-7.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ a label added on one issue does not show on another +37/38 passed +== fuzz +desk12-rx-7: 2/20 seeds found a hole + seed 8 step 24: dropped guess: #4 assignee: showed "guest" while it was in flight, now "dana" + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; user bob ; land closeIssue:3 ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #1 ; land fetchComments:1 ; assign ; rename Delta ; close ; row-close 0 ; land fetchIssue:1 ; land search:Login fails ; assign ; user guest ; fail closeIssue:4 ; close ; fail assign:1,guest ; land closeIssue:1 + seed 12 step 36: closed row shown: #1 is closed on the server + land fetchIssues:open ; focus ; select #1 ; row-assign 1 ; select #4 ; land assign:3,guest ; row-assign 2 ; assign ; land assign:4,guest ; assign ; fail fetchLabels:1 ; land fetchComments:1 ; land fetchLabels:4 ; land fetchComments:4 ; land fetchIssue:4 ; land fetchIssue:1 ; assign ; land search:Typo in footer ; land assign:4,guest ; post me too ; close ; fail addComment:4,guest,me too ; selec diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-8.txt new file mode 100644 index 00000000..3b0148ef --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-8.txt @@ -0,0 +1,11 @@ +== hidden +✗ base: retry after not found shows the issue +✗ base: settings: retrying a failed load shows the settings +75/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ rows: a declined row close leaves a rename out on another issue +35/38 passed +== fuzz +desk12-rx-8: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-9.txt new file mode 100644 index 00000000..9c0899c2 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-9.txt @@ -0,0 +1,17 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ a label added on one issue does not show on another +✗ rows: a declined row close leaves a rename out on another issue +33/38 passed +== fuzz +desk12-rx-9: 2/20 seeds found a hole + seed 11 step 44: detail title: "retry#3 Crash on savestack trace attachedsaving filescrash r", server "Delta" + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; row-assign 0 ; row-close 0 ; select #3 ; focus ; close ; land fetchIssue:4 ; post me too ; + seed 15 step 24: detail title: "retry#2 Old buglong gonesaving filescrash reportsassign to g", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-1.txt new file mode 100644 index 00000000..05639a16 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-1.txt @@ -0,0 +1,10 @@ +== hidden +✗ base: settings: saving an invalid name asks nothing +76/77 passed +== overlap +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +37/38 passed +== fuzz +desk12-vanilla-1: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-10.txt new file mode 100644 index 00000000..0c3333ac --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-10.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +29/38 passed +== fuzz +desk12-vanilla-10: 3/20 seeds found a hole + seed 3 step 39: count: tab Issues (3), 2 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row- + seed 8 step 16: row title: #4 shows "Typo in footer", server "Bravo" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-11.txt new file mode 100644 index 00000000..520f87a6 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-11.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-vanilla-11: 3/20 seeds found a hole + seed 3 step 38: count: tab Issues (4), 3 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug + seed 4 step 33: row title: #1 shows "Login fails", server "Bravo" + rename Alpha ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; row-close 2 ; land fetchLabels:4 ; add-label ui ; add-label p1 ; row-assign 0 ; rename Delta ; close ; select #1 ; add-label bug ; close ; fail renameIssue:4,Delta ; rename Bravo ; land renameIssue:1,Bravo ; close ; land fetchComments:1 ; land fetchLabels:1 ; land addLabel:4,p1 ; row-assi + seed 8 step 16: row title: #4 shows "Typo in footer", server "Bravo" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-12.txt new file mode 100644 index 00000000..50b406a3 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-12.txt @@ -0,0 +1,18 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-vanilla-12: 2/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-2.txt new file mode 100644 index 00000000..7358c9a6 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-3.txt new file mode 100644 index 00000000..11d5e963 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-3.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-3: 2/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-4.txt new file mode 100644 index 00000000..1d5ae00b --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-4.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-4: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-5.txt new file mode 100644 index 00000000..b20aeb74 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-6.txt new file mode 100644 index 00000000..de893d02 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-6.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-6: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-7.txt new file mode 100644 index 00000000..ad648c87 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-7.txt @@ -0,0 +1,14 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +34/38 passed +== fuzz +desk12-vanilla-7: 1/20 seeds found a hole + seed 3 step 43: at rest: labels: #4 shows [docs], server [docs,ui] + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,guest ; add-label bu diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-8.txt new file mode 100644 index 00000000..96b30bf8 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-8.txt @@ -0,0 +1,29 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-vanilla-8: 5/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-9.txt new file mode 100644 index 00000000..68ce208e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-9.txt @@ -0,0 +1,22 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-vanilla-9: 3/20 seeds found a hole + seed 8 step 37: detail title: "#3 Crash on savestack trace attachedretryassign to bobrename", server "Alpha" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 ; land addLabel:4,ui ; select #3 ; focus ; fail fetchLabels:3 ; row-assign 0 ; assign ; land + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo + seed 17 step 19: dropped guess: #3 title: showed "Alpha" while it was in flight, now "Crash on save" + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; row-assign 0 ; add-label bug ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; land assign:1,guest ; row-close 0 ; row-close 1 ; row-assign 0 ; select #3 ; rename Bravo ; row-assign 0 ; land fetchLabels:3 ; rename Alpha ; assign ; f diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-1.txt new file mode 100644 index 00000000..7adfa732 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-1: 1/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-10.txt new file mode 100644 index 00000000..64d01d85 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-10.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk12-xstate-10: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-11.txt new file mode 100644 index 00000000..73e5ec2e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-11.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save failing after a later one succeeded leaves the later one +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +34/38 passed +== fuzz +desk12-xstate-11: 1/20 seeds found a hole + seed 16 step 39: at rest: comments: #2 shows [guest: fixed? | guest: me too | guest: me too], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-12.txt new file mode 100644 index 00000000..8764a683 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-12.txt @@ -0,0 +1,28 @@ +== hidden +✗ base: a failing list leaves the detail alone +✗ base: help: a failing detail leaves the results alone +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk12-xstate-12: 5/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 39: at rest: detail: "retryassign to guestassign error: declinedrenameclosepostp1 ", server "#4 Bravo" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 34: at rest: detail: "retryassign to annrenamerename error: declinedclosepostpost ", server "#2 Old bug" + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; add-label p1 ; add-label ui ; close ; user guest ; rename Alpha ; filter open ; land fetchLabels:2 ; fail renameIssue:2,Alpha ; assign ; land addLabel:2,p1 ; focus + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-2.txt new file mode 100644 index 00000000..449aa353 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-3.txt new file mode 100644 index 00000000..7550f230 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-3.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ labels that load while an add is out keep the add +37/38 passed +== fuzz +desk12-xstate-3: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-4.txt new file mode 100644 index 00000000..cd4d74ca --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-4.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-xstate-4: 1/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-5.txt new file mode 100644 index 00000000..d98cee76 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-6.txt new file mode 100644 index 00000000..3cdbcf7d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-6.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-6: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-7.txt new file mode 100644 index 00000000..b8b1da4b --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-7.txt @@ -0,0 +1,28 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-xstate-7: 4/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 31: detail title: "#3 stack trace attachedretryassign to guestcloserenamerename", server "Crash on save" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-8.txt new file mode 100644 index 00000000..6a7fb93b --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-8.txt @@ -0,0 +1,30 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-xstate-8: 5/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 38: at rest: comments: #2 shows [guest: fixed?], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-9.txt new file mode 100644 index 00000000..d7533a05 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-9.txt @@ -0,0 +1,28 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-xstate-9: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: row assignee: #4 shows null, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/repro/guess-while-new-question-loads.spec.tsx b/eval/results/desk12-2026-11-11/repro/guess-while-new-question-loads.spec.tsx new file mode 100644 index 00000000..bb58ba69 --- /dev/null +++ b/eval/results/desk12-2026-11-11/repro/guess-while-new-question-loads.spec.tsx @@ -0,0 +1,40 @@ +/** + * @jsxImportSource @solidjs/web + * @vitest-environment jsdom + */ +import { test, expect } from "vitest"; +import { Loading, action, createOptimistic, createSignal, refresh, flush } from "solid-js"; +import { render } from "../src/index.js"; +const tick = async () => { for (let i = 0; i < 4; i++) { await new Promise(r => setTimeout(r, 0)); flush(); } }; +// A guess made while its source is loading a new question (the selection just changed). +for (const kind of ["object", "array"] as const) test(`guess while the source loads a new question (${kind})`, async () => { + const asks: { key: string; res: (v: any) => void }[] = []; + const ask = (key: string) => new Promise(res => asks.push({ key, res })); + const pick = (key: string) => { const i = asks.findIndex(a => a.key === key); if (i < 0) throw new Error("no ask " + key + " in " + asks.map(a => a.key)); return asks.splice(i, 1)[0]; }; + const c = document.body.appendChild(document.createElement("div")); + const frames: string[] = []; const snap = (l: string) => frames.push(`${l.padEnd(26)} ${c.textContent}`); + let select!: (id: number) => void, write!: () => any; + render(() => { + const [sel, setSel] = createSignal(0); + const [truth] = createSignal(() => sel() === 0 ? (kind === "object" ? null : []) : ask(`fetch:${sel()}`)); + const [shown, guess] = createOptimistic(() => truth()); + select = setSel; + write = action(function* () { + if (kind === "object") guess((d: any) => d == null ? null : { ...d, title: "Bravo" }); + else guess((cs: any[]) => [...cs, "ping"]); + yield ask("write"); + refresh(truth); + }); + return loading}> + {kind === "object" ? {shown()?.title ?? "-"} : {shown().join(",")}} + ; + }, c); + await tick(); snap("start"); + select(3); await tick(); snap("select #3"); + write(); await tick(); snap("write while #3 loads"); + pick("fetch:3").res(kind === "object" ? { title: "Old bug" } : ["seen it too"]); await tick(); snap("#3 lands, write out"); + frames.push("still out: " + asks.map(a => a.key).join(", ")); + console.log(frames.join("\n")); + // the guess either applies to #3's answer, or it is void (D-34) and #3's answer shows as it is + expect(c.textContent).toBe(kind === "object" ? "Bravo" : "seen it too,ping"); +}); diff --git a/eval/results/desk12-2026-11-11/repro/probe-guess-while-loading.mjs b/eval/results/desk12-2026-11-11/repro/probe-guess-while-loading.mjs new file mode 100644 index 00000000..2eb71320 --- /dev/null +++ b/eval/results/desk12-2026-11-11/repro/probe-guess-while-loading.mjs @@ -0,0 +1,45 @@ +// Rename while the detail is loading: select #3, hold its detail, rename to "Bravo", let the detail land (old title), +// and read the heading while the rename is still out. Then the same for a post while comments load. +import { build } from "/home/claude/reactivescript/eval/harness/build.mjs"; +import { launch, serve } from "/home/claude/reactivescript/eval/harness/driver.mjs"; +import { answer, freshServer } from "/home/claude/reactivescript/eval/harness/fuzz.mjs"; +const browser = await launch(); +const norm = s => (s ?? "").replace(/\s+/g, " ").trim(); +for (const dir of process.argv.slice(2)) { + const name = dir.split("/").pop(); const out = []; + try { + const b = await build(dir); if (!b.ok) { console.log(name, "BUILD FAIL"); continue; } + const server = await serve(b.dist); + for (const mode of ["rename", "post"]) { + const srv = freshServer(); const page = await browser.newPage(); + await page.goto(server.url); await page.waitForFunction(() => globalThis.__flights !== undefined, null, { timeout: 5000 }); + const pend = () => page.evaluate(() => globalThis.__flights.pending()); + const land = async k => { const a = answer(srv, k, false); await page.evaluate(([k, v]) => globalThis.__flights.land(k, v), [k, a.value]); await page.waitForTimeout(60); }; + const landAll = async (skip = () => false) => { for (let g = 0; g < 30; g++) { await page.waitForTimeout(60); const keys = (await pend()).filter(k => !skip(k)); if (!keys.length) return; for (const k of keys) await land(k); } }; + await landAll(); + await page.locator('[data-testid="issue"]', { hasText: "#3 " }).first().click({ timeout: 2000 }); + await page.waitForTimeout(80); + const isW = k => /^(renameIssue|addComment)/.test(k); + try { + if (mode === "rename") { + await page.locator('[data-testid="title-input"]').first().fill("Bravo", { timeout: 1000 }); + await page.locator('[data-testid="rename"]').first().click({ timeout: 1000 }); + await landAll(k => isW(k) || /^(fetchIssues|search)/.test(k) && false); // the detail (and the rest) land; the rename stays out + await landAll(isW); + const h = norm(await page.locator('[data-testid="detail"]').first().textContent()); + out.push(`rename:${(await pend()).some(isW) ? (h.includes("#3 Bravo") ? "ok" : "LOST[" + h.slice(0, 30) + "]") : "no-write"}`); + } else { + await page.locator('[data-testid="comment-input"]').first().fill("ping", { timeout: 1000 }); + await page.locator('[data-testid="post"]').first().click({ timeout: 1000 }); + await landAll(isW); + const c = norm((await page.locator('[data-testid="comments"]').allTextContents()).join(" ")) || norm(await page.locator('[data-testid="detail"]').first().textContent()); + out.push(`post:${(await pend()).some(isW) ? (c.includes("ping") && c.includes("seen it too") ? "ok" : c.includes("ping") ? "TRUTH-DROPPED" : "LOST") : "no-write"}`); + } + } catch (e) { out.push(`${mode}:unavailable`); } + await page.close(); + } + server.close(); + } catch (e) { out.push("ERR " + e.message.split("\n")[0].slice(0, 80)); } + console.log(name, out.join(" ")); +} +await browser.close(); diff --git a/eval/results/desk12-2026-11-11/repro/probe-guess-while-loading.txt b/eval/results/desk12-2026-11-11/repro/probe-guess-while-loading.txt new file mode 100644 index 00000000..eff63fb2 --- /dev/null +++ b/eval/results/desk12-2026-11-11/repro/probe-guess-while-loading.txt @@ -0,0 +1,60 @@ +desk12-foldkit-1 rename:ok post:ok +desk12-foldkit-2 rename:ok post:ok +desk12-foldkit-3 rename:ok post:ok +desk12-foldkit-4 rename:ok post:ok +desk12-foldkit-5 rename:ok post:ok +desk12-foldkit-6 rename:ok post:ok +desk12-foldkit-7 rename:LOST[#3 Crash on savestack trace at] post:LOST +desk12-foldkit-8 rename:LOST[#3 Crash on savestack trace at] post:LOST +desk12-foldkit-9 rename:no-write post:no-write +desk12-foldkit-10 rename:LOST[#3 Crash on savestack trace at] post:no-write +desk12-foldkit-11 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-foldkit-12 rename:LOST[retry#3 Crash on savestack tra] post:ok +desk12-react-1 rename:ok post:ok +desk12-react-2 rename:ok post:ok +desk12-react-3 rename:ok post:ok +desk12-react-4 rename:ok post:ok +desk12-react-5 rename:ok post:ok +desk12-react-6 rename:ok post:ok +desk12-react-7 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-react-8 rename:unavailable post:ok +desk12-react-9 rename:ok post:ok +desk12-react-10 rename:ok post:ok +desk12-react-11 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-react-12 rename:ok post:ok +desk12-rx-1 rename:LOST[detail error: issue #0 not fou] post:ok +desk12-rx-2 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-rx-3 rename:ok post:ok +desk12-rx-4 rename:ok post:ok +desk12-rx-5 rename:ok post:ok +desk12-rx-6 rename:ok post:ok +desk12-rx-7 rename:LOST[retry#3 Crash on savestack tra] post:TRUTH-DROPPED +desk12-rx-8 rename:ok post:ok +desk12-rx-9 rename:LOST[retry#3 Crash on savestack tra] post:TRUTH-DROPPED +desk12-rx-10 rename:LOST[retry#3 Crash on savestack tra] post:TRUTH-DROPPED +desk12-rx-11 rename:ok post:ok +desk12-rx-12 rename:unavailable post:unavailable +desk12-vanilla-1 rename:ok post:ok +desk12-vanilla-2 rename:ok post:ok +desk12-vanilla-3 rename:ok post:ok +desk12-vanilla-4 rename:ok post:ok +desk12-vanilla-5 rename:ok post:ok +desk12-vanilla-6 rename:ok post:ok +desk12-vanilla-7 rename:ok post:ok +desk12-vanilla-8 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-vanilla-9 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-vanilla-10 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-vanilla-11 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-vanilla-12 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-xstate-1 rename:ok post:ok +desk12-xstate-2 rename:ok post:ok +desk12-xstate-3 rename:ok post:ok +desk12-xstate-4 rename:ok post:ok +desk12-xstate-5 rename:ok post:ok +desk12-xstate-6 rename:ok post:ok +desk12-xstate-7 rename:LOST[#3 Crash on savestack trace at] post:ok +desk12-xstate-8 rename:LOST[#3 Crash on savestack trace at] post:LOST +desk12-xstate-9 rename:LOST[retry#3 Crash on savestack tra] post:ok +desk12-xstate-10 rename:no-write post:ok +desk12-xstate-11 rename:ok post:ok +desk12-xstate-12 rename:ok post:ok diff --git a/eval/results/desk12-2026-11-11/repro/switch-after-failed-list.spec.tsx b/eval/results/desk12-2026-11-11/repro/switch-after-failed-list.spec.tsx new file mode 100644 index 00000000..f90536c2 --- /dev/null +++ b/eval/results/desk12-2026-11-11/repro/switch-after-failed-list.spec.tsx @@ -0,0 +1,52 @@ +/** + * @jsxImportSource @solidjs/web + * @vitest-environment jsdom + */ +import { test, expect } from "vitest"; +import { For, Loading, Errored, action, createOptimisticStore, createMemo, createSignal, refresh, flush, untrack } from "solid-js"; +import { render } from "../src/index.js"; +const V = process.env.V ?? "full"; +const tick = async () => { for (let i = 0; i < 4; i++) { await new Promise(r => setTimeout(r, 0)); flush(); } }; +test("a switch between two lists shows the other list's rows", async () => { + const asks: { key: string; res: (v: any) => void; rej: (e: any) => void }[] = []; + const ask = (key: string) => new Promise((res, rej) => asks.push({ key, res, rej })); + const pick = (key: string) => { const i = asks.findIndex(a => a.key === key); if (i < 0) throw new Error("no ask " + key + " in " + asks.map(a => a.key)); return asks.splice(i, 1)[0]; }; + const c = document.body.appendChild(document.createElement("div")); + const frames: string[] = []; const snap = (l: string) => frames.push(`${l.padEnd(22)} ${c.textContent}`); + let assign!: (id: number) => any, toOpen!: () => void, toClosed!: () => void; + render(() => { + const [filter, setFilter] = createSignal("open"); + const openQ = createMemo(() => ask("fetch:open")); + const closedQ = createMemo(() => ask("fetch:closed")); + const fill = (q: () => any) => (d: any) => { const v = q(); d.splice(0, d.length, ...v.map((x: any) => ({ ...x }))); }; + const [open] = createOptimisticStore(fill(openQ), [] as any[], { key: "id" } as any); + const [closed, setClosed] = createOptimisticStore(fill(closedQ), [] as any[], { key: "id" } as any); + const issues = createMemo(() => filter() === "open" ? open : closed); + const visible = createMemo(() => issues().filter((r: any) => !r.closing)); + assign = action(function* (id: number) { + const i = untrack(() => closed.findIndex((r: any) => r.id === id)); + setClosed((d: any) => { d[i].a = "guest"; }); + yield ask(`assign:${id}`); + refresh(closedQ); refresh(closed); + }); + toOpen = () => { setFilter("open"); refresh(openQ); refresh(open); }; + toClosed = () => { setFilter("closed"); }; + return loading {filter()}}> + error: {e.message}}> + {filter()}:{(r: any) => [#{r.id} {r.a}]} + + ; + }, c); + await tick(); + toClosed(); await tick(); snap("filter closed"); + if (V === "nofail") pick("fetch:open").res([{ id: 1, a: "" }]); else pick("fetch:open").rej(new Error("declined")); await tick(); snap("open fetch fails"); + pick("fetch:closed").res([{ id: 2, a: "eve" }]); await tick(); snap("closed lands"); + if (V !== "noassign") { assign(2); await tick(); snap("assign #2"); } + toOpen(); await tick(); snap("filter open"); + if (V !== "noassign") { pick("assign:2").res({}); await tick(); snap("assign accepted"); + pick("fetch:closed").res([{ id: 2, a: "guest" }]); await tick(); snap("closed refetch lands"); } + pick("fetch:open").res([{ id: 1, a: "" }]); await tick(); snap("open refetch lands"); + frames.push("still out: " + asks.map(a => a.key).join(", ")); + console.log(frames.join("\n")); + expect(c.textContent).toBe("open:[#1 ]"); +}); diff --git a/eval/results/desk12-2026-11-11/repro/two-close-guesses.spec.tsx b/eval/results/desk12-2026-11-11/repro/two-close-guesses.spec.tsx new file mode 100644 index 00000000..75895aa4 --- /dev/null +++ b/eval/results/desk12-2026-11-11/repro/two-close-guesses.spec.tsx @@ -0,0 +1,39 @@ +/** + * @jsxImportSource @solidjs/web + * @vitest-environment jsdom + */ +import { test, expect } from "vitest"; +import { For, Loading, action, createOptimistic, createOptimisticStore, createMemo, createSignal, refresh, flush } from "solid-js"; +import { render } from "../src/index.js"; +const tick = async () => { for (let i = 0; i < 4; i++) { await new Promise(r => setTimeout(r, 0)); flush(); } }; +const V = process.env.V ?? "full"; +test("two close guesses on one list: both hold until the refresh lands", async () => { + const asks: { key: string; res: (v: any) => void }[] = []; + const ask = (key: string) => new Promise(res => asks.push({ key, res })); + const pick = (key: string) => { const i = asks.findIndex(a => a.key === key); if (i < 0) throw new Error("no ask " + key + " in " + asks.map(a => a.key)); return asks.splice(i, 1)[0]; }; + const c = document.body.appendChild(document.createElement("div")); + const frames: string[] = []; const snap = (l: string) => frames.push(`${l.padEnd(18)} ${c.textContent || "(empty)"}`); + let close!: (id: number) => any, assign!: (id: number) => any; + let server = [{ id: 1, a: "" }, { id: 4, a: "dana" }]; + render(() => { + const [issues, setIssues] = createOptimisticStore(async (d: any) => { const v = await ask("fetch"); d.splice(0, d.length, ...v); }, [] as any[], { key: "id" } as any); + const [truth] = createSignal(() => []); + const [hidden, guessHidden] = createOptimistic(() => truth()); + const shown = createMemo(() => issues.filter((i: any) => !hidden().includes(i.id))); + close = action(function* (id: number) { guessHidden(h => [...h, id]); yield ask(`close:${id}`); refresh(issues); }); + assign = action(function* (id: number) { setIssues((d: any) => { d.find((x: any) => x.id === id).a = "guest"; }); yield ask(`assign:${id}`); refresh(issues); }); + return loading}>{r => [#{r.id}]}; + }, c); + await tick(); + pick("fetch").res(server.map(x => ({ ...x }))); await tick(); snap("loaded"); + close(1); await tick(); snap("close #1"); + if (V !== "noassign") { assign(4); await tick(); snap("assign #4"); } + close(4); await tick(); snap("close #4"); + server = server.filter(x => x.id !== 4); + pick("close:4").res({}); await tick(); snap("close #4 accepted"); + server = server.filter(x => x.id !== 1); + pick("close:1").res({}); await tick(); snap("close #1 accepted"); + frames.push("still out: " + asks.map(a => a.key).join(", ")); + console.log(frames.join("\n")); + expect(c.textContent).toBe(""); // the fetch the closes asked is still out: both guesses hold (D-33, F-11) +}); diff --git a/eval/results/desk12-2026-11-11/stats.py b/eval/results/desk12-2026-11-11/stats.py new file mode 100644 index 00000000..5665da4e --- /dev/null +++ b/eval/results/desk12-2026-11-11/stats.py @@ -0,0 +1,39 @@ +import re, random, json, sys +rows={} +for l in open(sys.argv[1] if len(sys.argv)>1 else 'summary.txt'): + a,n,h,o,z=re.match(r'(\w+)-(\d+) hidden/overlap=(\d+)/77 (\d+)/38\s+fuzz=(\d+)/20',l).groups() + rows[(a,int(n))]=dict(h=77-int(h),o=38-int(o),z=int(z)) +excl={} # fuzz holes traced to legal holds: (arm,n) -> count +def D(k): r=rows[k]; return r['h']+r['o']+r['z']-excl.get(k,0) +def perm(x,y,seed=12,N=100000): + random.seed(seed); obs=sum(y)/len(y)-sum(x)/len(x); pool=x+y; c=0 + for _ in range(N): + random.shuffle(pool); xx=pool[:len(x)]; yy=pool[len(x):] + if sum(yy)/len(yy)-sum(xx)/len(xx)>=obs-1e-12: c+=1 + return (c+1)/(N+1) +def boot(x,y,seed=12,N=10000): + random.seed(seed); d=[] + for _ in range(N): + xx=[random.choice(x) for _ in x]; yy=[random.choice(y) for _ in y] + d.append(sum(xx)/len(xx)-sum(yy)/len(yy)) + d.sort(); return d[int(.05*N)], d[int(.95*N)-1] +try: usd=json.load(open('usd.json')) +except Exception: usd=None +front=range(1,7); small=range(7,13); A=('rx','react','vanilla','foldkit','xstate') +for a in A: + for tier,rg in (('frontier',front),('small',small)): + v=[D((a,n)) for n in rg]; c=sum(1 for x in v if x==0) + hz=[sum(rows[(a,n)][k] for n in rg)/len(rg) for k in 'hoz'] + s=f" {a:8}{tier:9} mean {sum(v)/len(v):5.2f} median {sorted(v)[2]/2+sorted(v)[3]/2:4.1f} per desk {v} h/o/z {hz[0]:.1f}/{hz[1]:.1f}/{hz[2]:.1f} correct {c}" + if usd: + cost=[x[0] for x in usd[f'{a}-{tier}']]; m=sum(cost)/len(cost) + s+=f" $/run {m:.3f} $/correct "+(f"${sum(cost)/c:.3f}" if c else "undefined") + print(s) +S={a:[D((a,n)) for n in small] for a in A}; F={a:[D((a,n)) for n in front] for a in A} +ps={'H12a rx holds if upper < 1: {hi<1}") +print(f" H12d small xstate holds if upper < 1: False + H12d small xstate