From 7e654650ed390609ac2b34c6d991ed0aedd4f0c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 8 Nov 2026 10:08:57 +0000 Subject: [PATCH 01/23] Round 11 prereg draft; brief: the detail's retry is always present PREREG-desk-11 (DRAFT): round 10's design again on Solid next 2789ca9c (every eval bug fixed, #4689 included) with the current guide. H11a replicates H10a; H11c compares frontier .rx with frontier React. The brief now says the detail's retry button is always present while an issue is selected, which round 10 left unclear (25 of 36 desks failed the rename-reload overlap test over it). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-11.md | 59 +++++++++++++++++++++++++++++++++ eval/tasks/desk9/brief-plain.md | 3 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 eval/PREREG-desk-11.md diff --git a/eval/PREREG-desk-11.md b/eval/PREREG-desk-11.md new file mode 100644 index 00000000..b0c814ac --- /dev/null +++ b/eval/PREREG-desk-11.md @@ -0,0 +1,59 @@ +# Round 11: round 10 again, on a Solid with every eval bug fixed and the current guide + +Pre-registration, committed before any round 11 agent runs. Status: DRAFT (needs Ryan's OK before it is frozen). + +## Why + +Round 10 (`results/desk-2026-11-02`) confirmed H10a: small .rx desks had 6.7 defects, small React desks 14.7. It +rejected H10c: frontier React desks had 3.0. Most of the .rx frontier defects traced to Solid bugs, and those have +all been fixed since (#4627 #4629 #4631 #4638 #4661 #4666 #4689). Regrading round 10's own apps on the fixed Solid +moved frontier .rx from 7.3 to 4.7 raw defects. Since round 10 the guide has gained `guide/reference.md`, the rules on +one page (PR #29), and the instant model (PR #34). A 3-desk check with `reference.md` (`results/cost-2026-11-03`, not +counted) gave frontier .rx 2.0 defects. Round 11 asks whether that holds with fresh agents, all four arms, both tiers, +and the same design as round 10, so the two rounds compare directly. + +Round 10's plan named maintenance as round 11. It moves to round 12 and runs on round 11's apps, so the apps it extends +were written with the current guide on a fixed Solid. + +## Task and arms (as round 10 unless stated) + +- Task: `tasks/desk9` with `brief-plain.md`. **One change from round 10**: the detail's `retry` button is now always + present while an issue is selected, not only after a failure. In round 10, 25 of 36 desks across all arms failed + "a detail that loads while a rename is out keeps the new title" because the brief left this unclear. Graded by + `hidden.mjs` (77) and `hidden-overlap.mjs` (38), unchanged. +- Arms: `rx` with `--inline-docs` (now `language.md`, `async.md` and `reference.md`), `foldkit` with `--inline-docs`, + `react`, `vanilla`. `rxc` is built from the commit that holds this file once it is frozen. +- Solid: `solid-next` at `next` 2789ca9c (2026-11-08; includes the #4689 fix, which is not yet in an rc), built from + source. The same build is used for every .rx run and grade. rc.22 isn't out; if it ships before the round starts and + holds every fix, it replaces this pin. +- Harness: subagents, as in round 10. Numbers are comparable only within one harness: on the Foldkit side check + (2026-11-08), the CLI harness alone moved Foldkit from 4.7 to 2.7 defects. + +## Runs, grading + +As round 10: 3 frontier and 6 small runs per arm (36), one interleaved batch, at most 15 `./check`/`./test` calls, +graded by `harness/grade.sh`. The definitions of legal holds, defects, correct and cost are round 10's. + +## Hypotheses and tests + +- **H11a** (primary, replication of H10a): small .rx runs have fewer defects than small React runs. One-sided + permutation test on mean defects, 100,000 relabellings, seed 11. +- **H11b**: small .rx has fewer defects than small plain, and fewer than small Foldkit. Same test, Holm-corrected with + H11a. +- **H11c**: frontier .rx runs have no more defects than frontier React runs. Difference in means with a 90% bootstrap + interval (10,000 resamples, seed 11); the claim holds if the upper end is under 1 defect. (Round 10's H10c compared + small .rx with frontier React. That comparison is reported again, as H10c was.) +- **Cost per correct app** (descriptive), by arm and tier. In round 10 no app was correct, so this was undefined. +- Reported alongside: the change from round 10 for each arm and tier, defects by kind, turns, cost per run, and lines. + +The direction is the claim being tested. A null or reversed result is reported as plainly. + +## Gate before freezing + +The 12 existing .rx desks (round 10 1-9, cost check 1-3) are regraded on the pinned Solid. A regression against +rc.21 is filed upstream and fixed or recorded before the round starts. + +## Cost + +About $30 at round 10's prices: frontier $2.90 (.rx), $1.35 (React), $1.24 (plain), $3.56 (Foldkit) per run; +small under $0.16. diff --git a/eval/tasks/desk9/brief-plain.md b/eval/tasks/desk9/brief-plain.md index d34fdeb3..bb82fb33 100644 --- a/eval/tasks/desk9/brief-plain.md +++ b/eval/tasks/desk9/brief-plain.md @@ -67,7 +67,8 @@ Elements the tests look for carry these `data-testid`s. Text in quotes must appe - The selected issue comes from `fetchIssue(id)`. While it loads: `loading #`. - An issue: `# ` and its body. - `null`: `detail error: issue #<id> not found`. A failure: `detail error: <message>`. - - A button `retry` (`data-testid="retry"`) calls `fetchIssue(id)` again and shows its answer. + - While an issue is selected, the detail always has a button `retry` (`data-testid="retry"`), whether it shows the + issue, `not found` or a failure. It calls `fetchIssue(id)` again and shows its answer. 5. **Assign.** While an issue is selected, the detail has a button `assign` (`data-testid="assign"`) reading `assign to <current user>`. Clicking it: - shows the current user as the issue's assignee in its list row straight away; From 2b2156d5153a43365423a7a1cfe85b4f54919a8a Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sun, 8 Nov 2026 21:56:33 +0000 Subject: [PATCH 02/23] Lower a whole optimistic guess to the store setter's return A whole guess on an optimistic store (optimistic rows := rs => rs.map(...)) was written into the draft index by index, which replaces each matched row object. It is now the setter's return, which Solid diffs shallowly against the view as guesses. Clears round 10 rx-8 fuzz seed 3 on next 2789ca9c; the other 11 desks grade unchanged. Gate result recorded in PREREG-desk-11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-11.md | 6 ++++++ ref/compile/rt-web.ts | 2 +- ref/compile/rt.ts | 2 +- ref/oracle/lower.ts | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/eval/PREREG-desk-11.md b/eval/PREREG-desk-11.md index b0c814ac..fd1480cd 100644 --- a/eval/PREREG-desk-11.md +++ b/eval/PREREG-desk-11.md @@ -53,6 +53,12 @@ The direction is the claim being tested. A null or reversed result is reported a The 12 existing .rx desks (round 10 1-9, cost check 1-3) are regraded on the pinned Solid. A regression against rc.21 is filed upstream and fixed or recorded before the round starts. +Result (2026-11-08): on 2789ca9c one desk had a new deterministic hole (round 10 rx-8, fuzz seed 3: a close guess +during another action's refresh, then the refresh fails). It came from the lowering, not from the app: a whole guess +(`optimistic rows := rs => rs.map(...)`) was written into the draft row by row, which replaces each matched row. It now +lowers to the setter's return, which Solid diffs shallowly against the view as guesses (its default). With that, the +12 desks grade as on rc.21 (total 52 defects), except that seed 3 is gone. The plain Solid case is solidjs/solid#4724. + ## Cost About $30 at round 10's prices: frontier $2.90 (.rx), $1.35 (React), $1.24 (plain), $3.56 (Foldkit) per run; diff --git a/ref/compile/rt-web.ts b/ref/compile/rt-web.ts index 849d001d..b3e13828 100644 --- a/ref/compile/rt-web.ts +++ b/ref/compile/rt-web.ts @@ -177,7 +177,7 @@ export function createApp(opts: { root: any; api?: Record<string, (...a: any[]) const [store, setStore] = (createOptimisticStore as any)((draft: any) => { const r = fn(draft); if (r instanceof Flight || r instanceof Promise) return (async () => assignDraft(draft, r instanceof Flight ? await r.first : await r))(); assignDraft(draft, r); }, key ? [] : {}, key ? { key } : undefined); const get = () => store; storeAccs.add(get); const plain = () => { throw new Error("a plain write to an optimistic store is not lowered yet"); }; - return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) { assignDraft(d, typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v); return; } let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; + return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) return typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v; /* a whole guess is the setter\'s return, which Solid diffs shallowly against the view as guesses */ let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; } if (mode.optimistic) { const [truth, setTruth] = createSignal(fn as any, opts as any); diff --git a/ref/compile/rt.ts b/ref/compile/rt.ts index ad40b5f5..333c7b64 100644 --- a/ref/compile/rt.ts +++ b/ref/compile/rt.ts @@ -187,7 +187,7 @@ export function createApp(opts: { api?: Record<string, (...a: any[]) => unknown> const [store, setStore] = (createOptimisticStore as any)((draft: any) => { const r = fn(draft); if (r instanceof Flight || r instanceof Promise) return (async () => assignDraft(draft, r instanceof Flight ? await r.first : await r))(); assignDraft(draft, r); }, key ? [] : {}, key ? { key } : undefined); const get = () => store; storeAccs.add(get); const plain = () => { throw new Error("a plain write to an optimistic store is not lowered yet"); }; - return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) { assignDraft(d, typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v); return; } let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; + return [get, { set: plain, path: plain, guess: (v, keys = []) => setStore((d: any) => { if (keys.length === 0) return typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v; /* a whole guess is the setter\'s return, which Solid diffs shallowly against the view as guesses */ let o = d; for (const k of keys.slice(0, -1)) o = o[k]; const last = keys[keys.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }) }]; } if (mode.optimistic) { const [truth, setTruth] = createSignal(fn as any, opts as any); diff --git a/ref/oracle/lower.ts b/ref/oracle/lower.ts index b26eae1c..11d87b12 100644 --- a/ref/oracle/lower.ts +++ b/ref/oracle/lower.ts @@ -472,7 +472,7 @@ export class Oracle { const path = s.target.path.map(p => typeof p === "string" ? p : String(untrack(() => this.ev(p, scope, env)))); if (s.k === "optimistic") { if (!b.setGuess) throw new Error("OPTIMISTIC_NO_QUESTION"); - b.setGuess((d: any) => { if (path.length === 0) { assignDraft(d, typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v); return; } let o = d; for (const k of path.slice(0, -1)) o = o[k]; const last = path[path.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }); // W-6 per key + b.setGuess((d: any) => { if (path.length === 0) return typeof v === "function" ? (v as (x: unknown) => unknown)(d) : v; /* a whole guess is the setter\'s return, which Solid diffs shallowly against the view as guesses */ let o = d; for (const k of path.slice(0, -1)) o = o[k]; const last = path[path.length - 1]; o[last] = typeof v === "function" ? (v as (x: unknown) => unknown)(o[last]) : v; }); // W-6 per key return; } if (path.length === 0) b.set(v); From e1b2f3483fb6f971334b6758918c8c5a85707cf8 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sun, 8 Nov 2026 21:56:44 +0000 Subject: [PATCH 03/23] PREREG-desk-11: state the gate result precisely Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-11.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eval/PREREG-desk-11.md b/eval/PREREG-desk-11.md index fd1480cd..32d682b9 100644 --- a/eval/PREREG-desk-11.md +++ b/eval/PREREG-desk-11.md @@ -57,7 +57,8 @@ Result (2026-11-08): on 2789ca9c one desk had a new deterministic hole (round 10 during another action's refresh, then the refresh fails). It came from the lowering, not from the app: a whole guess (`optimistic rows := rs => rs.map(...)`) was written into the draft row by row, which replaces each matched row. It now lowers to the setter's return, which Solid diffs shallowly against the view as guesses (its default). With that, the -12 desks grade as on rc.21 (total 52 defects), except that seed 3 is gone. The plain Solid case is solidjs/solid#4724. +other 11 desks grade as before the change (52 defects over the 12; round 10 rx-9 fuzz seed 8, a known hole, is +deterministic on both lowerings and was missed once by timing), and seed 3 is gone. The plain Solid case is solidjs/solid#4724. ## Cost From 01f3c562a00b6ddb8c988cbf59ad01f16abfb372 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 10 Nov 2026 21:46:50 +0000 Subject: [PATCH 04/23] PREREG-desk-11: repin Solid to next 6751472f, record the gate result Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-11.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eval/PREREG-desk-11.md b/eval/PREREG-desk-11.md index 32d682b9..6d23b9ca 100644 --- a/eval/PREREG-desk-11.md +++ b/eval/PREREG-desk-11.md @@ -23,8 +23,8 @@ were written with the current guide on a fixed Solid. `hidden.mjs` (77) and `hidden-overlap.mjs` (38), unchanged. - Arms: `rx` with `--inline-docs` (now `language.md`, `async.md` and `reference.md`), `foldkit` with `--inline-docs`, `react`, `vanilla`. `rxc` is built from the commit that holds this file once it is frozen. -- Solid: `solid-next` at `next` 2789ca9c (2026-11-08; includes the #4689 fix, which is not yet in an rc), built from - source. The same build is used for every .rx run and grade. rc.22 isn't out; if it ships before the round starts and +- Solid: `solid-next` at `next` 6751472f (2026-11-10; includes the fixes for #4689, #4724, #4737, #4738, #4739, #4748 + and #4751, none yet in an rc), built from source. The same build is used for every .rx run and grade. rc.22 isn't out; if it ships before the round starts and holds every fix, it replaces this pin. - Harness: subagents, as in round 10. Numbers are comparable only within one harness: on the Foldkit side check (2026-11-08), the CLI harness alone moved Foldkit from 4.7 to 2.7 defects. @@ -60,6 +60,11 @@ lowers to the setter's return, which Solid diffs shallowly against the view as g other 11 desks grade as before the change (52 defects over the 12; round 10 rx-9 fuzz seed 8, a known hole, is deterministic on both lowerings and was missed once by timing), and seed 3 is gone. The plain Solid case is solidjs/solid#4724. +Result (2026-11-10), repinned to 6751472f: regrading on the intermediate next builds (c9816808, 79b6541d) turned up +five more Solid bugs, each reduced to a plain Solid repro and fixed upstream (#4724 by #4729, #4737/#4738/#4739 by +#4740, #4748 by #4749, #4751 by #4753). On 6751472f the 12 desks total 51 defects, against 52 on 2789ca9c: no desk +gained a hole, and round 10 rx-6 fuzz seed 12 is gone. The gate passes. + ## Cost About $30 at round 10's prices: frontier $2.90 (.rx), $1.35 (React), $1.24 (plain), $3.56 (Foldkit) per run; From 60173beec607ae9aa73233e33e748cf61eda9ffb Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 10 Nov 2026 22:31:24 +0000 Subject: [PATCH 05/23] Count only defects: the assign-two overlap test accepts the H-11 hold; prereg says so The overlap test 'two issues assigned at once' required the declined row to revert while the other assignment was still out. A list held whole until both settle is a legal hold, not a defect, so the test now checks that the decline leaves the other row alone and that both rows are right once everything has answered. PREREG-desk-11 states that only defects are counted and that round 10's numbers are regraded the same way (without the brief-caused retry test). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-11.md | 12 +++++++++++- eval/tasks/tracker/hidden-overlap.mjs | 6 +++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/eval/PREREG-desk-11.md b/eval/PREREG-desk-11.md index 6d23b9ca..c0d918ad 100644 --- a/eval/PREREG-desk-11.md +++ b/eval/PREREG-desk-11.md @@ -32,7 +32,17 @@ were written with the current guide on a fixed Solid. ## Runs, grading As round 10: 3 frontier and 6 small runs per arm (36), one interleaved batch, at most 15 `./check`/`./test` calls, -graded by `harness/grade.sh`. The definitions of legal holds, defects, correct and cost are round 10's. +graded by `harness/grade.sh`. The definitions of correct and cost are round 10's. **Only defects are counted**: no +legal hold and no failure the brief caused is in any count or table. + +- A legal hold is a consistent hold that settles (2026-10-30 ruling). The one hidden test that asserted one (overlap, + "two issues assigned at once": it required the declined row to revert while the other assignment was still out, + which a list held whole until both settle does not do) now checks only what is a defect, that the decline does not + take the other row with it and that both rows are right once everything has answered. Committed with this file. +- A fuzz hole counts unless it is traced to a legal hold or to behaviour Solid's rules make legal; each exclusion is + listed with its trace. +- Round 10's numbers, wherever they are compared with round 11's, are regraded with the same tests, and without "a + detail that loads while a rename is out keeps the new title", which round 10's brief caused (above). ## Hypotheses and tests diff --git a/eval/tasks/tracker/hidden-overlap.mjs b/eval/tasks/tracker/hidden-overlap.mjs index 39132959..98f5ae60 100644 --- a/eval/tasks/tracker/hidden-overlap.mjs +++ b/eval/tasks/tracker/hidden-overlap.mjs @@ -1,5 +1,6 @@ // Round 3 hidden tests: overlapping requests and writes. Never shown to the agent. import { D1, OPEN, selected } from "./fixtures.mjs"; +const AFTER = OPEN.map(i => i.id === 1 ? { ...i, assignee: "ann" } : i); // the server once #1 is assigned and #3 declined export default [ { name: "two issues assigned at once: declining one reverts only that row", async run(app) { @@ -9,8 +10,11 @@ export default [ await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.click("assign"); await app.fail("assign:3,ann", "declined"); - await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); + // #3 may revert now, or stay "(ann)" until #1 lands too: a list held whole until both settle is a legal hold (.rx H-11, + // Solid D-8), not a defect. What must not happen is the decline taking #1 with it. + await app.see("list", "#1 Login fails (ann)"); await app.land("assign:1,ann", { id: 1, assignee: "ann" }); + await app.landIfAsked("fetchIssues:open", AFTER); // an app that re-asks the list after a write holds until it answers await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); } }, { name: "a list that loads while an assignment is out keeps the new assignee", async run(app) { From b8b9e0b41a261b06354a3d25274887f45538b32f Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 10 Nov 2026 22:54:19 +0000 Subject: [PATCH 06/23] Freeze PREREG-desk-11 (Ryan, 2026-11-10) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-11.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/PREREG-desk-11.md b/eval/PREREG-desk-11.md index c0d918ad..c445b762 100644 --- a/eval/PREREG-desk-11.md +++ b/eval/PREREG-desk-11.md @@ -1,6 +1,6 @@ # Round 11: round 10 again, on a Solid with every eval bug fixed and the current guide -Pre-registration, committed before any round 11 agent runs. Status: DRAFT (needs Ryan's OK before it is frozen). +Pre-registration, committed before any round 11 agent runs. Status: FROZEN (Ryan, 2026-11-10: "freeze"). ## Why From c8e5ab060303da29a456bc64ba97ac205daf4ff2 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 11 Nov 2026 04:23:39 +0000 Subject: [PATCH 07/23] eval: round 11 results (defects, cost, round 10 regraded) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/results/desk-2026-11-11/README.md | 110 +++++ eval/results/desk-2026-11-11/agents-11.txt | 36 ++ eval/results/desk-2026-11-11/cost.json | 445 ++++++++++++++++++ .../desk-2026-11-11/graded/foldkit-1.txt | 12 + .../desk-2026-11-11/graded/foldkit-2.txt | 8 + .../desk-2026-11-11/graded/foldkit-3.txt | 8 + .../desk-2026-11-11/graded/foldkit-4.txt | 27 ++ .../desk-2026-11-11/graded/foldkit-5.txt | 21 + .../desk-2026-11-11/graded/foldkit-6.txt | 20 + .../desk-2026-11-11/graded/foldkit-7.txt | 19 + .../desk-2026-11-11/graded/foldkit-8.txt | 17 + .../desk-2026-11-11/graded/foldkit-9.txt | 24 + .../desk-2026-11-11/graded/react-1.txt | 6 + .../desk-2026-11-11/graded/react-2.txt | 8 + .../desk-2026-11-11/graded/react-3.txt | 12 + .../desk-2026-11-11/graded/react-4.txt | 21 + .../desk-2026-11-11/graded/react-5.txt | 8 + .../desk-2026-11-11/graded/react-6.txt | 28 ++ .../desk-2026-11-11/graded/react-7.txt | 24 + .../desk-2026-11-11/graded/react-8.txt | 7 + .../desk-2026-11-11/graded/react-9.txt | 20 + eval/results/desk-2026-11-11/graded/rx-1.txt | 6 + eval/results/desk-2026-11-11/graded/rx-2.txt | 14 + eval/results/desk-2026-11-11/graded/rx-3.txt | 7 + eval/results/desk-2026-11-11/graded/rx-4.txt | 10 + eval/results/desk-2026-11-11/graded/rx-5.txt | 11 + eval/results/desk-2026-11-11/graded/rx-6.txt | 12 + eval/results/desk-2026-11-11/graded/rx-7.txt | 9 + eval/results/desk-2026-11-11/graded/rx-8.txt | 18 + eval/results/desk-2026-11-11/graded/rx-9.txt | 8 + .../desk-2026-11-11/graded/vanilla-1.txt | 10 + .../desk-2026-11-11/graded/vanilla-2.txt | 10 + .../desk-2026-11-11/graded/vanilla-3.txt | 10 + .../desk-2026-11-11/graded/vanilla-4.txt | 21 + .../desk-2026-11-11/graded/vanilla-5.txt | 26 + .../desk-2026-11-11/graded/vanilla-6.txt | 19 + .../desk-2026-11-11/graded/vanilla-7.txt | 15 + .../desk-2026-11-11/graded/vanilla-8.txt | 33 ++ .../desk-2026-11-11/graded/vanilla-9.txt | 29 ++ .../r10-regraded/foldkit-1.txt | 11 + .../r10-regraded/foldkit-2.txt | 13 + .../r10-regraded/foldkit-3.txt | 12 + .../r10-regraded/foldkit-4.txt | 24 + .../r10-regraded/foldkit-5.txt | 23 + .../r10-regraded/foldkit-6.txt | 25 + .../r10-regraded/foldkit-7.txt | 21 + .../r10-regraded/foldkit-8.txt | 23 + .../r10-regraded/foldkit-9.txt | 22 + .../desk-2026-11-11/r10-regraded/react-1.txt | 10 + .../desk-2026-11-11/r10-regraded/react-2.txt | 12 + .../desk-2026-11-11/r10-regraded/react-3.txt | 8 + .../desk-2026-11-11/r10-regraded/react-4.txt | 23 + .../desk-2026-11-11/r10-regraded/react-5.txt | 13 + .../desk-2026-11-11/r10-regraded/react-6.txt | 32 ++ .../desk-2026-11-11/r10-regraded/react-7.txt | 28 ++ .../desk-2026-11-11/r10-regraded/react-8.txt | 26 + .../desk-2026-11-11/r10-regraded/react-9.txt | 27 ++ .../desk-2026-11-11/r10-regraded/rx-1.txt | 7 + .../desk-2026-11-11/r10-regraded/rx-2.txt | 15 + .../desk-2026-11-11/r10-regraded/rx-3.txt | 8 + .../desk-2026-11-11/r10-regraded/rx-4.txt | 9 + .../desk-2026-11-11/r10-regraded/rx-5.txt | 9 + .../desk-2026-11-11/r10-regraded/rx-6.txt | 11 + .../desk-2026-11-11/r10-regraded/rx-7.txt | 23 + .../desk-2026-11-11/r10-regraded/rx-8.txt | 9 + .../desk-2026-11-11/r10-regraded/rx-9.txt | 18 + .../r10-regraded/vanilla-1.txt | 9 + .../r10-regraded/vanilla-2.txt | 10 + .../r10-regraded/vanilla-3.txt | 8 + .../r10-regraded/vanilla-4.txt | 22 + .../r10-regraded/vanilla-5.txt | 16 + .../r10-regraded/vanilla-6.txt | 15 + .../r10-regraded/vanilla-7.txt | 17 + .../r10-regraded/vanilla-8.txt | 18 + .../r10-regraded/vanilla-9.txt | 15 + eval/results/desk-2026-11-11/stats.py | 36 ++ eval/results/desk-2026-11-11/stats.txt | 13 + eval/results/desk-2026-11-11/summary.txt | 36 ++ eval/results/desk-2026-11-11/usd.json | 1 + 79 files changed, 1827 insertions(+) create mode 100644 eval/results/desk-2026-11-11/README.md create mode 100644 eval/results/desk-2026-11-11/agents-11.txt create mode 100644 eval/results/desk-2026-11-11/cost.json create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-1.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-2.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-3.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-4.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-5.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-6.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-7.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-8.txt create mode 100644 eval/results/desk-2026-11-11/graded/foldkit-9.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-1.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-2.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-3.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-4.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-5.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-6.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-7.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-8.txt create mode 100644 eval/results/desk-2026-11-11/graded/react-9.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-1.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-2.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-3.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-4.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-5.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-6.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-7.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-8.txt create mode 100644 eval/results/desk-2026-11-11/graded/rx-9.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-1.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-2.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-3.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-4.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-5.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-6.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-7.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-8.txt create mode 100644 eval/results/desk-2026-11-11/graded/vanilla-9.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-1.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-2.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-3.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-4.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-5.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-6.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-7.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-8.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/foldkit-9.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-1.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-2.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-3.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-4.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-5.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-6.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-7.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-8.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/react-9.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-1.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-2.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-3.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-4.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-5.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-6.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-7.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-8.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/rx-9.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-1.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-2.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-3.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-4.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-5.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-6.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-7.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-8.txt create mode 100644 eval/results/desk-2026-11-11/r10-regraded/vanilla-9.txt create mode 100644 eval/results/desk-2026-11-11/stats.py create mode 100644 eval/results/desk-2026-11-11/stats.txt create mode 100644 eval/results/desk-2026-11-11/summary.txt create mode 100644 eval/results/desk-2026-11-11/usd.json diff --git a/eval/results/desk-2026-11-11/README.md b/eval/results/desk-2026-11-11/README.md new file mode 100644 index 00000000..0b99c06a --- /dev/null +++ b/eval/results/desk-2026-11-11/README.md @@ -0,0 +1,110 @@ +# Round 11 results: defects and cost by model tier, four approaches (2026-11-11) + +**Pre-registration:** `eval/PREREG-desk-11.md`, frozen at b8b9e0b. + +**Runs:** 36, which is 4 arms × (3 frontier + 6 small). Frontier runs used the current Opus model, small runs the current Haiku model; the model is read from each transcript. + +**Toolchain:** Solid `next` 6751472f, built from source; rxc from b8b9e0b. + +**Grading:** `harness/grade.sh` (hidden 77, overlap 38, 20 fuzz seeds). Defects are failed hidden tests, plus failed overlap tests, plus fuzz seeds that found a hole. Only defects are counted. No fuzz hole was traced to a legal hold, so nothing is excluded. + +## Defects per app + +| arm | frontier mean | small mean | small per desk | +|---|---|---|---| +| .rx (inline docs) | 2.0 | **3.8** | 3, 3, 4, 2, 9, 2 | +| React | 2.0 | 9.7 | 13, 2, 17, 14, 1, 11 | +| plain TypeScript | 3.0 | 14.5 | 12, 16, 11, 9, 20, 19 | +| Foldkit (inline docs) | 2.7 | 13.2 | 17, 13, 13, 12, 10, 14 | + +Frontier per desk: .rx 0, 5, 1; React 0, 2, 4; plain 3, 3, 3; Foldkit 4, 2, 2. + +## Tests + +All tests are on the small tier unless named otherwise; full output is in `stats.txt`. + +- **H11a** (.rx < React): permutation p = 0.047, Holm 0.047. **Supported.** +- **H11b** (.rx < plain): p = 0.0021, Holm 0.0041. **Supported.** +- **H11b** (.rx < Foldkit): p = 0.0010, Holm 0.0031. **Supported.** +- **H11c** (frontier .rx no worse than frontier React): difference 0.0, 90% bootstrap interval [−2.3, 2.3]. The upper end is not under 1, so it is **not shown**. Both arms average 2.0, but three runs each are too few. +- **H10c again** (small .rx no worse than frontier React): +1.8, interval [−0.3, 4.2]. **Not shown.** + +## Correct apps and cost + +Two apps have no defect, the first in any round: frontier `.rx-1` and frontier `React-1`. + +**Cost per correct app:** frontier .rx $6.14 and frontier React $4.15 (each arm's three runs over its one correct app). Every other cell has no correct app, so its cost per correct app is undefined. + +Cost per run is billed tokens (`cost.json`) at round 10's rates (`usd.json`): + +| arm | frontier $/run (round 10) | small $/run (round 10) | frontier calls | small calls | +|---|---|---|---|---| +| .rx | **2.05** (2.90) | 0.092 (0.085) | 21.3 | 15.5 | +| React | 1.38 (1.35) | 0.069 (0.066) | 12.7 | 11.8 | +| plain | 1.27 (1.24) | 0.077 (0.082) | 12.3 | 10.0 | +| Foldkit | 3.63 (3.56) | 0.175 (0.155) | 37.3 | 38.5 | + +Frontier .rx cost about 30% less than in round 10. The other arms cost about the same. + +## Change from round 10, same tests + +Round 10's desks were regraded on this round's toolchain and tests (`r10-regraded/`). As pre-registered, the rename-reload test that round 10's brief caused is dropped from those numbers. + +| arm, tier | round 10 regraded | round 11 | +|---|---|---| +| .rx frontier | 2.7 | 2.0 | +| .rx small | 4.7 | 3.8 | +| React frontier | 3.0 | 2.0 | +| React small | 13.7 | 9.7 | +| plain frontier | 2.7 | 3.0 | +| plain small | 8.8 | 14.5 | +| Foldkit frontier | 4.0 | 2.7 | +| Foldkit small | 13.5 | 13.2 | + +Round 10 as first published had .rx at 6.7 in both tiers. The Solid fixes since then account for most of the drop to 2.7 and 4.7. + +## Defects by kind + +- **.rx:** no failure repeats in more than two desks. The overlap tests that fail in most desks of the other arms ("labels / comments that load while an add is out keep it") fail in only one .rx desk (rx-2). +- **React, plain, Foldkit (small):** most defects are overlap. Each fails 7 to 11 of 38 overlap tests on average: loads that land while a change is out drop the change, and late saves overwrite newer ones. + +### .rx fuzz holes, traced + +- **rx-2, seeds 1, 3, 18: a Solid bug.** A keyed projection whose function returns another keyed projection loses a row when the question switches while a guess is out. In a browser this shows as a duplicated row. The plain Solid repro is filed as solidjs/solid#4762. It counts against .rx, as pre-registered. +- **rx-5, rx-6, rx-7, the "row assignee" holes: agent mistakes.** The assign guess is written only when the row is in the list on screen (`if (i >= 0)`), so a row hidden by a close guess never gets it. +- **rx-5 and rx-8, the "detail title" holes: agent mistakes.** rx-5 seed 15 was replayed; rx-8 has the same code shape and was not replayed. The rename guess is written to a source derived from the detail before the detail has loaded. Solid drops a guess that was never shown (D-34), so the old title stays on screen while the rename is out. +- **Not traced:** rx-4 seed 15 (detail title, with the guesses kept in their own list). + +## Lines of source per app + +Non-blank lines, without the given `api` and `flights` files. + +| arm | frontier | small | +|---|---|---| +| .rx | 414 | 373 | +| React | 784 | 744 | +| plain | 873 | 967 | +| Foldkit | 2276 | 2340 | + +## Deviations + +- foldkit-6 and foldkit-7 ended without a final report and were graded as they stood. +- rx-5 used 16 check/test runs, one over the cap of 15. +- react-5 listed the file names in `eval/` (it read no contents) and said so. +- Three agents (rx-1, rx-2, react-4) added temporary visible tests and then restored the file. +- None of these affects grading. + +## Reading + +- **Small tier:** the round 10 result replicates. Small .rx writes fewer than half of React's defects and about a quarter of plain's and Foldkit's. +- **Frontier tier:** all four arms are now close, between 2 and 3 defects. .rx is level with React, but three runs per arm cannot show "no worse". +- **Cost:** frontier .rx is still dearer than React and plain ($2.05 against about $1.30) but cheaper than Foldkit. With 3 of its 6 frontier defects from one Solid bug, the frontier .rx count is partly Solid's still. + +## Files + +- `summary.txt`: per desk. +- `graded/`: `grade.sh` output. +- `r10-regraded/`: round 10's desks on this toolchain. +- `stats.py`, `stats.txt`. +- `cost.json`, `usd.json`. +- `agents-11.txt`: desk to transcript. diff --git a/eval/results/desk-2026-11-11/agents-11.txt b/eval/results/desk-2026-11-11/agents-11.txt new file mode 100644 index 00000000..f6ba6e2c --- /dev/null +++ b/eval/results/desk-2026-11-11/agents-11.txt @@ -0,0 +1,36 @@ +desk11-rx-1 a84e1656cb0ba6df6 +desk11-react-1 a2f24068924bc4fa4 +desk11-vanilla-1 afe87582e2a06182e +desk11-foldkit-1 adefd18cee0357245 +desk11-rx-4 aa00b48b4d63e1f4f +desk11-react-4 a9ad76d0fefab4fb9 +desk11-vanilla-4 a44863179ca8f66d2 +desk11-foldkit-4 ae41cd1296a281f94 +desk11-rx-2 a22838ba99d99ae05 +desk11-react-2 ae4cfba0be84e5fde +desk11-vanilla-2 abbd5e1185c32e849 +desk11-foldkit-2 a86728ef1f49707e3 +desk11-rx-5 aca8962526f6412eb +desk11-react-5 accf3998eda68b899 +desk11-vanilla-5 acdf73c2cc217936e +desk11-foldkit-5 a267361eee853d3d5 +desk11-rx-6 aacc6363979cdbc04 +desk11-react-6 a7113a69ad5ba8dc5 +desk11-vanilla-6 a3aa74d9de8c92134 +desk11-foldkit-6 a6f3d7ae6bddace40 +desk11-rx-3 a981489adbd708190 +desk11-react-3 afd3f0953f7a4f889 +desk11-vanilla-3 a3a9a9c11a9591692 +desk11-foldkit-3 abf32ed818e83da60 +desk11-rx-7 ac6e74d314c0c60b7 +desk11-react-7 a354f1917af348048 +desk11-vanilla-7 ad6fbfda274aef3d6 +desk11-foldkit-7 afe411d8be72215e2 +desk11-rx-8 ad97b6d0dc3a855a3 +desk11-react-8 adef2bade38df0577 +desk11-vanilla-8 ae61d4a5ff5667c62 +desk11-foldkit-8 a68e6145c8637d77a +desk11-rx-9 aa3711f9a5483d068 +desk11-react-9 adf12270373046254 +desk11-vanilla-9 ac7ec2aeac924ce56 +desk11-foldkit-9 aafaabb4bcd01167f diff --git a/eval/results/desk-2026-11-11/cost.json b/eval/results/desk-2026-11-11/cost.json new file mode 100644 index 00000000..8ddc8cc5 --- /dev/null +++ b/eval/results/desk-2026-11-11/cost.json @@ -0,0 +1,445 @@ +{ + "weights": { + "input": 1, + "cacheWrite": 1.25, + "cacheRead": 0.1, + "output": 5 + }, + "runs": [ + { + "run": "desk11-rx-1", + "task": "desk11", + "arm": "rx", + "n": 1, + "input": 48, + "cacheWrite": 114816, + "cacheRead": 2054728, + "output": 37627, + "calls": 24, + "billed": 537175.8 + }, + { + "run": "desk11-react-1", + "task": "desk11", + "arm": "react", + "n": 1, + "input": 20, + "cacheWrite": 88727, + "cacheRead": 586612, + "output": 36111, + "calls": 10, + "billed": 350144.95 + }, + { + "run": "desk11-vanilla-1", + "task": "desk11", + "arm": "vanilla", + "n": 1, + "input": 18, + "cacheWrite": 83028, + "cacheRead": 482086, + "output": 31010, + "calls": 9, + "billed": 307061.6 + }, + { + "run": "desk11-foldkit-1", + "task": "desk11", + "arm": "foldkit", + "n": 1, + "input": 78, + "cacheWrite": 157838, + "cacheRead": 4085803, + "output": 57285, + "calls": 39, + "billed": 892380.8 + }, + { + "run": "desk11-rx-4", + "task": "desk11", + "arm": "rx", + "n": 4, + "input": 42, + "cacheWrite": 242602, + "cacheRead": 2742416, + "output": 131811, + "calls": 21, + "billed": 1236591.1 + }, + { + "run": "desk11-react-4", + "task": "desk11", + "arm": "react", + "n": 4, + "input": 30, + "cacheWrite": 181839, + "cacheRead": 1622394, + "output": 99373, + "calls": 15, + "billed": 886433.15 + }, + { + "run": "desk11-vanilla-4", + "task": "desk11", + "arm": "vanilla", + "n": 4, + "input": 16, + "cacheWrite": 168758, + "cacheRead": 564851, + "output": 93255, + "calls": 8, + "billed": 733723.6 + }, + { + "run": "desk11-foldkit-4", + "task": "desk11", + "arm": "foldkit", + "n": 4, + "input": 66, + "cacheWrite": 279117, + "cacheRead": 5373929, + "output": 188260, + "calls": 33, + "billed": 1827655.15 + }, + { + "run": "desk11-rx-2", + "task": "desk11", + "arm": "rx", + "n": 2, + "input": 52, + "cacheWrite": 127955, + "cacheRead": 2452851, + "output": 46742, + "calls": 26, + "billed": 638990.85 + }, + { + "run": "desk11-react-2", + "task": "desk11", + "arm": "react", + "n": 2, + "input": 24, + "cacheWrite": 80906, + "cacheRead": 689435, + "output": 28687, + "calls": 12, + "billed": 313535 + }, + { + "run": "desk11-vanilla-2", + "task": "desk11", + "arm": "vanilla", + "n": 2, + "input": 32, + "cacheWrite": 85314, + "cacheRead": 1051601, + "output": 31156, + "calls": 16, + "billed": 367614.6 + }, + { + "run": "desk11-foldkit-2", + "task": "desk11", + "arm": "foldkit", + "n": 2, + "input": 78, + "cacheWrite": 173428, + "cacheRead": 4429660, + "output": 68564, + "calls": 39, + "billed": 1002649 + }, + { + "run": "desk11-rx-5", + "task": "desk11", + "arm": "rx", + "n": 5, + "input": 30, + "cacheWrite": 172922, + "cacheRead": 1624483, + "output": 107574, + "calls": 15, + "billed": 916500.8 + }, + { + "run": "desk11-react-5", + "task": "desk11", + "arm": "react", + "n": 5, + "input": 20, + "cacheWrite": 110131, + "cacheRead": 658688, + "output": 58065, + "calls": 10, + "billed": 493877.55 + }, + { + "run": "desk11-vanilla-5", + "task": "desk11", + "arm": "vanilla", + "n": 5, + "input": 18, + "cacheWrite": 137313, + "cacheRead": 705916, + "output": 85399, + "calls": 9, + "billed": 669245.85 + }, + { + "run": "desk11-foldkit-5", + "task": "desk11", + "arm": "foldkit", + "n": 5, + "input": 64, + "cacheWrite": 255010, + "cacheRead": 4969325, + "output": 168981, + "calls": 32, + "billed": 1660664 + }, + { + "run": "desk11-rx-6", + "task": "desk11", + "arm": "rx", + "n": 6, + "input": 28, + "cacheWrite": 154278, + "cacheRead": 1341121, + "output": 88846, + "calls": 14, + "billed": 771217.6 + }, + { + "run": "desk11-react-6", + "task": "desk11", + "arm": "react", + "n": 6, + "input": 26, + "cacheWrite": 137985, + "cacheRead": 1077077, + "output": 81181, + "calls": 13, + "billed": 686119.95 + }, + { + "run": "desk11-vanilla-6", + "task": "desk11", + "arm": "vanilla", + "n": 6, + "input": 26, + "cacheWrite": 192119, + "cacheRead": 1406897, + "output": 112557, + "calls": 13, + "billed": 943649.45 + }, + { + "run": "desk11-foldkit-6", + "task": "desk11", + "arm": "foldkit", + "n": 6, + "input": 58, + "cacheWrite": 231135, + "cacheRead": 3634198, + "output": 142901, + "calls": 29, + "billed": 1366901.55 + }, + { + "run": "desk11-rx-3", + "task": "desk11", + "arm": "rx", + "n": 3, + "input": 28, + "cacheWrite": 70566, + "cacheRead": 1083936, + "output": 32630, + "calls": 14, + "billed": 359779.1 + }, + { + "run": "desk11-react-3", + "task": "desk11", + "arm": "react", + "n": 3, + "input": 32, + "cacheWrite": 62105, + "cacheRead": 1117911, + "output": 36921, + "calls": 16, + "billed": 374059.35 + }, + { + "run": "desk11-vanilla-3", + "task": "desk11", + "arm": "vanilla", + "n": 3, + "input": 24, + "cacheWrite": 52234, + "cacheRead": 727341, + "output": 27242, + "calls": 12, + "billed": 274260.6 + }, + { + "run": "desk11-foldkit-3", + "task": "desk11", + "arm": "foldkit", + "n": 3, + "input": 68, + "cacheWrite": 130992, + "cacheRead": 3590620, + "output": 61354, + "calls": 34, + "billed": 829640 + }, + { + "run": "desk11-rx-7", + "task": "desk11", + "arm": "rx", + "n": 7, + "input": 24, + "cacheWrite": 169319, + "cacheRead": 1150004, + "output": 94694, + "calls": 12, + "billed": 800143.15 + }, + { + "run": "desk11-react-7", + "task": "desk11", + "arm": "react", + "n": 7, + "input": 16, + "cacheWrite": 99773, + "cacheRead": 501227, + "output": 78701, + "calls": 8, + "billed": 568359.95 + }, + { + "run": "desk11-vanilla-7", + "task": "desk11", + "arm": "vanilla", + "n": 7, + "input": 14, + "cacheWrite": 133174, + "cacheRead": 450749, + "output": 82397, + "calls": 7, + "billed": 623541.4 + }, + { + "run": "desk11-foldkit-7", + "task": "desk11", + "arm": "foldkit", + "n": 7, + "input": 82, + "cacheWrite": 212489, + "cacheRead": 6393572, + "output": 160068, + "calls": 41, + "billed": 1705390.4500000002 + }, + { + "run": "desk11-rx-8", + "task": "desk11", + "arm": "rx", + "n": 8, + "input": 22, + "cacheWrite": 118421, + "cacheRead": 913795, + "output": 80266, + "calls": 11, + "billed": 640757.75 + }, + { + "run": "desk11-react-8", + "task": "desk11", + "arm": "react", + "n": 8, + "input": 32, + "cacheWrite": 171479, + "cacheRead": 1614783, + "output": 117355, + "calls": 16, + "billed": 962634.05 + }, + { + "run": "desk11-vanilla-8", + "task": "desk11", + "arm": "vanilla", + "n": 8, + "input": 32, + "cacheWrite": 171439, + "cacheRead": 1876839, + "output": 118254, + "calls": 16, + "billed": 993284.65 + }, + { + "run": "desk11-foldkit-8", + "task": "desk11", + "arm": "foldkit", + "n": 8, + "input": 118, + "cacheWrite": 277940, + "cacheRead": 10575743, + "output": 193296, + "calls": 59, + "billed": 2371597.3 + }, + { + "run": "desk11-rx-9", + "task": "desk11", + "arm": "rx", + "n": 9, + "input": 40, + "cacheWrite": 209701, + "cacheRead": 2508495, + "output": 131593, + "calls": 20, + "billed": 1170980.75 + }, + { + "run": "desk11-react-9", + "task": "desk11", + "arm": "react", + "n": 9, + "input": 18, + "cacheWrite": 98703, + "cacheRead": 537861, + "output": 78219, + "calls": 9, + "billed": 568277.85 + }, + { + "run": "desk11-vanilla-9", + "task": "desk11", + "arm": "vanilla", + "n": 9, + "input": 14, + "cacheWrite": 142091, + "cacheRead": 468383, + "output": 92217, + "calls": 7, + "billed": 685551.05 + }, + { + "run": "desk11-foldkit-9", + "task": "desk11", + "arm": "foldkit", + "n": 9, + "input": 74, + "cacheWrite": 272538, + "cacheRead": 4876740, + "output": 145200, + "calls": 37, + "billed": 1554420.5 + } + ], + "diff": -207632.78888888902, + "ratio": 1.3591267418422939, + "p": 0.9435805641943581 +} diff --git a/eval/results/desk-2026-11-11/graded/foldkit-1.txt b/eval/results/desk-2026-11-11/graded/foldkit-1.txt new file mode 100644 index 00000000..a6ca8b5a --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-1.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-foldkit-1: 2/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/foldkit-2.txt b/eval/results/desk-2026-11-11/graded/foldkit-2.txt new file mode 100644 index 00000000..2af4cb66 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-foldkit-2: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/foldkit-3.txt b/eval/results/desk-2026-11-11/graded/foldkit-3.txt new file mode 100644 index 00000000..f357c830 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-foldkit-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/foldkit-4.txt b/eval/results/desk-2026-11-11/graded/foldkit-4.txt new file mode 100644 index 00000000..b0d4db20 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-4.txt @@ -0,0 +1,27 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk11-foldkit-4: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 10 step 25: row assignee: #2 shows bob, server guest + focus ; focus ; filter closed ; fail fetchIssues:open ; focus ; land fetchIssues:closed ; row-assign 0 ; land assign:2,guest ; select #2 ; close ; rename Bravo ; fail fetchLabels:2 ; select #2 ; fail fetchIssue:2 ; land closeIssue:2 ; row-assign 0 ; land fetchComments:2 ; land assign:2,guest ; assign ; user bob ; land assign:2,guest ; row-assign 0 ; rename Alpha ; assign ; fail assign:2,bob + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/graded/foldkit-5.txt b/eval/results/desk-2026-11-11/graded/foldkit-5.txt new file mode 100644 index 00000000..06c2b480 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-5.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-foldkit-5: 2/20 seeds found a hole + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 18 step 38: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; (move failed: locator.click: Timeout 2000ms exceeded.) ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; diff --git a/eval/results/desk-2026-11-11/graded/foldkit-6.txt b/eval/results/desk-2026-11-11/graded/foldkit-6.txt new file mode 100644 index 00000000..bfa425f1 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-6.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save failing after a later one succeeded leaves the later one +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk11-foldkit-6: 1/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/graded/foldkit-7.txt b/eval/results/desk-2026-11-11/graded/foldkit-7.txt new file mode 100644 index 00000000..a0c4a90c --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-7.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-foldkit-7: 1/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/graded/foldkit-8.txt b/eval/results/desk-2026-11-11/graded/foldkit-8.txt new file mode 100644 index 00000000..cc802ae3 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-8.txt @@ -0,0 +1,17 @@ +== hidden +✗ base: a declined close brings the row back in place and shows the message +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +30/38 passed +== fuzz +desk11-foldkit-8: 1/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [bug,ui], server [ui,bug] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail remov diff --git a/eval/results/desk-2026-11-11/graded/foldkit-9.txt b/eval/results/desk-2026-11-11/graded/foldkit-9.txt new file mode 100644 index 00000000..7f3bbea7 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/foldkit-9.txt @@ -0,0 +1,24 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk11-foldkit-9: 4/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [bug,ui], server [ui,bug] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail remov + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 37: row assignee: #3 shows guest, server null + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/graded/react-1.txt b/eval/results/desk-2026-11-11/graded/react-1.txt new file mode 100644 index 00000000..dbd2d14a --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-1.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk11-react-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-2.txt b/eval/results/desk-2026-11-11/graded/react-2.txt new file mode 100644 index 00000000..839e7b30 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-react-2: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-3.txt b/eval/results/desk-2026-11-11/graded/react-3.txt new file mode 100644 index 00000000..51d4a922 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-3.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-react-3: 2/20 seeds found a hole + seed 4 step 36: at rest: comments: #3 shows [dana: seen it too | guest: me too | guest: me too], server [dana: seen it too | guest: me too] + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land closeIssue:1 ; select #3 ; row-close 1 ; rena + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/react-4.txt b/eval/results/desk-2026-11-11/graded/react-4.txt new file mode 100644 index 00000000..b574fd01 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-4.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-react-4: 2/20 seeds found a hole + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 39: at rest: comments: #2 shows [guest: fixed? | guest: me too | guest: me too], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk-2026-11-11/graded/react-5.txt b/eval/results/desk-2026-11-11/graded/react-5.txt new file mode 100644 index 00000000..28635c23 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-5.txt @@ -0,0 +1,8 @@ +== hidden +✗ base: an accepted post stays after switching issues and back +76/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk11-react-5: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-6.txt b/eval/results/desk-2026-11-11/graded/react-6.txt new file mode 100644 index 00000000..a39d1eaa --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-6.txt @@ -0,0 +1,28 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk11-react-6: 5/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 3 step 39: at rest: labels: #4 shows [docs,p1], server [docs] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; row-close 0 ; fail search:Typo in footer ; land fetchLabels:4 ; row-close 1 ; row-close 0 ; land closeIssue:4 ; close ; add-label p1 ; remove-label 0 ; remove-label 0 ; filter closed ; add-label p1 ; assign ; land addLabel:4,p1 ; fai + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/graded/react-7.txt b/eval/results/desk-2026-11-11/graded/react-7.txt new file mode 100644 index 00000000..2f357b3c --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-7.txt @@ -0,0 +1,24 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk11-react-7: 4/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 39: at rest: comments: #2 shows [guest: fixed? | guest: me too | guest: me too], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk-2026-11-11/graded/react-8.txt b/eval/results/desk-2026-11-11/graded/react-8.txt new file mode 100644 index 00000000..3f4b8692 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-8.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk11-react-8: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/react-9.txt b/eval/results/desk-2026-11-11/graded/react-9.txt new file mode 100644 index 00000000..5b99cbc7 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/react-9.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk11-react-9: 3/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 39: at rest: labels: #2 shows [old], server [old,bug] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk-2026-11-11/graded/rx-1.txt b/eval/results/desk-2026-11-11/graded/rx-1.txt new file mode 100644 index 00000000..21eab8c1 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-1.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/rx-2.txt b/eval/results/desk-2026-11-11/graded/rx-2.txt new file mode 100644 index 00000000..552929e5 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-2.txt @@ -0,0 +1,14 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-rx-2: 3/20 seeds found a hole + seed 1 step 24: open row in closed list: #4 + user bob ; focus ; land fetchIssues:open ; row-assign 2 ; row-close 0 ; row-assign 0 ; select #3 ; rename Alpha ; land assign:3,bob ; rename Alpha ; row-assign 1 ; land fetchLabels:3 ; focus ; land fetchComments:3 ; fail closeIssue:1 ; land renameIssue:3,Alpha ; fail assign:4,bob ; post me too ; add-label ui ; land addComment:3,bob,me too ; row-close 2 ; filter closed ; land fetchIssue:3 ; lan + seed 3 step 37: open row in closed list: #1 + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 18 step 16: open row in closed list: #3 + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; filter closed ; land fetchIssues:closed ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/graded/rx-3.txt b/eval/results/desk-2026-11-11/graded/rx-3.txt new file mode 100644 index 00000000..9be7d5bc --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-3.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk11-rx-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/rx-4.txt b/eval/results/desk-2026-11-11/graded/rx-4.txt new file mode 100644 index 00000000..abb997d1 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-4.txt @@ -0,0 +1,10 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +✗ base: an earlier rename declined after a later one was accepted keeps the later title +37/38 passed +== fuzz +desk11-rx-4: 1/20 seeds found a hole + seed 15 step 38: detail title: "retry#2 Bravolong goneassign to guestcloserenamesaving files", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk-2026-11-11/graded/rx-5.txt b/eval/results/desk-2026-11-11/graded/rx-5.txt new file mode 100644 index 00000000..c06c929a --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-5.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ a label added on one issue does not show on another +37/38 passed +== fuzz +desk11-rx-5: 2/20 seeds found a hole + seed 15 step 24: detail title: "retry#2 Old buglong gonesaving filescrash reportsassign to g", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 + seed 18 step 32: row assignee: #4 shows dana, server guest + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; filter closed ; land fetchIssues:closed ; land closeIssue:1 ; close ; land closeIssue:4 ; land fetchIssues:closed ; rename Alpha ; land fetchIssues:open ; land renameIssue:4, diff --git a/eval/results/desk-2026-11-11/graded/rx-6.txt b/eval/results/desk-2026-11-11/graded/rx-6.txt new file mode 100644 index 00000000..92c0d054 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-6.txt @@ -0,0 +1,12 @@ +== hidden +✗ base: selecting after not found shows loading, not the old message +✗ related help landing out of order shows the selected issue's +75/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-6: 2/20 seeds found a hole + seed 11 step 34: row assignee: #4 shows dana, server ann + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; close ; land fetchLabels:4 ; fail fetchComments:4 ; row-assign 0 ; close ; land fetchIssue: + seed 18 step 32: row assignee: #4 shows dana, server guest + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; filter closed ; land fetchIssues:closed ; land closeIssue:1 ; close ; land closeIssue:4 ; land fetchIssues:closed ; rename Alpha ; land fetchIssues:open ; land renameIssue:4, diff --git a/eval/results/desk-2026-11-11/graded/rx-7.txt b/eval/results/desk-2026-11-11/graded/rx-7.txt new file mode 100644 index 00000000..914efbb8 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-7.txt @@ -0,0 +1,9 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-7: 1/20 seeds found a hole + seed 11 step 34: row assignee: #4 shows dana, server ann + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; close ; land fetchLabels:4 ; fail fetchComments:4 ; row-assign 0 ; close ; land fetchIssue: diff --git a/eval/results/desk-2026-11-11/graded/rx-8.txt b/eval/results/desk-2026-11-11/graded/rx-8.txt new file mode 100644 index 00000000..23b52ade --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-8.txt @@ -0,0 +1,18 @@ +== hidden +✗ base: selecting after not found shows loading, not the old message +✗ base: help: selecting after a failure shows loading…, not the old failure +75/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ a label added on one issue does not show on another +✗ rows: a declined row close leaves a rename out on another issue +34/38 passed +== fuzz +desk11-rx-8: 3/20 seeds found a hole + seed 8 step 46: detail title: "assign to guest close retryrename#3 Crash on savestack trace", server "Alpha" + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; select #4 ; land closeIssue:3 ; select #1 ; land fetchIssues:closed ; post me too ; rename Bravo ; land fetchComments:1 ; rename Delta ; close ; row-assign 1 ; land fetchLabels:4 ; land assign:3,guest ; select #2 ; focus ; fail fetchComments:2 ; close ; fail addComment:1,guest,me too ; land closeIssue:2 ; land fetc + seed 11 step 44: detail title: "assign to ann close retryrename#3 Crash on savestack trace a", server "Delta" + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; row-assign 0 ; row-close 0 ; select #3 ; focus ; close ; land fetchIssue:4 ; post me too ; + seed 15 step 24: detail title: "assign to guest close retryrename#2 Old buglong gonesaving f", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk-2026-11-11/graded/rx-9.txt b/eval/results/desk-2026-11-11/graded/rx-9.txt new file mode 100644 index 00000000..69806e36 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/rx-9.txt @@ -0,0 +1,8 @@ +== hidden +✗ base: help: selecting after a failure shows loading…, not the old failure +✗ related help landing out of order shows the selected issue's +75/77 passed +== overlap +38/38 passed +== fuzz +desk11-rx-9: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/vanilla-1.txt b/eval/results/desk-2026-11-11/graded/vanilla-1.txt new file mode 100644 index 00000000..4228f3ca --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-vanilla-1: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-2.txt b/eval/results/desk-2026-11-11/graded/vanilla-2.txt new file mode 100644 index 00000000..24331212 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-2.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-vanilla-2: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-3.txt b/eval/results/desk-2026-11-11/graded/vanilla-3.txt new file mode 100644 index 00000000..9460fce0 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-3.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk11-vanilla-3: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-4.txt b/eval/results/desk-2026-11-11/graded/vanilla-4.txt new file mode 100644 index 00000000..20214ecd --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-4.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk11-vanilla-4: 3/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/graded/vanilla-5.txt b/eval/results/desk-2026-11-11/graded/vanilla-5.txt new file mode 100644 index 00000000..76057d93 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-5.txt @@ -0,0 +1,26 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk11-vanilla-5: 4/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/graded/vanilla-6.txt b/eval/results/desk-2026-11-11/graded/vanilla-6.txt new file mode 100644 index 00000000..f79ff1f4 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-6.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk11-vanilla-6: 2/20 seeds found a hole + seed 2 step 33: detail title: "#2 Old buglong goneretryassign to guestrenamecloseassign err", server "Bravo" + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land fetchLabels:2 ; rename Alpha ; rename + seed 15 step 36: two views: row "#3 Crash on save", detail "#3 Deltastack trace attachedretryassign " + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; diff --git a/eval/results/desk-2026-11-11/graded/vanilla-7.txt b/eval/results/desk-2026-11-11/graded/vanilla-7.txt new file mode 100644 index 00000000..2f8ab878 --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-7.txt @@ -0,0 +1,15 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk11-vanilla-7: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/graded/vanilla-8.txt b/eval/results/desk-2026-11-11/graded/vanilla-8.txt new file mode 100644 index 00000000..37cf050b --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-8.txt @@ -0,0 +1,33 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ two closes at once: a declined one leaves the other off the count +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk11-vanilla-8: 7/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 7 step 6: count: tab Issues (1), 2 open rows shown + land fetchIssues:open ; row-close 1 ; row-assign 0 ; row-close 0 ; land closeIssue:1 ; fail closeIssue:3 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 14 step 20: count: tab Issues (0), 1 open rows shown + filter closed ; filter open ; land fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; select #1 ; land fetchIssue:1 ; land fetchLabels:1 ; close ; row-close 1 ; fail fetchComments:1 ; rename Alpha ; rename Alpha ; row-assign 0 ; row-close 0 ; close ; land closeIssue:1 ; rename Bravo ; fail assign:3,guest ; fail closeIssue:4 + seed 15 step 36: count: tab Issues (0), 1 open rows shown + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 10: count: tab Issues (0), 1 open rows shown + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 diff --git a/eval/results/desk-2026-11-11/graded/vanilla-9.txt b/eval/results/desk-2026-11-11/graded/vanilla-9.txt new file mode 100644 index 00000000..29d11bad --- /dev/null +++ b/eval/results/desk-2026-11-11/graded/vanilla-9.txt @@ -0,0 +1,29 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk11-vanilla-9: 4/20 seeds found a hole + seed 3 step 39: count: tab Issues (3), 2 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row- + seed 8 step 16: row title: #4 shows "Typo in footer", server "Bravo" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedretryassign to guest ren", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo + seed 17 step 19: dropped guess: #3 title: showed "Alpha" while it was in flight, now "Crash on save" + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; row-assign 0 ; add-label bug ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; land assign:1,guest ; row-close 0 ; row-close 1 ; row-assign 0 ; select #3 ; rename Bravo ; row-assign 0 ; land fetchLabels:3 ; rename Alpha ; assign ; f diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-1.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-1.txt new file mode 100644 index 00000000..8c8e4b52 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-1.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +33/38 passed +== fuzz +desk10-foldkit-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-2.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-2.txt new file mode 100644 index 00000000..e123e888 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-2.txt @@ -0,0 +1,13 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +35/38 passed +== fuzz +desk10-foldkit-2: 2/20 seeds found a hole + seed 2 step 37: at rest: comments: #2 shows [], server [guest: fixed? | guest: fixed?] + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-3.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-3.txt new file mode 100644 index 00000000..775eae3b --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-3.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-foldkit-3: 2/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-4.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-4.txt new file mode 100644 index 00000000..9bd2825f --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-4.txt @@ -0,0 +1,24 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk10-foldkit-4: 2/20 seeds found a hole + seed 2 step 36: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 15 step 32: detail title: "#3 Crash on savestack trace attachedassign to guestcloserena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-5.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-5.txt new file mode 100644 index 00000000..6bff2dcb --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-5.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk10-foldkit-5: 4/20 seeds found a hole + seed 2 step 35: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 12 step 36: two views: row "#4 Alpha", detail "#4 Bravofooter says 2019assign to guestr" + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Typo in footer ; land closeIssue:4 ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land renameIssue:4,Alpha ; fail assign:4,guest ; assign ; land closeIssue + seed 15 step 37: row assignee: #3 shows guest, server null + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-6.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-6.txt new file mode 100644 index 00000000..a1778a64 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-6.txt @@ -0,0 +1,25 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-foldkit-6: 4/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019assign to guestc" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedassign to guestcloserena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-7.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-7.txt new file mode 100644 index 00000000..baa65a90 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-7.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk10-foldkit-7: 1/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-8.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-8.txt new file mode 100644 index 00000000..4f5fd511 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-8.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +25/38 passed +== fuzz +desk10-foldkit-8: 2/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk-2026-11-11/r10-regraded/foldkit-9.txt b/eval/results/desk-2026-11-11/r10-regraded/foldkit-9.txt new file mode 100644 index 00000000..7f62c409 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/foldkit-9.txt @@ -0,0 +1,22 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk10-foldkit-9: 2/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 9 step 40: row assignee: #4 shows null, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-1.txt b/eval/results/desk-2026-11-11/r10-regraded/react-1.txt new file mode 100644 index 00000000..d66e2727 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-react-1: 1/20 seeds found a hole + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-2.txt b/eval/results/desk-2026-11-11/r10-regraded/react-2.txt new file mode 100644 index 00000000..846c1800 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-2.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-react-2: 2/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-3.txt b/eval/results/desk-2026-11-11/r10-regraded/react-3.txt new file mode 100644 index 00000000..ba0de1c2 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-react-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-4.txt b/eval/results/desk-2026-11-11/r10-regraded/react-4.txt new file mode 100644 index 00000000..8158bbf6 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-4.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-react-4: 3/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 12 step 39: row assignee: #1 shows guest, server null + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchComments:1 ; land fetchComments:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land fetchIssue:4 ; land fetchLabels:1 ; land fetchIssue:1 ; post me too ; close ; fail closeIssue:1 ; select #1 ; row-assign 0 ; assign ; land closeIssue:4 ; + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-5.txt b/eval/results/desk-2026-11-11/r10-regraded/react-5.txt new file mode 100644 index 00000000..12bed383 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-5.txt @@ -0,0 +1,13 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ an add and a remove at once: a declined add does not undo the remove +✗ two adds at once: a declined one does not take the other with it +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +31/38 passed +== fuzz +desk10-react-5: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-6.txt b/eval/results/desk-2026-11-11/r10-regraded/react-6.txt new file mode 100644 index 00000000..f2d25d13 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-6.txt @@ -0,0 +1,32 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk10-react-6: 8/20 seeds found a hole + seed 2 step 36: two views: row "#2 Bravo", detail "#2 Alphalong goneNew titlerenameassign t" + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchLabels:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchComments:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest ; focus ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; + seed 3 step 39: at rest: comments: #4 shows [bob: me too | bob: me too], server [bob: me too] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land fetchIssue:4 ; row-close 0 ; fail assign:4,bob ; land addLabel:4,p1 ; row-close 1 ; row-assign 0 ; land closeIssue:1 ; close ; row-assign 0 ; row-close 0 ; add-label ui ; filter closed ; add-label p1 ; assign ; land closeIssue:3 ; fail assign:3,bob + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 10 step 17: detail title: "#2 Old buglong goneNew titlerenameassign to guestclosefindin", server "Bravo" + focus ; focus ; filter closed ; fail fetchIssues:open ; focus ; land fetchIssues:closed ; row-assign 0 ; land assign:2,guest ; select #2 ; close ; rename Bravo ; fail fetchIssue:2 ; select #2 ; fail fetchLabels:2 ; land fetchIssue:2 ; row-assign 0 ; land renameIssue:2,Bravo + seed 15 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchIssue:3 ; close ; row-assign 0 ; row-close 0 ; land assign:1,guest ; row-assign 0 ; select #4 ; land fetchLabels:3 ; assign ; add-label ui ; land fetchLabels:4 ; land fetchComments:3 ; assign ; land addLabel:4,ui ; assign ; rename + seed 16 step 38: detail title: "#2 Old buglong goneNew titlerenameassign to guestassign erro", server "Bravo" + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchIssue:2 ; fail addComment:2,guest,ping ; select #2 ; rename Br + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta + seed 19 step 17: detail title: "#3 Crash on savestack trace attachedNew titlerenameassign to", server "Alpha" + land fetchIssues:open ; user guest ; row-assign 1 ; row-close 2 ; land assign:3,guest ; fail closeIssue:4 ; select #3 ; post fixed? ; land fetchLabels:3 ; rename Alpha ; rename Delta ; row-assign 0 ; land assign:1,guest ; land fetchIssue:3 ; row-assign 1 ; land addComment:3,guest,fixed? ; land renameIssue:3,Alpha diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-7.txt b/eval/results/desk-2026-11-11/r10-regraded/react-7.txt new file mode 100644 index 00000000..b724b7a1 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-7.txt @@ -0,0 +1,28 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk10-react-7: 5/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 39: two views: row "#3 Crash on save", detail "#3 Deltastack trace attachedassign to gu" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land closeIssue:4 ; land fetchIssu + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-8.txt b/eval/results/desk-2026-11-11/r10-regraded/react-8.txt new file mode 100644 index 00000000..e376a7f1 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-8.txt @@ -0,0 +1,26 @@ +== hidden +✗ rows: row errors and detail errors leave each other alone +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-react-8: 4/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 10 step 17: detail title: "#2 Old buglong goneassign to guestNew title RenamecloseComme", server "Bravo" + focus ; focus ; filter closed ; fail fetchIssues:open ; focus ; land fetchIssues:closed ; row-assign 0 ; land assign:2,guest ; select #2 ; close ; rename Bravo ; fail fetchIssue:2 ; select #2 ; fail fetchComments:2 ; land fetchIssue:2 ; row-assign 0 ; land renameIssue:2,Bravo + seed 15 step 39: at rest: comments: #4 shows [], server [guest: fixed?] + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchIssue:3 ; close ; row-assign 0 ; row-close 0 ; land assign:1,guest ; row-assign 0 ; select #4 ; land fetchComments:3 ; assign ; add-label ui ; land assign:4,guest ; land fetchLabels:3 ; assign ; land assign:4,guest ; assign ; rena diff --git a/eval/results/desk-2026-11-11/r10-regraded/react-9.txt b/eval/results/desk-2026-11-11/r10-regraded/react-9.txt new file mode 100644 index 00000000..0c5e52e0 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/react-9.txt @@ -0,0 +1,27 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk10-react-9: 5/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019assign to guesta" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 35: two views: row "#4 Alpha", detail "#4 Bravofooter says 2019assign to guestr" + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-1.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-1.txt new file mode 100644 index 00000000..926ea8de --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-1.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +37/38 passed +== fuzz +desk10-rx-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-2.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-2.txt new file mode 100644 index 00000000..19872761 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-2.txt @@ -0,0 +1,15 @@ +== hidden +✗ base: a failing list leaves the detail alone +76/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-rx-2: 3/20 seeds found a hole + seed 11 step 33: detail title: "#4 Typo in footerfooter says 2019retryfinding related helpas", server "Delta" + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; close ; land fetchIssue:4 ; fail closeIssue:4 ; row-assign 0 ; close ; land fetchComments:4 + seed 15 step 40: two views: row "#2 Bravo", detail "#2 Deltalong goneretrysaving filescrash " + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchIssue:2 ; rename Bravo ; select #2 ; assign ; land fetchComments:2 ; row-assign 0 ; select #2 ; land fetchLabels:2 ; assign ; remove-label 0 ; land assign:2,guest ; land search:Old bu + seed 18 step 36: detail title: "#4 Typo in footerfooter says 2019retrysaving filescrash repo", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-assign 0 ; row-close 1 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta ; user bob ; row-assign 0 ; row-close 1 ; rename Delta ; assign ; row-assign 0 ; land fetchLabels:4 ; rename Alpha ; close ; fail renameIssue:4,Alpha ; focus ; close ; focus ; r diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-3.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-3.txt new file mode 100644 index 00000000..272a3dab --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-rx-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-4.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-4.txt new file mode 100644 index 00000000..79a74023 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-4.txt @@ -0,0 +1,9 @@ +== hidden +✗ base: help: selecting after a failure shows loading…, not the old failure +76/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ a label added on one issue does not show on another +36/38 passed +== fuzz +desk10-rx-4: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-5.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-5.txt new file mode 100644 index 00000000..d6cbdf46 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-5.txt @@ -0,0 +1,9 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +37/38 passed +== fuzz +desk10-rx-5: 1/20 seeds found a hole + seed 8 step 34: open row in closed list: #4 + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; user bob ; land closeIssue:3 ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #1 ; land fetchComments:1 ; select #4 ; land fetchComments:4 ; row-close 1 ; row-assign 0 ; land fetchLabels:1 ; row-assign 0 ; user guest ; fail assign:1,guest ; row-close 0 ; user ann ; focus ; land assign:4,guest diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-6.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-6.txt new file mode 100644 index 00000000..8d3ca347 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-6.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a post on one issue does not show on another +✗ a label added on one issue does not show on another +35/38 passed +== fuzz +desk10-rx-6: 1/20 seeds found a hole + seed 19 step 39: detail title: "#1 Alphaon Safari onlysaving filescrash reportsassign to gue", server "Login fails" + land fetchIssues:open ; user guest ; row-assign 1 ; row-close 2 ; land assign:3,guest ; fail closeIssue:4 ; select #3 ; post fixed? ; land fetchIssue:3 ; rename Alpha ; rename Delta ; close ; land fetchLabels:3 ; select #1 ; filter closed ; close ; land search:Crash on save ; land renameIssue:3,Delta ; filter open ; rename Alpha ; row-close 1 ; select #1 ; assign ; fail renameIssue:3,Delta ; s diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-7.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-7.txt new file mode 100644 index 00000000..aa905117 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-7.txt @@ -0,0 +1,23 @@ +== hidden +✗ base: an accepted post stays after switching issues and back +76/77 passed +== overlap +✗ base: two issues assigned at once: declining one reverts only that row +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a declined close keeps an assignment in flight on another issue +✗ base: a declined assignment keeps a close in flight on another issue +✗ base: two closes at once: a declined one does not undo the other +✗ base: two posts at once: a declined one does not take the other with it +✗ two adds at once: a declined one does not take the other with it +✗ two closes at once: a declined one leaves the other off the count +30/38 passed +== fuzz +desk10-rx-7: 4/20 seeds found a hole + seed 2 step 36: at rest: comments: #2 shows [guest: fixed? | guest: fixed?], server [guest: fixed?] + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail search:Delta ; fail assign:2,guest ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail renameIssue:2,Delta ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; row-assign 0 ; filter open ; rename Alpha ; rename Bravo + seed 8 step 34: open row in closed list: #4 + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; user bob ; land closeIssue:3 ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #1 ; land fetchComments:1 ; select #4 ; land fetchComments:4 ; row-close 1 ; row-assign 0 ; land fetchLabels:1 ; row-assign 0 ; user guest ; fail assign:1,guest ; row-close 0 ; user ann ; focus ; land assign:4,guest + seed 13 step 32: detail title: "#1 Login failson Safari onlyassign to guestrenamecloseguest:", server "Bravo" + filter closed ; fail fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; filter open ; land assign:2,guest ; land fetchIssues:open ; select #3 ; row-close 1 ; land fetchLabels:3 ; fail fetchIssue:3 ; land fetchComments:3 ; fail closeIssue:3 ; row-assign 0 ; select #3 ; land assign:1,guest ; select #1 ; land fetchLabels:1 ; land fetchComments:1 ; land fetchIssue:1 ; add-label bug ; land + seed 15 step 27: closed row shown: #3 is closed on the server + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; row-assign 1 ; select #4 ; select #1 ; land fetchComments:1 ; row-close 2 ; land fetchLabels:1 ; land fetchLabels:4 ; land fetchIssue:1 ; land fetchComments:3 ; rename Alpha ; row-assign 0 ; land search:Alpha ; assign ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-8.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-8.txt new file mode 100644 index 00000000..4cd277b9 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-8.txt @@ -0,0 +1,9 @@ +== hidden +77/77 passed +== overlap +✗ base: a detail that loads while a rename is out keeps the new title +37/38 passed +== fuzz +desk10-rx-8: 1/20 seeds found a hole + seed 13 step 31: page error: declined + filter closed ; fail fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; filter open ; land assign:2,guest ; land fetchIssues:open ; select #3 ; row-close 1 ; land fetchIssue:3 ; fail fetchLabels:3 ; land fetchComments:3 ; land search:Crash on save ; land closeIssue:3 ; fail fetchIssues:open ; assign ; assign ; land assign:3,guest ; land fetchIssues:open ; assign ; land assign:3,guest diff --git a/eval/results/desk-2026-11-11/r10-regraded/rx-9.txt b/eval/results/desk-2026-11-11/r10-regraded/rx-9.txt new file mode 100644 index 00000000..767e41fe --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/rx-9.txt @@ -0,0 +1,18 @@ +== hidden +✗ base: selecting after not found shows loading, not the old message +✗ base: help: selecting after a failure shows loading…, not the old failure +✗ related help landing out of order shows the selected issue's +74/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ base: a detail that loads while a rename is out keeps the new title +✗ a label added on one issue does not show on another +✗ rows: a declined row close leaves a rename out on another issue +33/38 passed +== fuzz +desk10-rx-9: 2/20 seeds found a hole + seed 8 step 12: open row in closed list: #1 + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; select #4 ; land closeIssue:3 ; user ann ; land fetchComments:4 ; focus ; land fetchIssue:4 ; rename Bravo ; land renameIssue:4,Bravo + seed 12 step 38: at rest: labels: #1 shows [docs,bug], server [] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-close 1 ; row-assign 0 ; row-assign 0 ; land fetchComments:1 ; land fetchLabels:1 ; land fetchIssue:4 ; rename Bravo ; land fetchComments:4 ; assign ; land search:Typo in footer ; land fetchIssue:1 ; land fetchLabels:4 ; post me too ; close ; fail renameIssue:4,Bravo ; select #1 ; row-assign 0 ; assig diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-1.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-1.txt new file mode 100644 index 00000000..03fdec29 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-1.txt @@ -0,0 +1,9 @@ +== hidden +✗ base: settings: saving an invalid name asks nothing +76/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-vanilla-1: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-2.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-2.txt new file mode 100644 index 00000000..fc3bc1c6 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-2.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-vanilla-2: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-3.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-3.txt new file mode 100644 index 00000000..54f0629d --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk10-vanilla-3: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-4.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-4.txt new file mode 100644 index 00000000..a6874019 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-4.txt @@ -0,0 +1,22 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk10-vanilla-4: 3/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 31: detail title: "retry#3 stack trace attachedrenameassign to guestcloserename", server "Crash on save" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-5.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-5.txt new file mode 100644 index 00000000..6b13ef2d --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-5.txt @@ -0,0 +1,16 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +32/38 passed +== fuzz +desk10-vanilla-5: 1/20 seeds found a hole + seed 10 step 37: row assignee: #2 shows guest, server eve + tab help ; tab issues ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; fail fetchIssues:open ; focus ; tab help ; tab issues ; land fetchIssues:open ; filter closed ; land fetchIssues:closed ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; rename Delta ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 200 diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-6.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-6.txt new file mode 100644 index 00000000..d8a48df3 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-6.txt @@ -0,0 +1,15 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +29/38 passed +== fuzz +desk10-vanilla-6: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-7.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-7.txt new file mode 100644 index 00000000..eba1ff32 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-7.txt @@ -0,0 +1,17 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk10-vanilla-7: 1/20 seeds found a hole + seed 9 step 40: row assignee: #4 shows guest, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-8.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-8.txt new file mode 100644 index 00000000..550ad65c --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-8.txt @@ -0,0 +1,18 @@ +== hidden +✗ base: retry after not found shows the issue +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +33/38 passed +== fuzz +desk10-vanilla-8: 3/20 seeds found a hole + seed 3 step 38: count: tab Issues (3), 2 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug + seed 8 step 36: detail title: "#3 Crash on savestack trace attachedassign to bobrenameclose", server "Alpha" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 ; land addLabel:4,ui ; select #3 ; focus ; fail fetchLabels:3 ; row-assign 0 ; assign ; land + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedassign to guestrenameclo", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo diff --git a/eval/results/desk-2026-11-11/r10-regraded/vanilla-9.txt b/eval/results/desk-2026-11-11/r10-regraded/vanilla-9.txt new file mode 100644 index 00000000..2936dda0 --- /dev/null +++ b/eval/results/desk-2026-11-11/r10-regraded/vanilla-9.txt @@ -0,0 +1,15 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk10-vanilla-9: 0/20 seeds found a hole diff --git a/eval/results/desk-2026-11-11/stats.py b/eval/results/desk-2026-11-11/stats.py new file mode 100644 index 00000000..894bb5c8 --- /dev/null +++ b/eval/results/desk-2026-11-11/stats.py @@ -0,0 +1,36 @@ +import re, random, json +rows={} +for l in open('summary.txt'): + a,n,h,o,z=re.match(r'(\w+)-(\d) hidden/overlap=(\d+)/77 (\d+)/38\s+fuzz=(\d+)/20',l).groups() + rows[(a,int(n))]=dict(h=77-int(h),o=38-int(o),z=int(z)) +excl={} # fuzz holes traced to legal holds: (arm,n) -> count +def D(k): r=rows[k]; return r['h']+r['o']+r['z']-excl.get(k,0) +def perm(x,y,seed=11,N=100000): + random.seed(seed); obs=sum(y)/len(y)-sum(x)/len(x); pool=x+y; c=0 + for _ in range(N): + random.shuffle(pool); xx=pool[:len(x)]; yy=pool[len(x):] + if sum(yy)/len(yy)-sum(xx)/len(xx)>=obs-1e-12: c+=1 + return (c+1)/(N+1) +def boot(x,y,seed=11,N=10000): + random.seed(seed); d=[] + for _ in range(N): + xx=[random.choice(x) for _ in x]; yy=[random.choice(y) for _ in y] + d.append(sum(xx)/len(xx)-sum(yy)/len(yy)) + d.sort(); return d[int(.05*N)], d[int(.95*N)-1] +usd=json.load(open('usd.json')) +small=range(4,10); front=range(1,4); A=('rx','react','vanilla','foldkit') +for a in A: + for tier,rg in (('frontier',front),('small',small)): + v=[D((a,n)) for n in rg]; c=sum(1 for x in v if x==0) + hz=[sum(rows[(a,n)][k] for n in rg)/len(rg) for k in 'hoz'] + cost=[x[0] for x in usd[f'{a}-{tier}']]; m=sum(cost)/len(cost) + cpc=f"${sum(cost)/c:.3f}" if c else "undefined" + print(f" {a:8}{tier:9} mean {sum(v)/len(v):5.2f} per desk {v} h/o/z {hz[0]:.1f}/{hz[1]:.1f}/{hz[2]:.1f} correct {c} $/run {m:.3f} $/correct {cpc}") +S={a:[D((a,n)) for n in small] for a in A} +ps={'H11a rx<react':perm(S['rx'],S['react']),'H11b rx<plain':perm(S['rx'],S['vanilla']),'H11b rx<foldkit':perm(S['rx'],S['foldkit'])} +order=sorted(ps.items(),key=lambda kv:kv[1]); m=len(order); prev=0 +for i,(k,p) in enumerate(order): + adjp=max(prev,min(1,(m-i)*p)); prev=adjp; print(f" {k}: p={p:.5f} Holm {adjp:.5f}") +F={a:[D((a,n)) for n in front] for a in A} +lo,hi=boot(F['rx'],F['react']); print(f" H11c frontier rx - frontier react: {sum(F['rx'])/3-sum(F['react'])/3:.2f}, 90% CI [{lo:.2f}, {hi:.2f}] -> holds if upper < 1: {hi<1}") +lo,hi=boot(S['rx'],F['react']); print(f" (H10c again) small rx - frontier react: {sum(S['rx'])/6-sum(F['react'])/3:.2f}, 90% CI [{lo:.2f}, {hi:.2f}] -> upper < 1: {hi<1}") diff --git a/eval/results/desk-2026-11-11/stats.txt b/eval/results/desk-2026-11-11/stats.txt new file mode 100644 index 00000000..7a7f261b --- /dev/null +++ b/eval/results/desk-2026-11-11/stats.txt @@ -0,0 +1,13 @@ + rx frontier mean 2.00 per desk [0, 5, 1] h/o/z 0.0/1.0/1.0 correct 1 $/run 2.048 $/correct $6.144 + rx small mean 3.83 per desk [3, 3, 4, 2, 9, 2] h/o/z 1.3/1.0/1.5 correct 0 $/run 0.092 $/correct undefined + react frontier mean 2.00 per desk [0, 2, 4] h/o/z 0.0/1.3/0.7 correct 1 $/run 1.384 $/correct $4.151 + react small mean 9.67 per desk [13, 2, 17, 14, 1, 11] h/o/z 0.2/7.2/2.3 correct 0 $/run 0.069 $/correct undefined + vanilla frontier mean 3.00 per desk [3, 3, 3] h/o/z 0.0/2.0/1.0 correct 0 $/run 1.265 $/correct undefined + vanilla small mean 14.50 per desk [12, 16, 11, 9, 20, 19] h/o/z 0.7/10.5/3.3 correct 0 $/run 0.077 $/correct undefined + foldkit frontier mean 2.67 per desk [4, 2, 2] h/o/z 0.0/2.0/0.7 correct 0 $/run 3.633 $/correct undefined + foldkit small mean 13.17 per desk [17, 13, 13, 12, 10, 14] h/o/z 0.2/10.8/2.2 correct 0 $/run 0.175 $/correct undefined + H11b rx<foldkit: p=0.00103 Holm 0.00309 + H11b rx<plain: p=0.00206 Holm 0.00412 + H11a rx<react: p=0.04727 Holm 0.04727 + H11c frontier rx - frontier react: 0.00, 90% CI [-2.33, 2.33] -> holds if upper < 1: False + (H10c again) small rx - frontier react: 1.83, 90% CI [-0.33, 4.17] -> upper < 1: False diff --git a/eval/results/desk-2026-11-11/summary.txt b/eval/results/desk-2026-11-11/summary.txt new file mode 100644 index 00000000..fee9e073 --- /dev/null +++ b/eval/results/desk-2026-11-11/summary.txt @@ -0,0 +1,36 @@ +rx-1 hidden/overlap=77/77 38/38 fuzz=0/20 +rx-2 hidden/overlap=77/77 36/38 fuzz=3/20 +rx-3 hidden/overlap=77/77 37/38 fuzz=0/20 +rx-4 hidden/overlap=76/77 37/38 fuzz=1/20 +rx-5 hidden/overlap=77/77 37/38 fuzz=2/20 +rx-6 hidden/overlap=75/77 38/38 fuzz=2/20 +rx-7 hidden/overlap=76/77 38/38 fuzz=1/20 +rx-8 hidden/overlap=75/77 34/38 fuzz=3/20 +rx-9 hidden/overlap=75/77 38/38 fuzz=0/20 +react-1 hidden/overlap=77/77 38/38 fuzz=0/20 +react-2 hidden/overlap=77/77 36/38 fuzz=0/20 +react-3 hidden/overlap=77/77 36/38 fuzz=2/20 +react-4 hidden/overlap=77/77 27/38 fuzz=2/20 +react-5 hidden/overlap=76/77 37/38 fuzz=0/20 +react-6 hidden/overlap=77/77 26/38 fuzz=5/20 +react-7 hidden/overlap=77/77 28/38 fuzz=4/20 +react-8 hidden/overlap=77/77 37/38 fuzz=0/20 +react-9 hidden/overlap=77/77 30/38 fuzz=3/20 +vanilla-1 hidden/overlap=77/77 36/38 fuzz=1/20 +vanilla-2 hidden/overlap=77/77 36/38 fuzz=1/20 +vanilla-3 hidden/overlap=77/77 36/38 fuzz=1/20 +vanilla-4 hidden/overlap=77/77 29/38 fuzz=3/20 +vanilla-5 hidden/overlap=77/77 26/38 fuzz=4/20 +vanilla-6 hidden/overlap=77/77 29/38 fuzz=2/20 +vanilla-7 hidden/overlap=77/77 29/38 fuzz=0/20 +vanilla-8 hidden/overlap=75/77 27/38 fuzz=7/20 +vanilla-9 hidden/overlap=75/77 25/38 fuzz=4/20 +foldkit-1 hidden/overlap=77/77 36/38 fuzz=2/20 +foldkit-2 hidden/overlap=77/77 36/38 fuzz=0/20 +foldkit-3 hidden/overlap=77/77 36/38 fuzz=0/20 +foldkit-4 hidden/overlap=77/77 25/38 fuzz=4/20 +foldkit-5 hidden/overlap=77/77 27/38 fuzz=2/20 +foldkit-6 hidden/overlap=77/77 26/38 fuzz=1/20 +foldkit-7 hidden/overlap=77/77 27/38 fuzz=1/20 +foldkit-8 hidden/overlap=76/77 30/38 fuzz=1/20 +foldkit-9 hidden/overlap=77/77 28/38 fuzz=4/20 diff --git a/eval/results/desk-2026-11-11/usd.json b/eval/results/desk-2026-11-11/usd.json new file mode 100644 index 00000000..c27bada1 --- /dev/null +++ b/eval/results/desk-2026-11-11/usd.json @@ -0,0 +1 @@ +{"rx-frontier": [[2.1487032, 24, 537175.8, 1], [2.5559634, 26, 638990.85, 2], [1.4391163999999999, 14, 359779.1, 3]], "react-frontier": [[1.4005798, 10, 350144.95, 1], [1.25414, 12, 313535, 2], [1.4962373999999998, 16, 374059.35, 3]], "vanilla-frontier": [[1.2282464, 9, 307061.6, 1], [1.4704583999999998, 16, 367614.6, 2], [1.0970423999999999, 12, 274260.6, 3]], "foldkit-frontier": [[3.5695232, 39, 892380.8, 1], [4.010596, 39, 1002649, 2], [3.3185599999999997, 34, 829640, 3]], "rx-small": [[0.12365911, 21, 1236591.1, 4], [0.09165008, 15, 916500.8, 5], [0.07712176, 14, 771217.6, 6], [0.080014315, 12, 800143.15, 7], [0.064075775, 11, 640757.75, 8], [0.117098075, 20, 1170980.75, 9]], "react-small": [[0.088643315, 15, 886433.15, 4], [0.049387755, 10, 493877.55, 5], [0.068611995, 13, 686119.95, 6], [0.05683599499999999, 8, 568359.95, 7], [0.096263405, 16, 962634.05, 8], [0.05682778499999999, 9, 568277.85, 9]], "vanilla-small": [[0.07337236, 8, 733723.6, 4], [0.066924585, 9, 669245.85, 5], [0.09436494499999999, 13, 943649.45, 6], [0.06235414, 7, 623541.4, 7], [0.099328465, 16, 993284.65, 8], [0.068555105, 7, 685551.05, 9]], "foldkit-small": [[0.182765515, 33, 1827655.15, 4], [0.1660664, 32, 1660664, 5], [0.136690155, 29, 1366901.55, 6], [0.170539045, 41, 1705390.4500000002, 7], [0.23715972999999996, 59, 2371597.3, 8], [0.15544205, 37, 1554420.5, 9]]} \ No newline at end of file From 93f7c5d76c1b0577f34b7003b299e7b8c75fe58d Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 11 Nov 2026 05:19:19 +0000 Subject: [PATCH 08/23] eval: round 11, rx-4 seed 15 traced Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/results/desk-2026-11-11/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/results/desk-2026-11-11/README.md b/eval/results/desk-2026-11-11/README.md index 0b99c06a..f366616f 100644 --- a/eval/results/desk-2026-11-11/README.md +++ b/eval/results/desk-2026-11-11/README.md @@ -73,7 +73,7 @@ Round 10 as first published had .rx at 6.7 in both tiers. The Solid fixes since - **rx-2, seeds 1, 3, 18: a Solid bug.** A keyed projection whose function returns another keyed projection loses a row when the question switches while a guess is out. In a browser this shows as a duplicated row. The plain Solid repro is filed as solidjs/solid#4762. It counts against .rx, as pre-registered. - **rx-5, rx-6, rx-7, the "row assignee" holes: agent mistakes.** The assign guess is written only when the row is in the list on screen (`if (i >= 0)`), so a row hidden by a close guess never gets it. - **rx-5 and rx-8, the "detail title" holes: agent mistakes.** rx-5 seed 15 was replayed; rx-8 has the same code shape and was not replayed. The rename guess is written to a source derived from the detail before the detail has loaded. Solid drops a guess that was never shown (D-34), so the old title stays on screen while the rename is out. -- **Not traced:** rx-4 seed 15 (detail title, with the guesses kept in their own list). +- **rx-4 seed 15 (detail title): agent mistake.** Rename guesses are kept in their own list, and the heading reads it with `find`, which returns the oldest guess. With two renames out, the heading shows the first one, while the row shows the second. ## Lines of source per app From 40de1d038c8ba1cea7a4bfe3efd9c1d178292d9f Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 11 Nov 2026 06:19:22 +0000 Subject: [PATCH 09/23] eval: xstate arm (React 19 + XState v5, Stately's own docs) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/harness/build.mjs | 4 ++-- eval/harness/check.mjs | 2 +- eval/harness/prompt-arms.json | 12 ++++++++++++ eval/harness/scaffold.mjs | 20 +++++++++++++++----- eval/package.json | 4 +++- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/eval/harness/build.mjs b/eval/harness/build.mjs index 769bfdde..e39698d8 100644 --- a/eval/harness/build.mjs +++ b/eval/harness/build.mjs @@ -1,6 +1,6 @@ // Builds one app (a run directory or a reference) into `<dir>/.build/dist` with Vite, for the tests to drive. // rx arm: `rxc compile src/main.rx` → JavaScript over Solid rc.19, mounted with the .rx runtime (ref/compile/rt-web.ts) -// react arm: `src/App.tsx` (default export) mounted with createRoot under StrictMode, as Vite's React template does +// react and xstate arms: `src/App.tsx` (default export) mounted with createRoot under StrictMode, as Vite's React template does // solid arm: `src/App.tsx` (default export) compiled by @solidjs/compiler (Solid 2's JSX compiler) and mounted with // @solidjs/web's `render` // vanilla arm: `src/main.ts` default-exports `mount(root: HTMLElement)`, called once with the `#app` container @@ -57,7 +57,7 @@ export async function build(dir, arm = armOf(dir)) { writeFileSync(path.join(work, "entry.js"), entry); writeFileSync(path.join(work, "index.html"), `<!doctype html><html><head><meta charset="utf-8"><title>app
\n`); const vite = await import(pathToFileURL(path.join(evalDir, "node_modules/vite/dist/node/index.js")).href); - const plugins = arm === "react" ? [(await import(pathToFileURL(path.join(evalDir, "node_modules/@vitejs/plugin-react/dist/index.js")).href)).default()] + const plugins = arm === "react" || arm === "xstate" ? [(await import(pathToFileURL(path.join(evalDir, "node_modules/@vitejs/plugin-react/dist/index.js")).href)).default()] : arm === "solid" ? [await solidPlugin()] : []; try { await vite.build({ diff --git a/eval/harness/check.mjs b/eval/harness/check.mjs index 6de099fa..45328c32 100644 --- a/eval/harness/check.mjs +++ b/eval/harness/check.mjs @@ -1,6 +1,6 @@ // The arm's compiler feedback, as the agent runs it. // rx: rxc check src/main.rx -// react: tsc --noEmit -p . and eslint (react-hooks rules) +// react, xstate: tsc --noEmit -p . and eslint (react-hooks rules) // solid: tsc --noEmit -p . and eslint (eslint-plugin-solid's v2 config) // vanilla: tsc --noEmit -p . (strict) // foldkit: tsc --noEmit -p . with Foldkit's template tsconfig (its lint, oxlint + effect-tsgo, is not on npm) diff --git a/eval/harness/prompt-arms.json b/eval/harness/prompt-arms.json index b176568a..2f18e30e 100644 --- a/eval/harness/prompt-arms.json +++ b/eval/harness/prompt-arms.json @@ -5,6 +5,18 @@ "API_TYPES": "`src/api.d.ts`", "CHECKER": "the TypeScript compiler (strict) and ESLint with eslint-plugin-react-hooks' recommended rules (including the React Compiler's)" }, + "xstate": { + "DOCS_RULE": "", + "LANG_RULE": "React 19 with TypeScript and XState v5: `src/App.tsx` must default-export the root component (it is mounted with `createRoot` under `StrictMode`). You may add files under `src/`. Installed packages you may use: react, react-dom, xstate 5.32 and @xstate/react 6.1 (XState v5, which differs from v4). Model the app's state and its server requests as XState machines and actors (`setup`, `createMachine`, `fromPromise`, `useMachine`/`useActor`/`useSelector`), as an XState codebase would.", + "API_TYPES": "`src/api.d.ts`", + "CHECKER": "the TypeScript compiler (strict) and ESLint with eslint-plugin-react-hooks' recommended rules (including the React Compiler's)" + }, + "xstate-inline": { + "DOCS_RULE": ", except that you may read XState's documentation in {{DIR}}/docs/xstate (Stately's v5 docs pages)", + "LANG_RULE": "React 19 with TypeScript and XState v5: `src/App.tsx` must default-export the root component (it is mounted with `createRoot` under `StrictMode`). You may add files under `src/`. Installed packages you may use: react, react-dom, xstate 5.32 and @xstate/react 6.1 (XState v5, which differs from v4). Model the app's state and its server requests as XState machines and actors (`setup`, `createMachine`, `fromPromise`, `useMachine`/`useActor`/`useSelector`), as an XState codebase would. The two pages at the end of this message are XState v5's own cheatsheet and @xstate/react's README; the rest of Stately's v5 docs are in `docs/xstate` (for example `xstate-react.mdx`, `promise-actors.mdx`, `invoke.mdx`, `actors.mdx`).", + "API_TYPES": "`src/api.d.ts`", + "CHECKER": "the TypeScript compiler (strict) and ESLint with eslint-plugin-react-hooks' recommended rules (including the React Compiler's)" + }, "rx": { "DOCS_RULE": ", except that you may read the language documentation in {{DIR}}/docs", "LANG_RULE": "ReactiveScript (`.rx`), a new language that compiles to Solid 2.0. Nothing about it is online. Read `docs/guide/async.md` first: one page on reading from and writing to a server. Its specification is in `docs/spec/` (the concrete syntax is `docs/spec/07-surface.md`; modules and imports are `docs/spec/09-modules.md`). `src/main.rx` must be the program's entry, `export default component App() { … }`; you may add `.rx` modules under `src/`. Import the backend from \"./api\": loaders with `import pure { … } from \"./api\"`, functions that write with `import { … } from \"./api\"`. The prelude gives `import pure { len, str, map, filter, sum, every, indexOf } from \"./prelude\"`. Give an element a test id with `data-testid=\"…\"`.", diff --git a/eval/harness/scaffold.mjs b/eval/harness/scaffold.mjs index 1d741264..ffd3211e 100644 --- a/eval/harness/scaffold.mjs +++ b/eval/harness/scaffold.mjs @@ -12,15 +12,20 @@ // For foldkit, the docs are Foldkit's own FOLDKIT.md (create-foldkit-app's template) at the end of the prompt, and the // Foldkit repository at the installed release's tag readable in docs/foldkit, as Foldkit tells agents to vendor it // (examples, skills, AGENTS.md, the framework source); FOLDKIT_REPO names the checkout (default ../../foldkit). -// Arms: rx, react, solid (Solid 2 TSX, round 5), foldkit (round 6), vanilla (plain TypeScript and DOM, no framework). -// Usage: node harness/scaffold.mjs [--reference] [--brief ] [--base ] [--inline-docs] -import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, symlinkSync, writeFileSync } from "node:fs"; +// Arms: rx, react, solid (Solid 2 TSX, round 5), foldkit (round 6), vanilla (plain TypeScript and DOM, no framework), +// xstate (React with XState v5 machines, no TanStack Query; 2026-11-11). +// Usage: node harness/scaffold.mjs [--reference] [--brief ] [--base ] [--inline-docs] +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; const here = path.dirname(fileURLToPath(import.meta.url)); const evalDir = path.resolve(here, ".."); const foldkitRepo = process.env.FOLDKIT_REPO ?? path.resolve(evalDir, "../../foldkit"); +// For xstate, the docs are Stately's own (github.com/statelyai/docs, content/docs, the XState v5 pages as .mdx): the +// cheatsheet and @xstate/react's README at the end of the prompt, every page readable in docs/xstate. STATELY_DOCS names the +// checkout (default ../../stately-docs). +const statelyDocs = process.env.STATELY_DOCS ?? path.resolve(evalDir, "../../stately-docs"); export function scaffold(task, arm, dir, { reference = false, brief = null, base = null, inline = false } = {}) { const t = path.join(evalDir, "tasks", task); @@ -39,8 +44,8 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base }, null, 2) + "\n"); cpSync(path.join(t, "visible.mjs"), path.join(dir, "tests/visible.mjs")); cpSync(path.join(t, "fixtures.mjs"), path.join(dir, "tests/fixtures.mjs")); - if (arm === "react" || arm === "solid" || arm === "vanilla") { - const jsx = arm === "react" ? { jsx: "react-jsx" } : arm === "solid" ? { jsx: "preserve", jsxImportSource: "@solidjs/web" } : {}; + if (arm === "react" || arm === "xstate" || arm === "solid" || arm === "vanilla") { + const jsx = arm === "react" || arm === "xstate" ? { jsx: "react-jsx" } : arm === "solid" ? { jsx: "preserve", jsxImportSource: "@solidjs/web" } : {}; writeFileSync(path.join(dir, "tsconfig.json"), JSON.stringify({ compilerOptions: { target: "ES2022", module: "ESNext", moduleResolution: "bundler", ...jsx, strict: true, noEmit: true, allowJs: true, checkJs: false, skipLibCheck: true, lib: ["ES2022", "DOM", "DOM.Iterable"] }, include: ["src"], @@ -58,6 +63,10 @@ export function scaffold(task, arm, dir, { reference = false, brief = null, base cpSync(path.join(foldkitRepo, p), path.join(dir, "docs/foldkit", p), { recursive: true }); if (!existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules")); } + if (arm === "xstate") { + cpSync(path.join(statelyDocs, "content/docs"), path.join(dir, "docs/xstate"), { recursive: true, filter: f => !/[\\/]assets([\\/]|$)/.test(f) && (statSync(f).isDirectory() || /\.(mdx?|json)$/.test(f)) }); + if (!existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules")); + } // Solid's own docs ship in its package: the agent can read them there before its first build if (arm === "solid" && !existsSync(path.join(dir, "node_modules"))) symlinkSync(path.join(evalDir, "node_modules"), path.join(dir, "node_modules")); if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true }); @@ -81,6 +90,7 @@ export function prompt(arm, dir, { plain = false, change = false, inline = false for (let i = 0; i < 2; i++) for (const k of [...Object.keys(fill), "DIR"]) text = text.replaceAll(`{{${k}}}`, k === "DIR" ? dir : fill[k]); if (docs && arm === "rx") for (const f of ["language.md", "async.md", "reference.md"]) text += `\n---\n\n${readFileSync(path.join(evalDir, "../guide", f), "utf8")}`; if (docs && arm === "foldkit") text += `\n---\n\n${readFileSync(path.join(foldkitRepo, "packages/create-foldkit-app/templates/base/FOLDKIT.md"), "utf8")}`; + if (docs && arm === "xstate") text += `\n---\n\n${readFileSync(path.join(statelyDocs, "content/docs/cheatsheet.mdx"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}\n---\n\n${readFileSync(path.join(evalDir, "node_modules/@xstate/react/README.md"), "utf8")}`; if (docs && arm === "solid") text += `\n---\n\n${readFileSync(path.join(evalDir, "node_modules/solid-js/skills/solid-2.0/SKILL.md"), "utf8").replace(/^---\n[\s\S]*?\n---\n+/, "")}`; return text; } diff --git a/eval/package.json b/eval/package.json index 9992f9b0..4dd7abd3 100644 --- a/eval/package.json +++ b/eval/package.json @@ -12,6 +12,7 @@ "@types/react": "19.2.2", "@types/react-dom": "19.2.1", "@vitejs/plugin-react": "5.0.4", + "@xstate/react": "6.1.0", "effect": "^4.1.0", "eslint": "9.37.0", "eslint-plugin-react-hooks": "7.0.0", @@ -23,6 +24,7 @@ "solid-js": "2.0.0-rc.21", "typescript": "5.9.3", "typescript-eslint": "8.46.0", - "vite": "7.1.9" + "vite": "7.1.9", + "xstate": "5.32.6" } } From d10f7a045be46f4dfeb2f39f00f9de6194e5e58c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 07:46:50 +0000 Subject: [PATCH 10/23] eval: XState side check results; cost.mjs knows the xstate arm Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/harness/cost.mjs | 2 +- eval/results/xstate-2026-11-11/README.md | 61 +++++++++ eval/results/xstate-2026-11-11/agents.txt | 9 ++ eval/results/xstate-2026-11-11/cost.json | 118 ++++++++++++++++++ .../xstate-2026-11-11/graded/xstate-1.txt | 8 ++ .../xstate-2026-11-11/graded/xstate-2.txt | 8 ++ .../xstate-2026-11-11/graded/xstate-3.txt | 8 ++ .../xstate-2026-11-11/graded/xstate-4.txt | 9 ++ .../xstate-2026-11-11/graded/xstate-5.txt | 11 ++ .../xstate-2026-11-11/graded/xstate-6.txt | 12 ++ .../xstate-2026-11-11/graded/xstate-7.txt | 26 ++++ .../xstate-2026-11-11/graded/xstate-8.txt | 7 ++ .../xstate-2026-11-11/graded/xstate-9.txt | 20 +++ eval/results/xstate-2026-11-11/summary.txt | 9 ++ 14 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 eval/results/xstate-2026-11-11/README.md create mode 100644 eval/results/xstate-2026-11-11/agents.txt create mode 100644 eval/results/xstate-2026-11-11/cost.json create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-1.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-2.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-3.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-4.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-5.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-6.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-7.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-8.txt create mode 100644 eval/results/xstate-2026-11-11/graded/xstate-9.txt create mode 100644 eval/results/xstate-2026-11-11/summary.txt diff --git a/eval/harness/cost.mjs b/eval/harness/cost.mjs index 29bda216..1e6865f2 100644 --- a/eval/harness/cost.mjs +++ b/eval/harness/cost.mjs @@ -45,7 +45,7 @@ export function permutation(a, b, n = 100000, seed = 5) { if (import.meta.url === pathToFileURL(process.argv[1]).href) { const [list, dir, out] = process.argv.slice(2); const runs = readFileSync(list, "utf8").trim().split("\n").map(l => l.trim().split(/\s+/)).map(([run, id]) => { - const [, task, arm, n] = run.match(/^(\w+)-(rx|react|solid|foldkit|vanilla)-(\d+)$/); + const [, task, arm, n] = run.match(/^(\w+)-(rx|react|xstate|solid|foldkit|vanilla)-(\d+)$/); return { run, task, arm, n: Number(n), ...usage(path.join(dir, `agent-${id}.jsonl`)) }; }); for (const r of runs) console.log(`${r.run.padEnd(16)} billed ${Math.round(r.billed).toString().padStart(7)} calls ${r.calls} out ${r.output} cacheRead ${r.cacheRead} cacheWrite ${r.cacheWrite}`); diff --git a/eval/results/xstate-2026-11-11/README.md b/eval/results/xstate-2026-11-11/README.md new file mode 100644 index 00000000..7c33219b --- /dev/null +++ b/eval/results/xstate-2026-11-11/README.md @@ -0,0 +1,61 @@ +# XState side check (2026-11-11, not pre-registered) + +This side check runs nine desks in a new `xstate` arm, compared against round 11's arms. Everything else matches round 11 (`eval/results/desk-2026-11-11`): + +- **Brief:** desk9, `--brief plain`. +- **Grading:** the same grader (`harness/grade.sh`) and tests. +- **Toolchain:** Solid next 6751472f, used only by the .rx and Solid arms. +- **Tiers:** 1–3 are the frontier tier on Opus, 4–9 the small tier on Haiku. + +Ryan chose to run this side check first and then add XState as a pre-registered arm in round 12. + +## The arm + +- **Stack:** React 19, TypeScript, xstate 5.32.6 and @xstate/react 6.1.0. TanStack Query is not available, so the machines own the server state. +- **Instruction:** model state and requests as XState machines and actors. +- **Docs:** Stately's own, unedited: + - in the prompt: the v5 cheatsheet and @xstate/react's README; + - readable in `docs/xstate`: every v5 page (statelyai/docs at 7ee0c60). +- **Checker:** the same as React's, tsc strict plus react-hooks. + +## Defects per app + +| arm | frontier mean | small mean | small per desk | small median | +|---|---|---|---|---| +| **XState** | **2.0** (2, 2, 2) | **7.0** | 3, 5, 6, 16, 1, 11 | 5.5 | +| .rx (round 11) | 2.0 | 3.8 | 3, 3, 4, 2, 9, 2 | 3 | +| React (round 11) | 2.0 | 9.7 | 13, 2, 17, 14, 1, 11 | 12 | +| Foldkit (round 11) | 2.7 | 13.2 | 17, 13, 13, 12, 10, 14 | 13 | + +Small tier, one-sided permutation tests (seed 11), descriptive only: + +| comparison | p | +|---|---| +| XState < Foldkit | 0.021 | +| XState < React | 0.25 | +| .rx < XState | 0.13 | + +Every frontier XState desk fails the same two overlap tests ("comments / labels that load while an add is out keep it"), as most desks in other arms do. The most common small-tier failures are settings saves out of order (a later save failing while an earlier one is out) and the same two overlap tests. Two small desks (7, 9) account for most of the fuzz holes. + +## Cost and size + +| | frontier $/run | small $/run | frontier calls | small calls | lines | +|---|---|---|---|---|---| +| XState | 2.30 | 0.142 | 24.0 | 25.8 | about 1,300 | +| .rx | 2.05 | 0.092 | 21.3 | 15.5 | about 390 | +| React | 1.38 | 0.069 | 12.7 | 11.8 | about 760 | +| Foldkit | 3.63 | 0.175 | 37.3 | 38.5 | about 2,300 | + +Cost is billed tokens at round 10's rates (`cost.json`). + +## Reading + +- **Against Foldkit:** state machines do better than the Effect/TEA process arm in the small tier, at about half the defects, and cost less. +- **Against React:** XState sits between plain React and .rx, but six runs don't separate it from either. +- **Frontier:** XState is level with React and .rx at 2.0, at about 1.7× React's cost. +- **Caveat:** this check is not pre-registered and compares with another day's runs of the other arms. Round 12 runs XState as a pre-registered arm. + +## Deviations + +- Desks 5, 6 and 8 drove their machines in Node with scratch scripts, which they then deleted. That is allowed (scratch inside the run dir). +- Desk 2 reports that `reenter: true` on one region of a parallel state restarted its siblings, and worked around it. This was not checked against XState. diff --git a/eval/results/xstate-2026-11-11/agents.txt b/eval/results/xstate-2026-11-11/agents.txt new file mode 100644 index 00000000..dce1fad2 --- /dev/null +++ b/eval/results/xstate-2026-11-11/agents.txt @@ -0,0 +1,9 @@ +desk12-xstate-1 a696f287a3a667250 +desk12-xstate-2 a3363fbe4e9baf4ae +desk12-xstate-3 a9248d39569a18575 +desk12-xstate-4 a54d219a267005bfd +desk12-xstate-5 a3cc3ebeb5e9d7b54 +desk12-xstate-6 aee0f2d1c55f5a34a +desk12-xstate-7 a0591e9fbd928c710 +desk12-xstate-8 a8f09afb82951c9a4 +desk12-xstate-9 a7261369155078e71 diff --git a/eval/results/xstate-2026-11-11/cost.json b/eval/results/xstate-2026-11-11/cost.json new file mode 100644 index 00000000..11eddb11 --- /dev/null +++ b/eval/results/xstate-2026-11-11/cost.json @@ -0,0 +1,118 @@ +{ + "weights": { + "input": 1, + "cacheWrite": 1.25, + "cacheRead": 0.1, + "output": 5 + }, + "runs": [ + { + "run": "desk12-xstate-1", + "task": "desk12", + "arm": "xstate", + "n": 1, + "input": 50, + "cacheWrite": 114420, + "cacheRead": 2077920, + "output": 44908, + "calls": 25, + "billed": 575407 + }, + { + "run": "desk12-xstate-2", + "task": "desk12", + "arm": "xstate", + "n": 2, + "input": 48, + "cacheWrite": 116767, + "cacheRead": 1962689, + "output": 45752, + "calls": 24, + "billed": 571035.65 + }, + { + "run": "desk12-xstate-3", + "task": "desk12", + "arm": "xstate", + "n": 3, + "input": 46, + "cacheWrite": 116897, + "cacheRead": 1943544, + "output": 48264, + "calls": 23, + "billed": 581841.65 + }, + { + "run": "desk12-xstate-4", + "task": "desk12", + "arm": "xstate", + "n": 4, + "input": 32, + "cacheWrite": 204894, + "cacheRead": 1709493, + "output": 139689, + "calls": 16, + "billed": 1125543.8 + }, + { + "run": "desk12-xstate-5", + "task": "desk12", + "arm": "xstate", + "n": 5, + "input": 54, + "cacheWrite": 248625, + "cacheRead": 4765754, + "output": 172493, + "calls": 27, + "billed": 1649875.65 + }, + { + "run": "desk12-xstate-6", + "task": "desk12", + "arm": "xstate", + "n": 6, + "input": 74, + "cacheWrite": 235710, + "cacheRead": 6095807, + "output": 158761, + "calls": 37, + "billed": 1698097.2000000002 + }, + { + "run": "desk12-xstate-7", + "task": "desk12", + "arm": "xstate", + "n": 7, + "input": 58, + "cacheWrite": 222507, + "cacheRead": 4140931, + "output": 148063, + "calls": 29, + "billed": 1432599.85 + }, + { + "run": "desk12-xstate-8", + "task": "desk12", + "arm": "xstate", + "n": 8, + "input": 50, + "cacheWrite": 227533, + "cacheRead": 3793204, + "output": 145328, + "calls": 25, + "billed": 1390426.65 + }, + { + "run": "desk12-xstate-9", + "task": "desk12", + "arm": "xstate", + "n": 9, + "input": 42, + "cacheWrite": 215867, + "cacheRead": 2761986, + "output": 140794, + "calls": 21, + "billed": 1250044.35 + } + ] +} diff --git a/eval/results/xstate-2026-11-11/graded/xstate-1.txt b/eval/results/xstate-2026-11-11/graded/xstate-1.txt new file mode 100644 index 00000000..9632b1d5 --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-1.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-1: 0/20 seeds found a hole diff --git a/eval/results/xstate-2026-11-11/graded/xstate-2.txt b/eval/results/xstate-2026-11-11/graded/xstate-2.txt new file mode 100644 index 00000000..449aa353 --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-2: 0/20 seeds found a hole diff --git a/eval/results/xstate-2026-11-11/graded/xstate-3.txt b/eval/results/xstate-2026-11-11/graded/xstate-3.txt new file mode 100644 index 00000000..84340051 --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-3: 0/20 seeds found a hole diff --git a/eval/results/xstate-2026-11-11/graded/xstate-4.txt b/eval/results/xstate-2026-11-11/graded/xstate-4.txt new file mode 100644 index 00000000..0cd77356 --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-4.txt @@ -0,0 +1,9 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: an earlier save failing after a later one succeeded leaves the later one +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +35/38 passed +== fuzz +desk12-xstate-4: 0/20 seeds found a hole diff --git a/eval/results/xstate-2026-11-11/graded/xstate-5.txt b/eval/results/xstate-2026-11-11/graded/xstate-5.txt new file mode 100644 index 00000000..65a0f89e --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-5.txt @@ -0,0 +1,11 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: settings: an earlier save failing after a later one succeeded leaves the later one +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +35/38 passed +== fuzz +desk12-xstate-5: 0/20 seeds found a hole diff --git a/eval/results/xstate-2026-11-11/graded/xstate-6.txt b/eval/results/xstate-2026-11-11/graded/xstate-6.txt new file mode 100644 index 00000000..b5edcfb5 --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-6.txt @@ -0,0 +1,12 @@ +== hidden +✗ base: settings: saving an invalid name asks nothing +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +74/77 passed +== overlap +✗ base: settings: an earlier save failing after a later one succeeded leaves the later one +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +35/38 passed +== fuzz +desk12-xstate-6: 0/20 seeds found a hole diff --git a/eval/results/xstate-2026-11-11/graded/xstate-7.txt b/eval/results/xstate-2026-11-11/graded/xstate-7.txt new file mode 100644 index 00000000..cc34481a --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-7.txt @@ -0,0 +1,26 @@ +== hidden +✗ base: a failing list leaves the detail alone +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-xstate-7: 4/20 seeds found a hole + seed 3 step 38: at rest: detail: "loading #4retryassign to bobrenamecloseloading commentspostl", server "#4 Delta" + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land removeLabel:4,docs ; fail c + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 16 step 36: at rest: detail: "loading #2retryassign to annrenamecloseclose error: declined", server "#2 Bravo" + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as + seed 18 step 36: detail title: "#4 Typo in footerfooter says 2019retryassign to guestrenamec", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/xstate-2026-11-11/graded/xstate-8.txt b/eval/results/xstate-2026-11-11/graded/xstate-8.txt new file mode 100644 index 00000000..b2c18856 --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-8.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk12-xstate-8: 0/20 seeds found a hole diff --git a/eval/results/xstate-2026-11-11/graded/xstate-9.txt b/eval/results/xstate-2026-11-11/graded/xstate-9.txt new file mode 100644 index 00000000..d4a96c6d --- /dev/null +++ b/eval/results/xstate-2026-11-11/graded/xstate-9.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-xstate-9: 3/20 seeds found a hole + seed 2 step 34: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land addComment:2,guest,fixed? ; rename Alp + seed 15 step 37: row assignee: #3 shows guest, server null + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land closeIssue:4 ; land fetchIssue:3 ; assign ; land search:Crash on save ; rename Delta ; + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/xstate-2026-11-11/summary.txt b/eval/results/xstate-2026-11-11/summary.txt new file mode 100644 index 00000000..469d74f6 --- /dev/null +++ b/eval/results/xstate-2026-11-11/summary.txt @@ -0,0 +1,9 @@ +xstate-1 hidden/overlap=77/77 36/38 fuzz=0/20 defects=2 +xstate-2 hidden/overlap=77/77 36/38 fuzz=0/20 defects=2 +xstate-3 hidden/overlap=77/77 36/38 fuzz=0/20 defects=2 +xstate-4 hidden/overlap=77/77 35/38 fuzz=0/20 defects=3 +xstate-5 hidden/overlap=75/77 35/38 fuzz=0/20 defects=5 +xstate-6 hidden/overlap=74/77 35/38 fuzz=0/20 defects=6 +xstate-7 hidden/overlap=76/77 27/38 fuzz=4/20 defects=16 +xstate-8 hidden/overlap=77/77 37/38 fuzz=0/20 defects=1 +xstate-9 hidden/overlap=77/77 30/38 fuzz=3/20 defects=11 From 071994811706a2650500859fcb85eb4b7b000665 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 08:41:52 +0000 Subject: [PATCH 11/23] eval: round 12 pre-registration (draft) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-12.md | 75 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 eval/PREREG-desk-12.md diff --git a/eval/PREREG-desk-12.md b/eval/PREREG-desk-12.md new file mode 100644 index 00000000..7dfe8cc8 --- /dev/null +++ b/eval/PREREG-desk-12.md @@ -0,0 +1,75 @@ +# Round 12: round 11 again with five arms and twice the frontier runs + +Pre-registration, committed before any round 12 agent runs. Status: DRAFT. + +## Why + +Round 11 (`results/desk-2026-11-11`) replicated round 10's small-tier result. + +- **Small tier:** .rx had 3.8 defects per app and React 9.7. H11a holds, at p = 0.047. +- **Frontier tier:** every arm was between 2 and 3. With three runs per arm, H11c ("no worse than React") could not be shown: the interval was [−2.3, 2.3]. +- **XState:** a side check the same day (`results/xstate-2026-11-11`, not pre-registered) put small XState at 7.0, between .rx and React and below Foldkit's 13.2. + +Round 12 asks whether these hold with fresh agents. It adds XState as a fifth arm and doubles the frontier runs, so the frontier comparison has a chance to resolve. + +Ryan, 2026-11-11: replication now, maintenance next. Maintenance moves to round 13 and runs on round 12's apps. + +## Task and arms (as round 11 unless stated) + +**Task:** `tasks/desk9` with `brief-plain.md`, unchanged. Graded by `hidden.mjs` (77) and `hidden-overlap.mjs` (38), unchanged. + +**Arms:** +- **`rx`** with `--inline-docs` (`language.md`, `async.md`, `reference.md`). +- **`foldkit`** with `--inline-docs`. +- **`react`**. +- **`vanilla`**. +- **`xstate`**, new: React 19 with xstate 5.32.6 and @xstate/react 6.1.0, with no TanStack Query. It uses `--inline-docs`, which here means Stately's own v5 cheatsheet and @xstate/react's README in the prompt. Every Stately v5 docs page is readable in `docs/xstate`, from statelyai/docs at 7ee0c60. The checker is the same as React's. + +**Language and guide changes since round 11's freeze:** +- `guess` replaces `optimistic` (#47). +- Errors are cleared with `onFlush` (#47). +- A guess over an equation is allowed (#48). +- Destructuring works (#49). + +The guide follows all four. `rxc` is built from the commit that holds this file once it is frozen. Round 11's .rx desks use `optimistic`, so the gate below grades them with round 11's `rxc`, which isolates the Solid change. + +**Guide additions** (principle only, no task code): +- **Guess over an equation:** `async.md` shows it. +- **Held writes:** the section "What lands together, and what waits" now says that a write after `await` in one action waits for any other action guessing on the same source. + +**Solid:** `solid-next` at `next`, at a commit holding the fix for both cases of solidjs/solid#4762, built from source. TBD: 0a11dbb8 fixes the first case; the second, which is two guesses with one landing after a switch, is reported on the issue. + +**Harness:** subagents, as in round 11. + +## Runs, grading + +**Runs:** 6 frontier and 6 small per arm, 60 in all, in one interleaved batch. Each run gets at most 15 `./check`/`./test` calls. + +**Grading:** `harness/grade.sh`, with round 11's definitions of correct and cost. Only defects are counted. A fuzz hole counts unless it is traced to a legal hold, and each exclusion is listed with its trace. + +## Hypotheses and tests + +All tests use seed 12. + +- **H12a** (primary, replication of H11a): small .rx runs have fewer defects than small React runs. One-sided permutation test on mean defects, 100,000 relabellings. +- **H12b**: three comparisons, Holm-corrected together with H12a: + - small .rx has fewer defects than small XState; + - small .rx has fewer defects than small plain; + - small .rx has fewer defects than small Foldkit. +- **H12c**: frontier .rx runs have no more defects than frontier React runs. Difference in means with a 90% bootstrap interval, 10,000 resamples. The claim holds if the upper end is under 1 defect. +- **H12d**: small XState runs have fewer defects than small Foldkit runs. Same test as H12a, reported apart from the .rx family. This is the PROCESS question: state machines against Effect/TEA. +- **Descriptive:** cost per correct app, by arm and tier. +- **Reported alongside:** + - the change from round 11 for each arm and tier; + - defects by kind; + - turns, cost per run and lines. + +The direction is the claim being tested. A null or reversed result is reported as plainly. + +## Gate before freezing + +Round 11's nine .rx desks are regraded on the pinned Solid, with round 11's `rxc`. Any desk that gains a hole is traced. A Solid regression is filed upstream, then fixed or recorded, before the round starts. + +## Cost + +About $67 at round 11's prices: 6 × ($2.05 + $1.38 + $1.27 + $3.63 + $2.30) for the frontier runs, plus about $3.30 for the small ones. From 561924c4ab28a935a6409296cfbebb2e91b94578 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 08:47:24 +0000 Subject: [PATCH 12/23] guide: guessing a prop goes through the child's own equation; prereg 12 notes it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-12.md | 4 +--- guide/async.md | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eval/PREREG-desk-12.md b/eval/PREREG-desk-12.md index 7dfe8cc8..90c2cb7c 100644 --- a/eval/PREREG-desk-12.md +++ b/eval/PREREG-desk-12.md @@ -33,9 +33,7 @@ Ryan, 2026-11-11: replication now, maintenance next. Maintenance moves to round The guide follows all four. `rxc` is built from the commit that holds this file once it is frozen. Round 11's .rx desks use `optimistic`, so the gate below grades them with round 11's `rxc`, which isolates the Solid change. -**Guide additions** (principle only, no task code): -- **Guess over an equation:** `async.md` shows it. -- **Held writes:** the section "What lands together, and what waits" now says that a write after `await` in one action waits for any other action guessing on the same source. +**Guide addition** (principle only, no task code): `async.md` says how to guess a prop. You guess over the child's own equation of it, never `props.x`. **Solid:** `solid-next` at `next`, at a commit holding the fix for both cases of solidjs/solid#4762, built from source. TBD: 0a11dbb8 fixes the first case; the second, which is two guesses with one landing after a switch, is reported on the issue. diff --git a/guide/async.md b/guide/async.md index c5525ccc..28e68002 100644 --- a/guide/async.md +++ b/guide/async.md @@ -91,6 +91,9 @@ by what the screen should do: | Record the server's answer | `x := answer` after the `await` | | Handle a failure | `try { … } catch (e) { … }`: the catch's writes and the revert are one frame (AC-7) | +To guess a value a component gets as a prop, guess its own equation of it: `shown = props.title`, then +`guess shown := t` in the action. A prop itself is read-only (O-4), and the guess shows wherever the child reads `shown`. + ### Writing one row of a list When several actions can each change a different row of one list, guess the row, not the list: From 63b2cfc219c73755036ad596642e2f6cae761d30 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 10:59:03 +0000 Subject: [PATCH 13/23] Round 12 prereg: pin Solid next bcef1041, record the gate Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-12.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eval/PREREG-desk-12.md b/eval/PREREG-desk-12.md index 90c2cb7c..b88fd57d 100644 --- a/eval/PREREG-desk-12.md +++ b/eval/PREREG-desk-12.md @@ -35,7 +35,7 @@ The guide follows all four. `rxc` is built from the commit that holds this file **Guide addition** (principle only, no task code): `async.md` says how to guess a prop. You guess over the child's own equation of it, never `props.x`. -**Solid:** `solid-next` at `next`, at a commit holding the fix for both cases of solidjs/solid#4762, built from source. TBD: 0a11dbb8 fixes the first case; the second, which is two guesses with one landing after a switch, is reported on the issue. +**Solid:** `solid-next` at `next` bcef1041, built from source. It holds the fixes for both cases of solidjs/solid#4762: cf32a50b (a keyed projection of a projection losing a row on a switch) and #4769 (two guesses, one landing after the switch, now hold together as D-33 says). **Harness:** subagents, as in round 11. @@ -68,6 +68,8 @@ The direction is the claim being tested. A null or reversed result is reported a Round 11's nine .rx desks are regraded on the pinned Solid, with round 11's `rxc`. Any desk that gains a hole is traced. A Solid regression is filed upstream, then fixed or recorded, before the round starts. +**Gate result** (2026-11-11, on `next` bcef1041's source, round 11's `rxc`): 26 defects across the nine desks, against 29 in round 11. Only rx-2 changed, from 5 to 2: its three fuzz holes (seeds 1, 3, 18, solidjs/solid#4762) are gone, and its two overlap failures, which are its own (a pending source in a component that remounts), remain. The other eight desks grade exactly as in round 11. No desk gained a hole, so nothing needed tracing. + ## Cost About $67 at round 11's prices: 6 × ($2.05 + $1.38 + $1.27 + $3.63 + $2.30) for the frontier runs, plus about $3.30 for the small ones. From 82c6e42a3e33d4d567982a91d9e0b2834e0ebba2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 15:56:02 +0000 Subject: [PATCH 14/23] Round 12 prereg: frozen Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-desk-12.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eval/PREREG-desk-12.md b/eval/PREREG-desk-12.md index b88fd57d..6085b981 100644 --- a/eval/PREREG-desk-12.md +++ b/eval/PREREG-desk-12.md @@ -1,6 +1,6 @@ # Round 12: round 11 again with five arms and twice the frontier runs -Pre-registration, committed before any round 12 agent runs. Status: DRAFT. +Pre-registration, committed before any round 12 agent runs. Status: FROZEN (Ryan, 2026-11-11 "Go"). ## Why From fe0a0e005b5b50f398e3ef552e573b1b3c5de6e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 22:35:09 +0000 Subject: [PATCH 15/23] eval: round 12 results (graded, stats, costs, first traces) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/results/desk12-2026-11-11/README.md | 145 ++++ eval/results/desk12-2026-11-11/agents-12.txt | 60 ++ eval/results/desk12-2026-11-11/cost.json | 733 ++++++++++++++++++ .../graded/desk12-foldkit-1.txt | 10 + .../graded/desk12-foldkit-10.txt | 20 + .../graded/desk12-foldkit-11.txt | 26 + .../graded/desk12-foldkit-12.txt | 19 + .../graded/desk12-foldkit-2.txt | 8 + .../graded/desk12-foldkit-3.txt | 11 + .../graded/desk12-foldkit-4.txt | 10 + .../graded/desk12-foldkit-5.txt | 8 + .../graded/desk12-foldkit-6.txt | 10 + .../graded/desk12-foldkit-7.txt | 24 + .../graded/desk12-foldkit-8.txt | 19 + .../graded/desk12-foldkit-9.txt | 21 + .../graded/desk12-react-1.txt | 10 + .../graded/desk12-react-10.txt | 16 + .../graded/desk12-react-11.txt | 20 + .../graded/desk12-react-12.txt | 25 + .../graded/desk12-react-2.txt | 6 + .../graded/desk12-react-3.txt | 8 + .../graded/desk12-react-4.txt | 10 + .../graded/desk12-react-5.txt | 8 + .../graded/desk12-react-6.txt | 8 + .../graded/desk12-react-7.txt | 26 + .../graded/desk12-react-8.txt | 21 + .../graded/desk12-react-9.txt | 13 + .../desk12-2026-11-11/graded/desk12-rx-1.txt | 8 + .../desk12-2026-11-11/graded/desk12-rx-10.txt | 10 + .../desk12-2026-11-11/graded/desk12-rx-11.txt | 27 + .../desk12-2026-11-11/graded/desk12-rx-12.txt | 9 + .../desk12-2026-11-11/graded/desk12-rx-2.txt | 20 + .../desk12-2026-11-11/graded/desk12-rx-3.txt | 14 + .../desk12-2026-11-11/graded/desk12-rx-4.txt | 6 + .../desk12-2026-11-11/graded/desk12-rx-5.txt | 8 + .../desk12-2026-11-11/graded/desk12-rx-6.txt | 8 + .../desk12-2026-11-11/graded/desk12-rx-7.txt | 11 + .../desk12-2026-11-11/graded/desk12-rx-8.txt | 11 + .../desk12-2026-11-11/graded/desk12-rx-9.txt | 17 + .../graded/desk12-vanilla-1.txt | 10 + .../graded/desk12-vanilla-10.txt | 21 + .../graded/desk12-vanilla-11.txt | 23 + .../graded/desk12-vanilla-12.txt | 18 + .../graded/desk12-vanilla-2.txt | 8 + .../graded/desk12-vanilla-3.txt | 12 + .../graded/desk12-vanilla-4.txt | 10 + .../graded/desk12-vanilla-5.txt | 8 + .../graded/desk12-vanilla-6.txt | 8 + .../graded/desk12-vanilla-7.txt | 14 + .../graded/desk12-vanilla-8.txt | 29 + .../graded/desk12-vanilla-9.txt | 22 + .../graded/desk12-xstate-1.txt | 10 + .../graded/desk12-xstate-10.txt | 7 + .../graded/desk12-xstate-11.txt | 12 + .../graded/desk12-xstate-12.txt | 28 + .../graded/desk12-xstate-2.txt | 8 + .../graded/desk12-xstate-3.txt | 7 + .../graded/desk12-xstate-4.txt | 8 + .../graded/desk12-xstate-5.txt | 8 + .../graded/desk12-xstate-6.txt | 10 + .../graded/desk12-xstate-7.txt | 28 + .../graded/desk12-xstate-8.txt | 30 + .../graded/desk12-xstate-9.txt | 28 + .../repro/two-close-guesses.spec.tsx | 39 + eval/results/desk12-2026-11-11/stats.py | 39 + eval/results/desk12-2026-11-11/stats.txt | 16 + eval/results/desk12-2026-11-11/summary.txt | 60 ++ eval/results/desk12-2026-11-11/usd.json | 1 + 68 files changed, 1966 insertions(+) create mode 100644 eval/results/desk12-2026-11-11/README.md create mode 100644 eval/results/desk12-2026-11-11/agents-12.txt create mode 100644 eval/results/desk12-2026-11-11/cost.json create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-1.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-10.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-11.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-12.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-2.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-3.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-4.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-5.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-6.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-7.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-8.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-foldkit-9.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-1.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-10.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-11.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-12.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-2.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-3.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-4.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-5.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-6.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-7.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-8.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-react-9.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-1.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-10.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-11.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-12.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-2.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-3.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-4.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-5.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-6.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-7.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-8.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-rx-9.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-1.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-10.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-11.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-12.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-2.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-3.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-4.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-5.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-6.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-7.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-8.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-vanilla-9.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-1.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-10.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-11.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-12.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-2.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-3.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-4.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-5.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-6.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-7.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-8.txt create mode 100644 eval/results/desk12-2026-11-11/graded/desk12-xstate-9.txt create mode 100644 eval/results/desk12-2026-11-11/repro/two-close-guesses.spec.tsx create mode 100644 eval/results/desk12-2026-11-11/stats.py create mode 100644 eval/results/desk12-2026-11-11/stats.txt create mode 100644 eval/results/desk12-2026-11-11/summary.txt create mode 100644 eval/results/desk12-2026-11-11/usd.json diff --git a/eval/results/desk12-2026-11-11/README.md b/eval/results/desk12-2026-11-11/README.md new file mode 100644 index 00000000..590c8093 --- /dev/null +++ b/eval/results/desk12-2026-11-11/README.md @@ -0,0 +1,145 @@ +# Round 12 results: round 11 again, five arms, six runs per tier (2026-11-11) + +**Pre-registration:** `eval/PREREG-desk-12.md`, frozen at 82c6e42. + +**Runs:** 60, which is 5 arms × (6 frontier + 6 small), in one interleaved batch. Desks 1–6 are the frontier tier (Opus), desks 7–12 the small tier (Haiku). + +**Toolchain:** +- Solid `next` bcef1041 (source identical to 0739445a), built from source. +- rxc from 82c6e42. + +**Grading:** `harness/grade.sh`, with hidden 77, overlap 38 and 20 fuzz seeds. +- Defects are failed hidden tests, plus failed overlap tests, plus fuzz seeds that found a hole. +- Only defects are counted. +- No fuzz hole has been traced to a legal hold yet, so nothing is excluded. The traces so far are below. + +## Defects per app + +| arm | frontier mean | frontier per desk | small mean | small per desk | small median | +|---|---|---|---|---|---| +| .rx (inline docs) | 3.2 | 1, 10, 4, 0, 2, 2 | **6.0** | 3, 5, 9, 2, 15, 2 | 4 | +| React | 1.8 | 3, 0, 1, 3, 2, 2 | 11.3 | 16, 12, 6, 8, 11, 15 | 11.5 | +| plain TypeScript | 2.7 | 3, 2, 4, 3, 2, 2 | 12.3 | 7, 18, 13, 12, 14, 10 | 12.5 | +| Foldkit (inline docs) | 2.8 | 3, 2, 4, 3, 2, 3 | 13.2 | 15, 12, 13, 12, 16, 11 | 12.5 | +| XState (inline docs) | 2.0 | 3, 2, 1, 1, 2, 3 | 13.0 | 18, 19, 18, 1, 5, 17 | 17.5 | + +## Tests + +All tests use seed 12. Full output is in `stats.txt`. + +| test | result | verdict | +|---|---|---| +| **H12a** (primary): small .rx < small React | p = 0.046, Holm 0.091 | **Not supported** | +| **H12b**: small .rx < small Foldkit | p = 0.011, Holm 0.043 | **Supported** | +| **H12b**: small .rx < small plain | p = 0.023, Holm 0.068 | **Not supported** | +| **H12b**: small .rx < small XState | p = 0.056, Holm 0.091 | **Not supported** | +| **H12c**: frontier .rx no worse than frontier React | difference +1.33, 90% interval [−0.83, 3.83] | **Not shown.** The upper end is not under 1. | +| **H12d**: small XState < small Foldkit | p = 0.50 | **Null** | + +H12a and H12b are Holm-corrected together, as pre-registered. H12a's raw p is about the same as round 11's (0.047). The difference is that round 12 pre-registered three more comparisons in the same family, so the correction is stricter. + +## Change from round 11 + +| arm, tier | round 11 | round 12 | +|---|---|---| +| .rx frontier | 2.0 | 3.2 | +| .rx small | 3.8 | 6.0 | +| React frontier | 2.0 | 1.8 | +| React small | 9.7 | 11.3 | +| plain frontier | 3.0 | 2.7 | +| plain small | 14.5 | 12.3 | +| Foldkit frontier | 2.7 | 2.8 | +| Foldkit small | 13.2 | 13.2 | +| XState frontier | 2.0 (side check) | 2.0 | +| XState small | 7.0 (side check) | 13.0 | + +## Correct apps and cost + +**Correct apps:** two have no defect, frontier `rx-4` and frontier `react-2`. + +**Cost per correct app:** +- frontier .rx: $14.29 (six runs over one correct app); +- frontier React: $7.85. + +Every other cell has no correct app. + +Cost per run is billed tokens (`cost.json`) at round 10's rates (`usd.json`): + +| arm | frontier $/run | small $/run | frontier calls | small calls | lines (frontier / small) | +|---|---|---|---|---|---| +| .rx | 2.38 | 0.095 | 25.2 | 17.5 | 431 / 370 | +| React | 1.31 | 0.064 | 14.5 | 11.2 | 857 / 736 | +| plain | 1.19 | 0.073 | 12.3 | 8.2 | 918 / 1000 | +| Foldkit | 3.29 | 0.169 | 35.7 | 41.5 | 2355 / 2113 | +| XState | 2.49 | 0.137 | 26.5 | 26.5 | 1388 / 1296 | + +Lines are non-blank source lines, without the given `api` and `flights` files. + +## Defects by kind + +- **Overlap tests:** the four non-.rx small arms average 8.3 to 10.8 failed overlap tests per app; small .rx averages 2.5. "Comments / labels that load while an add is out keep it" still fails in most desks of every arm, including rx-2, rx-5 and rx-6. +- **Fuzz holes:** .rx holes, by kind: + +| kind | holes | desks | +|---|---|---| +| detail title | 6 | rx-2, rx-3, rx-9, rx-10 | +| closed row shown | 4 | rx-1, rx-2, rx-3, rx-7 | +| tab count out of step with the rows | 4 | rx-3 (×2), rx-10, rx-12 | +| row assignee | 4 | all rx-11 | +| dropped guess | 2 | rx-7, rx-11 | +| row title | 1 | rx-2 | +| labels at rest | 1 | rx-11 | + +- **Small-tier outliers:** XState is bimodal (1, 5 against 17–19). In the high desks, most defects are overlap: late loads drop changes still in flight. + +### .rx traces so far + +- **rx-1, seed 15 ("closed row shown"): a Solid bug.** + - The app guesses two closes on a `createOptimistic` list of hidden ids and an assign on the optimistic issues store. + - When the first close lands, its `refresh` drops both close guesses while the refresh is still out. Under D-33 and F-11, they should hold until it answers. + - The plain-Solid repro is `repro/two-close-guesses.spec.tsx`. It passes without the assign guess. + - It also fails on round 11's Solid (6751472f), so it is not caused by #4769. + - Not filed yet. +- **rx-7, seed 12 ("closed row shown"): an agent mistake, most likely.** + - After its await, each close writes `closed := [...closed, r.id]`. With two closes out, the second write reads `closed` without the first close's id, so an accepted close comes back. + - The guide's functional form (`closed := c => [...c, r.id]`) avoids this. + - AC-1e only covers a read made before the await, so no check fired. +- **rx-2, seed 13 ("closed row shown"): not yet classified.** + - The app keeps two lists and chooses between them with `filter == "open" ? openIssues : closedIssues`. + - After a failed open load, an accepted assign on the closed list and a switch to open, the open view shows the closed list's rows, still at rest after the open list lands. + - This is the shape of #4762 (a keyed derivation over a choice of projections). It needs a plain-Solid repro. +- **The tab-count holes (rx-3, rx-10, rx-12):** each app passes the count to the app from an `effect` inside the list's loading region. While a row guess is out, the tab shows the count from before the guess. These are a tear between two views, not a hold, so they count. +- **The rest have not been traced.** That covers detail title, row assignee and dropped guess. + +## Deviations + +- **Temporary visible tests, restored byte for byte (checked):** rx-1, rx-2, rx-3, rx-5, rx-6, xstate-1, xstate-2, react-11, react-12. +- **Scratch files written to `/tmp` and then deleted:** react-7, foldkit-9. +- **foldkit-9 posted one stray message to the project thread.** It has been edited to say so. +- **Ran `tsc` outside the check/test budget:** foldkit-7, foldkit-12. +- **Listed file names in `eval/harness` (read no contents):** foldkit-7. +- **Scratch scripts or simulations inside the run dir, deleted:** xstate-7, xstate-9, xstate-11, xstate-12. +- **rx-9 ended at 29 of 30 visible tests:** a related hit never opens the help page. It is graded as it stands. +- **A machine restart interrupted grading twice.** The interrupted desks were regraded from the start, and no result mixes two runs. +- None of these affects grading. + +## Recurring .rx traps reported by the agents (guide candidates, not checks) + +- **A filter switch held by a row guess (rx-2, rx-3, rx-5):** with one list whose question reads the filter, a guess on it holds the filter switch. All three split the list per filter. +- **A hold at mount (rx-4, rx-8, rx-9, rx-12):** `pending?` or a mount-time read outside a loading region holds the tab switch. rx-9 never solved it. + +## Reading + +- **Small tier:** .rx still has the fewest defects, about half of React's, plain's, XState's and Foldkit's. With the pre-registered Holm family, only the Foldkit comparison clears 0.05, so the primary hypothesis is not supported this round. +- **Frontier tier:** .rx is the worst arm, at 3.2, though its median is 2. One desk (rx-2, 10 defects) accounts for much of that, and one of rx-1's holes is a Solid bug. React is best at 1.8. +- **XState:** level with React at the frontier. In the small tier it lost the side check's advantage (7.0 → 13.0) and is now level with Foldkit and plain. +- **Cost:** frontier .rx costs $2.38 per run, against $1.31 for React. That is up from round 11's $2.05. + +## Files + +- `summary.txt`: per desk. +- `graded/`: `grade.sh` output. +- `stats.py`, `stats.txt`. +- `cost.json`, `usd.json`. +- `agents-12.txt`: desk to transcript. +- `repro/`: the plain-Solid repro. diff --git a/eval/results/desk12-2026-11-11/agents-12.txt b/eval/results/desk12-2026-11-11/agents-12.txt new file mode 100644 index 00000000..2e7363c7 --- /dev/null +++ b/eval/results/desk12-2026-11-11/agents-12.txt @@ -0,0 +1,60 @@ +desk12-rx-1 a2adcc860413e93be +desk12-foldkit-1 a139926c988c12fa7 +desk12-react-1 a34fae9b65175a351 +desk12-vanilla-1 a3fe146269a48f327 +desk12-xstate-1 a4e38081558642822 +desk12-rx-7 a1a2fec31d1b8a77e +desk12-foldkit-7 a1cada09a3a45236b +desk12-react-7 a7bbe53d54a2bce3e +desk12-vanilla-7 ab8e8448de3a44a7a +desk12-xstate-7 aea72cfdb2062a71b +desk12-rx-2 aeb6d74755925b637 +desk12-foldkit-2 af7c64b4b83886410 +desk12-react-2 a0055201cb80640ed +desk12-vanilla-2 a7c7382a8117a3304 +desk12-xstate-2 ae0c06da1dfdb5c25 +desk12-rx-8 ac2a3ecdef7d9eb7e +desk12-foldkit-8 a118273811805976b +desk12-react-8 abe15c6315c2be8c3 +desk12-vanilla-8 a58410988f54ff148 +desk12-xstate-8 a94f397ea79b75bbf +desk12-rx-3 a0559f280b6399d43 +desk12-foldkit-3 aeaaaf05d03a789fc +desk12-react-3 aa321e6cbbd3be969 +desk12-vanilla-3 aac54043cc1125c25 +desk12-xstate-3 affb8ad029601946b +desk12-rx-9 aa329eef1001465b3 +desk12-foldkit-9 ac2c7d376ad5ad545 +desk12-react-9 ae521de90f5b7f35a +desk12-vanilla-9 a25ab8e6647b18e52 +desk12-xstate-9 a5da6342624907ee8 +desk12-rx-4 ac3350f0df75b6aeb +desk12-foldkit-4 a62b40df39af2a356 +desk12-react-4 afa956dff110d1ef4 +desk12-vanilla-4 ad4c9fb68b3c66e28 +desk12-xstate-4 a50c61cf3e926fedb +desk12-rx-10 a668866d052f45159 +desk12-foldkit-10 a8a531db059da8512 +desk12-react-10 a18751ce352dbcefe +desk12-vanilla-10 a14177a5f6843636a +desk12-xstate-10 a5c46d6a332c2a3cc +desk12-rx-5 a4bda814948c8cef4 +desk12-foldkit-5 a072930b5ee5d5ddd +desk12-react-5 aa26e94820fbcd504 +desk12-vanilla-5 a6ab112dab302338b +desk12-xstate-5 a5c96cc781d53458d +desk12-rx-11 ad4f28e198cbf742c +desk12-foldkit-11 a8d95ebe8b22826d3 +desk12-react-11 a77cf07d01b52dacc +desk12-vanilla-11 aa70ee82b32e4b49e +desk12-xstate-11 a86ccff47c7117834 +desk12-rx-6 add471b87949b59da +desk12-foldkit-6 a527785c5df3f9938 +desk12-react-6 a243b640faeee847d +desk12-vanilla-6 ae38f3b1526ae7d8d +desk12-xstate-6 a9bf2a63511c7fac6 +desk12-rx-12 a68f68bd9daa5f0ca +desk12-foldkit-12 a3dbb3a0ece54aff9 +desk12-react-12 a0dd5f8ce8884fa34 +desk12-vanilla-12 a7f4dd1341822892d +desk12-xstate-12 abd05c774a847eff7 diff --git a/eval/results/desk12-2026-11-11/cost.json b/eval/results/desk12-2026-11-11/cost.json new file mode 100644 index 00000000..3b1a5155 --- /dev/null +++ b/eval/results/desk12-2026-11-11/cost.json @@ -0,0 +1,733 @@ +{ + "weights": { + "input": 1, + "cacheWrite": 1.25, + "cacheRead": 0.1, + "output": 5 + }, + "runs": [ + { + "run": "desk12-rx-1", + "task": "desk12", + "arm": "rx", + "n": 1, + "input": 36, + "cacheWrite": 111782, + "cacheRead": 1535886, + "output": 32102, + "calls": 18, + "billed": 453862.1 + }, + { + "run": "desk12-foldkit-1", + "task": "desk12", + "arm": "foldkit", + "n": 1, + "input": 66, + "cacheWrite": 145625, + "cacheRead": 3126364, + "output": 62092, + "calls": 33, + "billed": 805193.65 + }, + { + "run": "desk12-react-1", + "task": "desk12", + "arm": "react", + "n": 1, + "input": 32, + "cacheWrite": 81477, + "cacheRead": 987557, + "output": 28800, + "calls": 16, + "billed": 344633.95 + }, + { + "run": "desk12-vanilla-1", + "task": "desk12", + "arm": "vanilla", + "n": 1, + "input": 30, + "cacheWrite": 88321, + "cacheRead": 971513, + "output": 35635, + "calls": 15, + "billed": 385757.55 + }, + { + "run": "desk12-xstate-1", + "task": "desk12", + "arm": "xstate", + "n": 1, + "input": 54, + "cacheWrite": 130052, + "cacheRead": 2609007, + "output": 61966, + "calls": 27, + "billed": 733349.7 + }, + { + "run": "desk12-rx-7", + "task": "desk12", + "arm": "rx", + "n": 7, + "input": 28, + "cacheWrite": 204874, + "cacheRead": 1424741, + "output": 98603, + "calls": 14, + "billed": 891609.6 + }, + { + "run": "desk12-foldkit-7", + "task": "desk12", + "arm": "foldkit", + "n": 7, + "input": 86, + "cacheWrite": 279518, + "cacheRead": 6220729, + "output": 163192, + "calls": 43, + "billed": 1787516.4 + }, + { + "run": "desk12-react-7", + "task": "desk12", + "arm": "react", + "n": 7, + "input": 30, + "cacheWrite": 126438, + "cacheRead": 1289461, + "output": 72552, + "calls": 15, + "billed": 649783.6 + }, + { + "run": "desk12-vanilla-7", + "task": "desk12", + "arm": "vanilla", + "n": 7, + "input": 18, + "cacheWrite": 182266, + "cacheRead": 754373, + "output": 106896, + "calls": 9, + "billed": 837767.8 + }, + { + "run": "desk12-xstate-7", + "task": "desk12", + "arm": "xstate", + "n": 7, + "input": 94, + "cacheWrite": 246043, + "cacheRead": 8196570, + "output": 156921, + "calls": 47, + "billed": 1911909.75 + }, + { + "run": "desk12-rx-2", + "task": "desk12", + "arm": "rx", + "n": 2, + "input": 60, + "cacheWrite": 129992, + "cacheRead": 2956711, + "output": 44047, + "calls": 30, + "billed": 678456.1000000001 + }, + { + "run": "desk12-foldkit-2", + "task": "desk12", + "arm": "foldkit", + "n": 2, + "input": 66, + "cacheWrite": 144902, + "cacheRead": 2983317, + "output": 63995, + "calls": 33, + "billed": 799500.2 + }, + { + "run": "desk12-react-2", + "task": "desk12", + "arm": "react", + "n": 2, + "input": 30, + "cacheWrite": 86281, + "cacheRead": 949412, + "output": 32199, + "calls": 15, + "billed": 363817.45 + }, + { + "run": "desk12-vanilla-2", + "task": "desk12", + "arm": "vanilla", + "n": 2, + "input": 26, + "cacheWrite": 81660, + "cacheRead": 767702, + "output": 28248, + "calls": 13, + "billed": 320111.2 + }, + { + "run": "desk12-xstate-2", + "task": "desk12", + "arm": "xstate", + "n": 2, + "input": 54, + "cacheWrite": 121006, + "cacheRead": 2403725, + "output": 51972, + "calls": 27, + "billed": 651544 + }, + { + "run": "desk12-rx-8", + "task": "desk12", + "arm": "rx", + "n": 8, + "input": 46, + "cacheWrite": 198415, + "cacheRead": 2987901, + "output": 126008, + "calls": 23, + "billed": 1176894.85 + }, + { + "run": "desk12-foldkit-8", + "task": "desk12", + "arm": "foldkit", + "n": 8, + "input": 124, + "cacheWrite": 295555, + "cacheRead": 10333210, + "output": 184349, + "calls": 62, + "billed": 2324633.75 + }, + { + "run": "desk12-react-8", + "task": "desk12", + "arm": "react", + "n": 8, + "input": 22, + "cacheWrite": 123959, + "cacheRead": 759288, + "output": 68764, + "calls": 11, + "billed": 574719.55 + }, + { + "run": "desk12-vanilla-8", + "task": "desk12", + "arm": "vanilla", + "n": 8, + "input": 14, + "cacheWrite": 185108, + "cacheRead": 470624, + "output": 107368, + "calls": 7, + "billed": 815301.4 + }, + { + "run": "desk12-xstate-8", + "task": "desk12", + "arm": "xstate", + "n": 8, + "input": 28, + "cacheWrite": 156652, + "cacheRead": 1366231, + "output": 95697, + "calls": 14, + "billed": 810951.1 + }, + { + "run": "desk12-rx-3", + "task": "desk12", + "arm": "rx", + "n": 3, + "input": 68, + "cacheWrite": 122334, + "cacheRead": 3781160, + "output": 52688, + "calls": 34, + "billed": 794541.5 + }, + { + "run": "desk12-foldkit-3", + "task": "desk12", + "arm": "foldkit", + "n": 3, + "input": 64, + "cacheWrite": 122311, + "cacheRead": 3119895, + "output": 57228, + "calls": 32, + "billed": 751082.25 + }, + { + "run": "desk12-react-3", + "task": "desk12", + "arm": "react", + "n": 3, + "input": 28, + "cacheWrite": 57513, + "cacheRead": 904710, + "output": 31995, + "calls": 14, + "billed": 322365.25 + }, + { + "run": "desk12-vanilla-3", + "task": "desk12", + "arm": "vanilla", + "n": 3, + "input": 18, + "cacheWrite": 52189, + "cacheRead": 504720, + "output": 27538, + "calls": 9, + "billed": 253416.25 + }, + { + "run": "desk12-xstate-3", + "task": "desk12", + "arm": "xstate", + "n": 3, + "input": 46, + "cacheWrite": 90390, + "cacheRead": 1973842, + "output": 53571, + "calls": 23, + "billed": 578272.7 + }, + { + "run": "desk12-rx-9", + "task": "desk12", + "arm": "rx", + "n": 9, + "input": 40, + "cacheWrite": 161078, + "cacheRead": 2307802, + "output": 113888, + "calls": 20, + "billed": 1001607.7 + }, + { + "run": "desk12-foldkit-9", + "task": "desk12", + "arm": "foldkit", + "n": 9, + "input": 68, + "cacheWrite": 208075, + "cacheRead": 4867095, + "output": 142588, + "calls": 34, + "billed": 1459811.25 + }, + { + "run": "desk12-react-9", + "task": "desk12", + "arm": "react", + "n": 9, + "input": 18, + "cacheWrite": 129682, + "cacheRead": 535259, + "output": 79881, + "calls": 9, + "billed": 615051.4 + }, + { + "run": "desk12-vanilla-9", + "task": "desk12", + "arm": "vanilla", + "n": 9, + "input": 18, + "cacheWrite": 144080, + "cacheRead": 745247, + "output": 96320, + "calls": 9, + "billed": 736242.7 + }, + { + "run": "desk12-xstate-9", + "task": "desk12", + "arm": "xstate", + "n": 9, + "input": 64, + "cacheWrite": 306521, + "cacheRead": 5685124, + "output": 175356, + "calls": 32, + "billed": 1828507.65 + }, + { + "run": "desk12-rx-4", + "task": "desk12", + "arm": "rx", + "n": 4, + "input": 34, + "cacheWrite": 72351, + "cacheRead": 1319097, + "output": 33516, + "calls": 17, + "billed": 389962.45 + }, + { + "run": "desk12-foldkit-4", + "task": "desk12", + "arm": "foldkit", + "n": 4, + "input": 70, + "cacheWrite": 125165, + "cacheRead": 3537625, + "output": 59319, + "calls": 35, + "billed": 806883.75 + }, + { + "run": "desk12-react-4", + "task": "desk12", + "arm": "react", + "n": 4, + "input": 30, + "cacheWrite": 57752, + "cacheRead": 955274, + "output": 31609, + "calls": 15, + "billed": 325792.4 + }, + { + "run": "desk12-vanilla-4", + "task": "desk12", + "arm": "vanilla", + "n": 4, + "input": 22, + "cacheWrite": 48707, + "cacheRead": 615320, + "output": 24211, + "calls": 11, + "billed": 243492.75 + }, + { + "run": "desk12-xstate-4", + "task": "desk12", + "arm": "xstate", + "n": 4, + "input": 56, + "cacheWrite": 87552, + "cacheRead": 2488186, + "output": 47315, + "calls": 28, + "billed": 594889.6 + }, + { + "run": "desk12-rx-10", + "task": "desk12", + "arm": "rx", + "n": 10, + "input": 32, + "cacheWrite": 133347, + "cacheRead": 1768004, + "output": 93753, + "calls": 16, + "billed": 812281.15 + }, + { + "run": "desk12-foldkit-10", + "task": "desk12", + "arm": "foldkit", + "n": 10, + "input": 78, + "cacheWrite": 212658, + "cacheRead": 4832273, + "output": 155865, + "calls": 39, + "billed": 1528452.8 + }, + { + "run": "desk12-react-10", + "task": "desk12", + "arm": "react", + "n": 10, + "input": 14, + "cacheWrite": 136140, + "cacheRead": 456511, + "output": 84795, + "calls": 7, + "billed": 639815.1 + }, + { + "run": "desk12-vanilla-10", + "task": "desk12", + "arm": "vanilla", + "n": 10, + "input": 14, + "cacheWrite": 104782, + "cacheRead": 469192, + "output": 79922, + "calls": 7, + "billed": 577520.7 + }, + { + "run": "desk12-xstate-10", + "task": "desk12", + "arm": "xstate", + "n": 10, + "input": 36, + "cacheWrite": 181392, + "cacheRead": 2214381, + "output": 139523, + "calls": 18, + "billed": 1145829.1 + }, + { + "run": "desk12-rx-5", + "task": "desk12", + "arm": "rx", + "n": 5, + "input": 58, + "cacheWrite": 118846, + "cacheRead": 3080349, + "output": 54445, + "calls": 29, + "billed": 728875.4 + }, + { + "run": "desk12-foldkit-5", + "task": "desk12", + "arm": "foldkit", + "n": 5, + "input": 84, + "cacheWrite": 130847, + "cacheRead": 4285215, + "output": 61954, + "calls": 42, + "billed": 901934.25 + }, + { + "run": "desk12-react-5", + "task": "desk12", + "arm": "react", + "n": 5, + "input": 28, + "cacheWrite": 54699, + "cacheRead": 878192, + "output": 29518, + "calls": 14, + "billed": 303810.95 + }, + { + "run": "desk12-vanilla-5", + "task": "desk12", + "arm": "vanilla", + "n": 5, + "input": 24, + "cacheWrite": 52353, + "cacheRead": 704634, + "output": 26828, + "calls": 12, + "billed": 270068.65 + }, + { + "run": "desk12-xstate-5", + "task": "desk12", + "arm": "xstate", + "n": 5, + "input": 50, + "cacheWrite": 81147, + "cacheRead": 2073044, + "output": 46338, + "calls": 25, + "billed": 540478.15 + }, + { + "run": "desk12-rx-11", + "task": "desk12", + "arm": "rx", + "n": 11, + "input": 30, + "cacheWrite": 194289, + "cacheRead": 1736541, + "output": 110711, + "calls": 15, + "billed": 970100.35 + }, + { + "run": "desk12-foldkit-11", + "task": "desk12", + "arm": "foldkit", + "n": 11, + "input": 60, + "cacheWrite": 235043, + "cacheRead": 4645895, + "output": 177656, + "calls": 30, + "billed": 1646733.25 + }, + { + "run": "desk12-react-11", + "task": "desk12", + "arm": "react", + "n": 11, + "input": 24, + "cacheWrite": 116341, + "cacheRead": 1092237, + "output": 88636, + "calls": 12, + "billed": 697853.95 + }, + { + "run": "desk12-vanilla-11", + "task": "desk12", + "arm": "vanilla", + "n": 11, + "input": 14, + "cacheWrite": 135642, + "cacheRead": 456472, + "output": 86738, + "calls": 7, + "billed": 648903.7 + }, + { + "run": "desk12-xstate-11", + "task": "desk12", + "arm": "xstate", + "n": 11, + "input": 46, + "cacheWrite": 234320, + "cacheRead": 3015494, + "output": 143895, + "calls": 23, + "billed": 1313970.4 + }, + { + "run": "desk12-rx-6", + "task": "desk12", + "arm": "rx", + "n": 6, + "input": 46, + "cacheWrite": 98331, + "cacheRead": 2108039, + "output": 38578, + "calls": 23, + "billed": 526653.65 + }, + { + "run": "desk12-foldkit-6", + "task": "desk12", + "arm": "foldkit", + "n": 6, + "input": 78, + "cacheWrite": 136075, + "cacheRead": 4162496, + "output": 56274, + "calls": 39, + "billed": 867791.3500000001 + }, + { + "run": "desk12-react-6", + "task": "desk12", + "arm": "react", + "n": 6, + "input": 26, + "cacheWrite": 55731, + "cacheRead": 804321, + "output": 30157, + "calls": 13, + "billed": 300906.85 + }, + { + "run": "desk12-vanilla-6", + "task": "desk12", + "arm": "vanilla", + "n": 6, + "input": 28, + "cacheWrite": 55362, + "cacheRead": 874689, + "output": 30548, + "calls": 14, + "billed": 309439.4 + }, + { + "run": "desk12-xstate-6", + "task": "desk12", + "arm": "xstate", + "n": 6, + "input": 58, + "cacheWrite": 93187, + "cacheRead": 2605904, + "output": 50524, + "calls": 29, + "billed": 629752.15 + }, + { + "run": "desk12-rx-12", + "task": "desk12", + "arm": "rx", + "n": 12, + "input": 34, + "cacheWrite": 136862, + "cacheRead": 1693903, + "output": 97148, + "calls": 17, + "billed": 826241.8 + }, + { + "run": "desk12-foldkit-12", + "task": "desk12", + "arm": "foldkit", + "n": 12, + "input": 82, + "cacheWrite": 187366, + "cacheRead": 5284252, + "output": 131529, + "calls": 41, + "billed": 1420359.7000000002 + }, + { + "run": "desk12-react-12", + "task": "desk12", + "arm": "react", + "n": 12, + "input": 26, + "cacheWrite": 113232, + "cacheRead": 1027119, + "output": 85810, + "calls": 13, + "billed": 673327.9 + }, + { + "run": "desk12-vanilla-12", + "task": "desk12", + "arm": "vanilla", + "n": 12, + "input": 20, + "cacheWrite": 143638, + "cacheRead": 888763, + "output": 95474, + "calls": 10, + "billed": 745813.8 + }, + { + "run": "desk12-xstate-12", + "task": "desk12", + "arm": "xstate", + "n": 12, + "input": 50, + "cacheWrite": 177895, + "cacheRead": 3455736, + "output": 132553, + "calls": 25, + "billed": 1230757.35 + } + ], + "diff": -286600.6916666669, + "ratio": 1.59175503905721, + "p": 0.998450015499845 +} diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-1.txt new file mode 100644 index 00000000..8c345411 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-1: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-10.txt new file mode 100644 index 00000000..b4b7eeaa --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-10.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk12-foldkit-10: 2/20 seeds found a hole + seed 3 step 35: count: tab Issues (2), 1 open rows shown + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 1 ; close ; add-label ui ; user guest ; tab help ; tab issues ; land removeLabel:4,bug ; fail as + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-11.txt new file mode 100644 index 00000000..c1ef612a --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-11.txt @@ -0,0 +1,26 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-foldkit-11: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 39: two views: row "#4 Typo in footer", detail "#4 Bravofooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-12.txt new file mode 100644 index 00000000..953973c7 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-12.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +29/38 passed +== fuzz +desk12-foldkit-12: 2/20 seeds found a hole + seed 9 step 40: row assignee: #4 shows null, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 33: detail title: "retry#3 Crash on savestack trace attachedassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-2.txt new file mode 100644 index 00000000..3a55c7bf --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-3.txt new file mode 100644 index 00000000..bc52abee --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-3.txt @@ -0,0 +1,11 @@ +== hidden +✗ base: settings: saving an invalid name asks nothing +76/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-3: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-4.txt new file mode 100644 index 00000000..d8014d67 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-4.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-4: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-5.txt new file mode 100644 index 00000000..8937e039 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-6.txt new file mode 100644 index 00000000..37fa968d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-6.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-foldkit-6: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-7.txt new file mode 100644 index 00000000..a6595775 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-7.txt @@ -0,0 +1,24 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-foldkit-7: 3/20 seeds found a hole + seed 3 step 35: count: tab Issues (3), 2 open rows shown + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug ; fail assign:4,bob ; land fetchComments:4 ; row-close 2 ; row-close 0 ; land addLabel:4,bug ; row-assign 0 ; remove-label 1 ; close ; add-label ui ; user guest ; tab help ; tab issues ; land assign:3,bob ; fail close + seed 4 step 24: count: tab Issues (3), 2 open rows shown + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-8.txt new file mode 100644 index 00000000..dddd6d21 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-8.txt @@ -0,0 +1,19 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-foldkit-8: 1/20 seeds found a hole + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-foldkit-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-9.txt new file mode 100644 index 00000000..b639133d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-foldkit-9.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-foldkit-9: 2/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 18 step 35: row assignee: #4 shows bob, server dana + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land fetchComments:4 ; land closeIssue:3 ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 ; fail closeIssue:1 ; rename Alpha ; close ; fail closeIssue:4 ; focus ; close ; focus ; row-assign 0 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-1.txt new file mode 100644 index 00000000..f022505d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-1: 1/20 seeds found a hole + seed 16 step 37: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-10.txt new file mode 100644 index 00000000..3c381073 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-10.txt @@ -0,0 +1,16 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +32/38 passed +== fuzz +desk12-react-10: 2/20 seeds found a hole + seed 2 step 37: at rest: comments: #2 shows [], server [guest: fixed? | guest: fixed?] + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-11.txt new file mode 100644 index 00000000..b5f6dbe5 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-11.txt @@ -0,0 +1,20 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-react-11: 3/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 3 step 39: at rest: labels: #4 shows [docs,p1], server [docs,bug,p1] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail search:Typo in footer ; land fetchLabels:4 ; row-close 2 ; row-close 0 ; land addLabel:4,bug ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land assign:3,bob ; fai + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-12.txt new file mode 100644 index 00000000..cf975039 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-12.txt @@ -0,0 +1,25 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-react-12: 4/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-2.txt new file mode 100644 index 00000000..5ac0761f --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-2.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-react-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-3.txt new file mode 100644 index 00000000..a0ef373e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-3.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-react-3: 1/20 seeds found a hole + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-4.txt new file mode 100644 index 00000000..a113d2b5 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-4.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-4: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-5.txt new file mode 100644 index 00000000..bbcb3194 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-6.txt new file mode 100644 index 00000000..fde3606a --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-6.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-react-6: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-7.txt new file mode 100644 index 00000000..105b7c74 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-7.txt @@ -0,0 +1,26 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-react-7: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 12 step 33: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Typo in footer ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land search:Alpha ; fail assign:4,guest ; assign ; land closeIssue + seed 15 step 37: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-8.txt new file mode 100644 index 00000000..69dcd6c3 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-8.txt @@ -0,0 +1,21 @@ +== hidden +✗ rows: row errors and detail errors leave each other alone +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-react-8: 3/20 seeds found a hole + seed 3 step 40: at rest: labels: #4 shows [docs,p1], server [docs] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; row-close 0 ; fail search:Typo in footer ; land fetchLabels:4 ; row-close 1 ; row-close 0 ; land closeIssue:4 ; close ; add-label p1 ; remove-label 0 ; remove-label 0 ; filter + seed 12 step 44: row assignee: #1 shows guest, server null + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; rename Alpha ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchComments:1 ; land fetchComments:4 ; row-close 0 ; rename Alpha ; (move failed: locator.click: Timeout 2000ms exceeded.) ; fail cl + seed 16 step 36: at rest: comments: #2 shows [guest: ping | guest: ping], server [guest: ping] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; row-assign 0 ; close ; post ping ; assign ; land addComment:2,guest,ping ; land fetchIssue:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; assign diff --git a/eval/results/desk12-2026-11-11/graded/desk12-react-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-react-9.txt new file mode 100644 index 00000000..dee0c45e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-react-9.txt @@ -0,0 +1,13 @@ +== hidden +77/77 passed +== overlap +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +33/38 passed +== fuzz +desk12-react-9: 1/20 seeds found a hole + seed 15 step 33: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-1.txt new file mode 100644 index 00000000..0e5dc065 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-1.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-1: 1/20 seeds found a hole + seed 15 step 31: closed row shown: #1 is closed on the server + land fetchIssue:0 ; user bob ; filter closed ; land fetchComments:0 ; land fetchIssues:open ; fail fetchLabels:0 ; focus ; filter open ; fail fetchIssues:closed ; user guest ; land fetchIssues:open ; row-close 1 ; land closeIssue:3 ; fail fetchIssues:open ; focus ; tab help ; tab issues ; focus ; land fetchLabels:0 ; land fetchComments:0 ; land fetchIssue:0 ; land fetchIssues:open ; tab help ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-10.txt new file mode 100644 index 00000000..2802bf1e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-10.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-10: 2/20 seeds found a hole + seed 1 step 17: count: tab Issues (2), 3 open rows shown + user bob ; focus ; land fetchIssues:open ; row-assign 2 ; row-close 0 ; row-assign 0 ; select #3 ; rename Alpha ; land assign:3,bob ; rename Alpha ; row-assign 1 ; land renameIssue:3,Alpha ; focus ; land fetchLabels:3 ; land fetchIssue:3 ; land fetchIssues:open ; fail closeIssue:1 + seed 15 step 24: detail title: "retry#2 Old buglong goneassign to guest rename closepostadd ", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-11.txt new file mode 100644 index 00000000..f02f9680 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-11.txt @@ -0,0 +1,27 @@ +== hidden +✗ base: an accepted post stays after switching issues and back +✗ base: settings: a failed save reverts the saved line, shows the error and keeps the draft +✗ base: settings: edits made while a save is in flight survive its failure +✗ base: settings: a failed save after a successful one reverts to the last save +73/77 passed +== overlap +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +33/38 passed +== fuzz +desk12-rx-11: 6/20 seeds found a hole + seed 2 step 15: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchLabels:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchComments:2 ; fail fetchIssue:2 ; land closeIssue:2 ; fail assign:2,guest + seed 3 step 37: row assignee: #3 shows bob, server null + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; post ping ; land fetchIssue:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; row-close 0 ; fail search:Typo in footer ; land fetchComments:4 ; row-close 1 ; row-assign 0 ; land closeIssue:1 ; close ; row-assign 0 ; row-close 0 ; post fixed? ; filter closed ; post ping ; assign ; land closeIssue:3 ; fail assign:3 + seed 9 step 40: row assignee: #4 shows guest, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 18 step 36: row assignee: #3 shows null, server bob + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchIssue:4 ; assign ; row-assign 0 ; row-close 1 ; assign ; rename Delta ; fail assign:4,guest ; fail closeIssue:4 ; land closeIssue:1 ; land renameIssue:4,Delta ; user bob ; row-assign 0 ; row-close 1 ; rename Delta ; assign ; row-assign 0 ; land fetchComments:4 ; rename Alpha ; rename Alpha ; rename Delta ; row-close 0 ; focus ; close + seed 19 step 17: dropped guess: #3 title: showed "Delta" while it was in flight, now "Alpha" + land fetchIssues:open ; user guest ; row-assign 1 ; row-close 2 ; land assign:3,guest ; fail closeIssue:4 ; select #3 ; post fixed? ; land fetchLabels:3 ; rename Alpha ; rename Delta ; row-assign 0 ; land assign:1,guest ; land fetchIssue:3 ; row-assign 1 ; land addComment:3,guest,fixed? ; land renameIssue:3,Alpha + seed 20 step 38: at rest: labels: #2 shows [p1,p1], server [p1] + filter closed ; filter open ; user guest ; filter closed ; user ann ; land fetchIssues:closed ; land fetchIssues:open ; select #2 ; land fetchLabels:2 ; filter open ; focus ; land fetchComments:2 ; land fetchIssue:2 ; fail fetchIssues:open ; close ; post ping ; add-label p1 ; land search:Old bug ; close ; land closeIssue:2 ; remove-label 1 ; close ; user ann ; assign ; land closeIssue:2 ; clos diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-12.txt new file mode 100644 index 00000000..961a6ab8 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-12.txt @@ -0,0 +1,9 @@ +== hidden +77/77 passed +== overlap +✗ a label added on one issue does not show on another +37/38 passed +== fuzz +desk12-rx-12: 1/20 seeds found a hole + seed 1 step 11: count: tab Issues (2), 1 open rows shown + user bob ; focus ; land fetchIssues:open ; row-assign 2 ; row-close 0 ; row-assign 0 ; select #3 ; row-assign 0 ; land assign:3,bob ; row-assign 0 ; row-close 1 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-2.txt new file mode 100644 index 00000000..1b74e847 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-2.txt @@ -0,0 +1,20 @@ +== hidden +✗ base: an accepted assignment survives switching filters and back +76/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +33/38 passed +== fuzz +desk12-rx-2: 4/20 seeds found a hole + seed 5 step 23: detail title: "#3 Crash on savestack trace attachedrelated help failed: dec", server "Delta" + focus ; fail fetchIssues:closed ; tab help ; tab issues ; land fetchIssues:closed ; user bob ; land fetchIssues:open ; user guest ; row-assign 0 ; row-assign 0 ; land assign:1,guest ; land fetchIssues:open ; row-assign 1 ; row-assign 1 ; land assign:3,guest ; row-close 2 ; select #3 ; rename Delta ; select #3 ; land fetchIssue:3 ; land closeIssue:4 ; fail search:Crash on save ; land renameIssu + seed 8 step 22: detail title: "#4 Typo in footerfooter says 2019related help failed: declin", server "Alpha" + land fetchIssues:closed ; tab help ; tab issues ; fail fetchIssues:closed ; user bob ; land fetchIssues:open ; row-close 0 ; land closeIssue:1 ; user guest ; land fetchIssues:open ; row-close 0 ; select #4 ; land fetchComments:4 ; row-close 0 ; rename Alpha ; land fetchIssue:4 ; rename Alpha ; user guest ; fail search:Typo in footer ; close ; land fetchLabels:4 ; land renameIssue:4,Alpha + seed 13 step 8: closed row shown: #2 is closed on the server + filter closed ; fail fetchIssues:open ; land fetchIssues:closed ; row-assign 0 ; filter open ; land assign:2,guest ; land fetchIssues:closed ; land fetchIssues:open + seed 16 step 42: row title: #1 shows "Login fails", server "Bravo" + user guest ; fail fetchIssues:open ; tab help ; land fetchIssues:closed ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #4 ; land fetchComments:4 ; close ; post me too ; fail addComment:4,guest,me too ; row-assign 1 ; row-assign 1 ; close ; post ping ; assign ; land addComment:4,guest,ping ; fail assign:3,guest ; fail assign:4,guest ; select #1 ; rename Bravo ; assign ; fail close diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-3.txt new file mode 100644 index 00000000..e46bb31f --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-3.txt @@ -0,0 +1,14 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-3: 4/20 seeds found a hole + seed 5 step 22: closed row shown: #3 is closed on the server + focus ; fail fetchIssues:open ; tab help ; tab issues ; focus ; land fetchIssues:open ; row-close 0 ; row-assign 1 ; land assign:4,guest ; land closeIssue:1 ; land fetchIssues:open ; row-close 1 ; land closeIssue:4 ; land fetchIssues:open ; row-assign 0 ; land assign:3,guest ; row-close 0 ; select #3 ; rename Delta ; select #3 ; land fetchIssue:3 ; land closeIssue:3 + seed 7 step 11: count: tab Issues (0), 1 open rows shown + land fetchIssues:open ; row-close 1 ; row-assign 0 ; row-close 0 ; land closeIssue:1 ; fail assign:1,guest ; row-assign 0 ; fail assign:4,guest ; fail closeIssue:3 ; user ann ; row-close 0 + seed 15 step 38: detail title: "#2 Bravolong gonesaving filescrash reportsretry assign to gu", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 + seed 18 step 19: count: tab Issues (1), 2 open rows shown + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta ; user bob ; row-assign 0 ; add-label ui ; rename Delta ; assign ; row-close 0 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-4.txt new file mode 100644 index 00000000..96c29b64 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-4.txt @@ -0,0 +1,6 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-rx-4: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-5.txt new file mode 100644 index 00000000..90180674 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-rx-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-6.txt new file mode 100644 index 00000000..17615345 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-6.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-rx-6: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-7.txt new file mode 100644 index 00000000..657323bb --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-7.txt @@ -0,0 +1,11 @@ +== hidden +77/77 passed +== overlap +✗ a label added on one issue does not show on another +37/38 passed +== fuzz +desk12-rx-7: 2/20 seeds found a hole + seed 8 step 24: dropped guess: #4 assignee: showed "guest" while it was in flight, now "dana" + land fetchIssues:open ; row-close 1 ; filter closed ; fail fetchIssues:closed ; user bob ; land closeIssue:3 ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; select #1 ; land fetchComments:1 ; assign ; rename Delta ; close ; row-close 0 ; land fetchIssue:1 ; land search:Login fails ; assign ; user guest ; fail closeIssue:4 ; close ; fail assign:1,guest ; land closeIssue:1 + seed 12 step 36: closed row shown: #1 is closed on the server + land fetchIssues:open ; focus ; select #1 ; row-assign 1 ; select #4 ; land assign:3,guest ; row-assign 2 ; assign ; land assign:4,guest ; assign ; fail fetchLabels:1 ; land fetchComments:1 ; land fetchLabels:4 ; land fetchComments:4 ; land fetchIssue:4 ; land fetchIssue:1 ; assign ; land search:Typo in footer ; land assign:4,guest ; post me too ; close ; fail addComment:4,guest,me too ; selec diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-8.txt new file mode 100644 index 00000000..3b0148ef --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-8.txt @@ -0,0 +1,11 @@ +== hidden +✗ base: retry after not found shows the issue +✗ base: settings: retrying a failed load shows the settings +75/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ rows: a declined row close leaves a rename out on another issue +35/38 passed +== fuzz +desk12-rx-8: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-rx-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-rx-9.txt new file mode 100644 index 00000000..9c0899c2 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-rx-9.txt @@ -0,0 +1,17 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a declined assignment keeps a rename in flight on the same issue +✗ base: a declined rename keeps an assignment in flight on the same issue +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +✗ a label added on one issue does not show on another +✗ rows: a declined row close leaves a rename out on another issue +33/38 passed +== fuzz +desk12-rx-9: 2/20 seeds found a hole + seed 11 step 44: detail title: "retry#3 Crash on savestack trace attachedsaving filescrash r", server "Delta" + tab help ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; focus ; tab help ; tab issues ; land fetchIssues:open ; row-assign 1 ; land assign:3,guest ; filter closed ; user ann ; row-assign 0 ; focus ; land fetchIssues:open ; land assign:1,ann ; select #4 ; close ; land fetchIssues:closed ; row-assign 0 ; row-close 0 ; select #3 ; focus ; close ; land fetchIssue:4 ; post me too ; + seed 15 step 24: detail title: "retry#2 Old buglong gonesaving filescrash reportsassign to g", server "Bravo" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; fail fetchIssues:open ; tab help ; tab issues ; land fetchIssues:open ; focus ; filter closed ; land fetchIssues:closed ; select #2 ; land fetchLabels:2 ; rename Bravo ; select #2 ; assign ; land fetchIssue:2 ; row-assign 0 ; select #2 ; land fetchComments:2 ; assign ; remove-label 0 ; land search:Old bug ; land renameIssue:2 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-1.txt new file mode 100644 index 00000000..05639a16 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-1.txt @@ -0,0 +1,10 @@ +== hidden +✗ base: settings: saving an invalid name asks nothing +76/77 passed +== overlap +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +37/38 passed +== fuzz +desk12-vanilla-1: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-10.txt new file mode 100644 index 00000000..0c3333ac --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-10.txt @@ -0,0 +1,21 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: open issues that load while a row close is out keep the row out +29/38 passed +== fuzz +desk12-vanilla-10: 3/20 seeds found a hole + seed 3 step 39: count: tab Issues (3), 2 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchLabels:4 ; row- + seed 8 step 16: row title: #4 shows "Typo in footer", server "Bravo" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-11.txt new file mode 100644 index 00000000..520f87a6 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-11.txt @@ -0,0 +1,23 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-vanilla-11: 3/20 seeds found a hole + seed 3 step 38: count: tab Issues (4), 3 open rows shown + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land fetchLabels:4 ; add-label bug + seed 4 step 33: row title: #1 shows "Login fails", server "Bravo" + rename Alpha ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; row-close 2 ; land fetchLabels:4 ; add-label ui ; add-label p1 ; row-assign 0 ; rename Delta ; close ; select #1 ; add-label bug ; close ; fail renameIssue:4,Delta ; rename Bravo ; land renameIssue:1,Bravo ; close ; land fetchComments:1 ; land fetchLabels:1 ; land addLabel:4,p1 ; row-assi + seed 8 step 16: row title: #4 shows "Typo in footer", server "Bravo" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-12.txt new file mode 100644 index 00000000..50b406a3 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-12.txt @@ -0,0 +1,18 @@ +== hidden +77/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-vanilla-12: 2/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-2.txt new file mode 100644 index 00000000..7358c9a6 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-3.txt new file mode 100644 index 00000000..11d5e963 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-3.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-3: 2/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-4.txt new file mode 100644 index 00000000..1d5ae00b --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-4.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-4: 1/20 seeds found a hole + seed 12 step 36: at rest: comments: #4 shows [guest: me too | guest: me too], server [guest: me too] + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; fail closeIssue:4 ; land assign:4,guest ; land fetchComments:1 ; land fetchLabels:1 ; post me too ; close ; fail renameIssue:4,Alpha ; assign ; land addComment:4,guest,me too ; close ; diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-5.txt new file mode 100644 index 00000000..b20aeb74 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-6.txt new file mode 100644 index 00000000..de893d02 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-6.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-vanilla-6: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-7.txt new file mode 100644 index 00000000..ad648c87 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-7.txt @@ -0,0 +1,14 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a detail assign and a newer row assign on one issue: the newer shows while the older answers +34/38 passed +== fuzz +desk12-vanilla-7: 1/20 seeds found a hole + seed 3 step 43: at rest: labels: #4 shows [docs], server [docs,ui] + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,guest ; add-label bu diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-8.txt new file mode 100644 index 00000000..96b30bf8 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-8.txt @@ -0,0 +1,29 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +27/38 passed +== fuzz +desk12-vanilla-8: 5/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 12 step 34: row assignee: #4 shows dana, server guest + land fetchIssues:open ; focus ; select #1 ; row-close 1 ; select #4 ; land closeIssue:3 ; row-assign 1 ; rename Alpha ; close ; rename Alpha ; land fetchIssue:1 ; land fetchIssue:4 ; row-close 0 ; rename Alpha ; land search:Login fails ; land renameIssue:4,Alpha ; land fetchComments:1 ; land fetchComments:4 ; filter closed ; land closeIssue:1 ; fail assign:4,guest ; assign ; land search:Typo i + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan diff --git a/eval/results/desk12-2026-11-11/graded/desk12-vanilla-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-9.txt new file mode 100644 index 00000000..68ce208e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-vanilla-9.txt @@ -0,0 +1,22 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +30/38 passed +== fuzz +desk12-vanilla-9: 3/20 seeds found a hole + seed 8 step 37: detail title: "#3 Crash on savestack trace attachedretryassign to bobrename", server "Alpha" + land fetchIssues:open ; row-close 1 ; post me too ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; select #4 ; land fetchComments:4 ; land fetchLabels:4 ; fail closeIssue:3 ; close ; remove-label 0 ; close ; rename Bravo ; land renameIssue:4,Bravo ; user bob ; add-label ui ; fail closeIssue:4 ; land addLabel:4,ui ; select #3 ; focus ; fail fetchLabels:3 ; row-assign 0 ; assign ; land + seed 11 step 46: detail title: "#3 Crash on savestack trace attachedretryassign to guestrena", server "Bravo" + close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; close ; (move failed: locator.click: Timeout 2000ms exceeded.) ; post fixed? ; (move failed: locator.fill: Timeo + seed 17 step 19: dropped guess: #3 title: showed "Alpha" while it was in flight, now "Crash on save" + assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; assign ; (move failed: locator.click: Timeout 2000ms exceeded.) ; land fetchIssues:open ; row-assign 0 ; add-label bug ; (move failed: locator.fill: Timeout 2000ms exceeded.) ; land assign:1,guest ; row-close 0 ; row-close 1 ; row-assign 0 ; select #3 ; rename Bravo ; row-assign 0 ; land fetchLabels:3 ; rename Alpha ; assign ; f diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-1.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-1.txt new file mode 100644 index 00000000..7adfa732 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-1.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-1: 1/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-10.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-10.txt new file mode 100644 index 00000000..64d01d85 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-10.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +37/38 passed +== fuzz +desk12-xstate-10: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-11.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-11.txt new file mode 100644 index 00000000..73e5ec2e --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-11.txt @@ -0,0 +1,12 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ base: settings: an earlier save failing after a later one succeeded leaves the later one +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ base: settings: an earlier save landing while a later one is out keeps showing the later one +34/38 passed +== fuzz +desk12-xstate-11: 1/20 seeds found a hole + seed 16 step 39: at rest: comments: #2 shows [guest: fixed? | guest: me too | guest: me too], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-12.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-12.txt new file mode 100644 index 00000000..8764a683 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-12.txt @@ -0,0 +1,28 @@ +== hidden +✗ base: a failing list leaves the detail alone +✗ base: help: a failing detail leaves the results alone +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +28/38 passed +== fuzz +desk12-xstate-12: 5/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 39: at rest: detail: "retryassign to guestassign error: declinedrenameclosepostp1 ", server "#4 Bravo" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 34: at rest: detail: "retryassign to annrenamerename error: declinedclosepostpost ", server "#2 Old bug" + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; add-label p1 ; add-label ui ; close ; user guest ; rename Alpha ; filter open ; land fetchLabels:2 ; fail renameIssue:2,Alpha ; assign ; land addLabel:2,p1 ; focus + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-2.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-2.txt new file mode 100644 index 00000000..449aa353 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-2.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-2: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-3.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-3.txt new file mode 100644 index 00000000..7550f230 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-3.txt @@ -0,0 +1,7 @@ +== hidden +77/77 passed +== overlap +✗ labels that load while an add is out keep the add +37/38 passed +== fuzz +desk12-xstate-3: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-4.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-4.txt new file mode 100644 index 00000000..cd4d74ca --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-4.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +38/38 passed +== fuzz +desk12-xstate-4: 1/20 seeds found a hole + seed 9 step 40: two views: row "#4 Bravo", detail "#4 Deltafooter says 2019retryassign to g" + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-5.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-5.txt new file mode 100644 index 00000000..d98cee76 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-5.txt @@ -0,0 +1,8 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-5: 0/20 seeds found a hole diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-6.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-6.txt new file mode 100644 index 00000000..3cdbcf7d --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-6.txt @@ -0,0 +1,10 @@ +== hidden +77/77 passed +== overlap +✗ base: comments that load while a post is out keep the post +✗ labels that load while an add is out keep the add +36/38 passed +== fuzz +desk12-xstate-6: 1/20 seeds found a hole + seed 3 step 39: at rest: labels: #4 shows [docs], server [docs,ui] + user bob ; land fetchIssues:open ; row-close 0 ; fail closeIssue:1 ; select #4 ; add-label p1 ; land fetchLabels:4 ; row-assign 2 ; select #4 ; land assign:4,bob ; add-label bug ; fail addLabel:4,bug ; land fetchIssue:4 ; row-close 2 ; row-close 0 ; land closeIssue:4 ; row-assign 0 ; remove-label 0 ; close ; row-close 0 ; filter closed ; add-label p1 ; assign ; land closeIssue:4 ; fail removeL diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-7.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-7.txt new file mode 100644 index 00000000..b8b1da4b --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-7.txt @@ -0,0 +1,28 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-xstate-7: 4/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 31: detail title: "#3 stack trace attachedretryassign to guestcloserenamerename", server "Crash on save" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-8.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-8.txt new file mode 100644 index 00000000..6a7fb93b --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-8.txt @@ -0,0 +1,30 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-xstate-8: 5/20 seeds found a hole + seed 2 step 37: row assignee: #2 shows guest, server eve + filter closed ; land fetchIssues:closed ; row-assign 0 ; select #2 ; land fetchIssue:2 ; assign ; rename Delta ; fail renameIssue:2,Delta ; fail fetchIssues:open ; close ; focus ; land fetchLabels:2 ; fail search:Old bug ; land closeIssue:2 ; fail fetchComments:2 ; row-assign 0 ; select #2 ; post fixed? ; select #2 ; post fixed? ; select #2 ; close ; land closeIssue:2 ; rename Alpha ; rename B + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 16 step 38: at rest: comments: #2 shows [guest: fixed?], server [guest: fixed? | guest: me too] + user guest ; fail fetchIssues:open ; focus ; tab help ; tab issues ; filter closed ; land fetchIssues:open ; focus ; land fetchIssues:closed ; select #2 ; land fetchComments:2 ; close ; post me too ; fail addComment:2,guest,me too ; row-assign 0 ; close ; post fixed? ; post ping ; assign ; land addComment:2,guest,fixed? ; land fetchLabels:2 ; fail assign:2,guest ; select #2 ; rename Bravo ; as + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/graded/desk12-xstate-9.txt b/eval/results/desk12-2026-11-11/graded/desk12-xstate-9.txt new file mode 100644 index 00000000..d7533a05 --- /dev/null +++ b/eval/results/desk12-2026-11-11/graded/desk12-xstate-9.txt @@ -0,0 +1,28 @@ +== hidden +✗ the results for a related hit may land after the help page opens +✗ a related item that fails can be retried on the help page +75/77 passed +== overlap +✗ base: a list that loads while an assignment is out keeps the new assignee +✗ base: a list that loads while a rename is out keeps the new title +✗ base: a detail that loads while a rename is out keeps the new title +✗ base: an earlier rename declined after a later one was accepted keeps the later title +✗ base: a list that loads while a close is out keeps the row out +✗ base: two closes at once: a declined one does not undo the other +✗ base: comments that load while a post is out keep the post +✗ base: settings: a later save failing while an earlier one is out shows its error and then the earlier result +✗ labels that load while an add is out keep the add +✗ open issues that load while a close is out keep it off the count +✗ rows: a row assign and a newer detail assign on one issue: a declined older one leaves the newer +✗ rows: open issues that load while a row close is out keep the row out +26/38 passed +== fuzz +desk12-xstate-9: 4/20 seeds found a hole + seed 4 step 24: closed row shown: #1 is closed on the server + filter closed ; land fetchIssues:open ; user guest ; tab help ; tab issues ; focus ; tab help ; tab issues ; focus ; filter closed ; land fetchIssues:open ; tab help ; fail fetchIssues:closed ; tab issues ; focus ; land fetchIssues:open ; row-assign 1 ; fail assign:3,guest ; row-close 0 ; filter closed ; filter open ; land fetchIssues:closed ; land fetchIssues:open ; land closeIssue:1 + seed 9 step 40: row assignee: #4 shows null, server dana + land fetchIssues:open ; filter closed ; tab help ; land fetchIssues:closed ; tab issues ; tab help ; land fetchIssues:open ; tab issues ; filter closed ; focus ; filter open ; user guest ; land fetchIssues:open ; land fetchIssues:closed ; select #4 ; assign ; land fetchComments:4 ; rename Bravo ; row-assign 0 ; rename Bravo ; row-assign 1 ; row-close 1 ; land closeIssue:3 ; row-assign 1 ; filt + seed 15 step 36: row title: #3 shows "Crash on save", server "Delta" + land fetchIssues:open ; row-assign 0 ; focus ; land assign:1,guest ; row-assign 0 ; fail assign:1,guest ; row-assign 0 ; focus ; row-assign 0 ; select #3 ; land fetchLabels:3 ; close ; row-close 0 ; row-close 0 ; land assign:1,guest ; add-label ui ; assign ; rename Alpha ; remove-label 0 ; post fixed? ; land assign:3,guest ; land fetchIssue:3 ; assign ; land assign:3,guest ; rename Delta ; lan + seed 18 step 13: row title: #4 shows "Typo in footer", server "Delta" + land fetchIssues:open ; select #4 ; row-close 0 ; land fetchLabels:4 ; assign ; row-close 0 ; row-close 0 ; assign ; rename Delta ; fail closeIssue:3 ; fail closeIssue:4 ; land fetchComments:4 ; land renameIssue:4,Delta diff --git a/eval/results/desk12-2026-11-11/repro/two-close-guesses.spec.tsx b/eval/results/desk12-2026-11-11/repro/two-close-guesses.spec.tsx new file mode 100644 index 00000000..02e902e2 --- /dev/null +++ b/eval/results/desk12-2026-11-11/repro/two-close-guesses.spec.tsx @@ -0,0 +1,39 @@ +/** + * @jsxImportSource @solidjs/web + * @vitest-environment jsdom + */ +import { test, expect } from "vitest"; +import { For, Loading, action, createOptimistic, createOptimisticStore, createMemo, createSignal, refresh, flush } from "solid-js"; +import { render } from "../src/index.js"; +const tick = async () => { for (let i = 0; i < 4; i++) { await new Promise(r => setTimeout(r, 0)); flush(); } }; +const V = process.env.V ?? "full"; +test("two close guesses on one list: both hold until the refresh lands", async () => { + const asks: { key: string; res: (v: any) => void }[] = []; + const ask = (key: string) => new Promise(res => asks.push({ key, res })); + const pick = (key: string) => { const i = asks.findIndex(a => a.key === key); if (i < 0) throw new Error("no ask " + key + " in " + asks.map(a => a.key)); return asks.splice(i, 1)[0]; }; + const c = document.body.appendChild(document.createElement("div")); + let close!: (id: number) => any, assign!: (id: number) => any; + let server = [{ id: 1, a: "" }, { id: 4, a: "dana" }]; + render(() => { + const [issues, setIssues] = createOptimisticStore(async (d: any) => { const v = await ask("fetch"); d.splice(0, d.length, ...v); }, [] as any[], { key: "id" } as any); + const [truth] = createSignal(() => []); + const [hidden, guessHidden] = createOptimistic(() => truth()); + const shown = createMemo(() => issues.filter((i: any) => !hidden().includes(i.id))); + close = action(function* (id: number) { guessHidden(h => [...h, id]); yield ask(`close:${id}`); refresh(issues); }); + assign = action(function* (id: number) { setIssues((d: any) => { d.find((x: any) => x.id === id).a = "guest"; }); yield ask(`assign:${id}`); refresh(issues); }); + return loading}>{r => [#{r.id}]}; + }, c); + await tick(); + pick("fetch").res(server.map(x => ({ ...x }))); await tick(); + expect(c.textContent).toBe("[#1][#4]"); + close(1); await tick(); + if (V !== "noassign") { assign(4); await tick(); } + close(4); await tick(); + expect(c.textContent).toBe(""); + server = server.filter(x => x.id !== 4); + pick("close:4").res({}); await tick(); + expect(c.textContent).toBe(""); + server = server.filter(x => x.id !== 1); + pick("close:1").res({}); await tick(); + expect(c.textContent).toBe(""); // the fetch the closes asked is still out: both guesses hold (D-33, F-11) +}); diff --git a/eval/results/desk12-2026-11-11/stats.py b/eval/results/desk12-2026-11-11/stats.py new file mode 100644 index 00000000..5665da4e --- /dev/null +++ b/eval/results/desk12-2026-11-11/stats.py @@ -0,0 +1,39 @@ +import re, random, json, sys +rows={} +for l in open(sys.argv[1] if len(sys.argv)>1 else 'summary.txt'): + a,n,h,o,z=re.match(r'(\w+)-(\d+) hidden/overlap=(\d+)/77 (\d+)/38\s+fuzz=(\d+)/20',l).groups() + rows[(a,int(n))]=dict(h=77-int(h),o=38-int(o),z=int(z)) +excl={} # fuzz holes traced to legal holds: (arm,n) -> count +def D(k): r=rows[k]; return r['h']+r['o']+r['z']-excl.get(k,0) +def perm(x,y,seed=12,N=100000): + random.seed(seed); obs=sum(y)/len(y)-sum(x)/len(x); pool=x+y; c=0 + for _ in range(N): + random.shuffle(pool); xx=pool[:len(x)]; yy=pool[len(x):] + if sum(yy)/len(yy)-sum(xx)/len(xx)>=obs-1e-12: c+=1 + return (c+1)/(N+1) +def boot(x,y,seed=12,N=10000): + random.seed(seed); d=[] + for _ in range(N): + xx=[random.choice(x) for _ in x]; yy=[random.choice(y) for _ in y] + d.append(sum(xx)/len(xx)-sum(yy)/len(yy)) + d.sort(); return d[int(.05*N)], d[int(.95*N)-1] +try: usd=json.load(open('usd.json')) +except Exception: usd=None +front=range(1,7); small=range(7,13); A=('rx','react','vanilla','foldkit','xstate') +for a in A: + for tier,rg in (('frontier',front),('small',small)): + v=[D((a,n)) for n in rg]; c=sum(1 for x in v if x==0) + hz=[sum(rows[(a,n)][k] for n in rg)/len(rg) for k in 'hoz'] + s=f" {a:8}{tier:9} mean {sum(v)/len(v):5.2f} median {sorted(v)[2]/2+sorted(v)[3]/2:4.1f} per desk {v} h/o/z {hz[0]:.1f}/{hz[1]:.1f}/{hz[2]:.1f} correct {c}" + if usd: + cost=[x[0] for x in usd[f'{a}-{tier}']]; m=sum(cost)/len(cost) + s+=f" $/run {m:.3f} $/correct "+(f"${sum(cost)/c:.3f}" if c else "undefined") + print(s) +S={a:[D((a,n)) for n in small] for a in A}; F={a:[D((a,n)) for n in front] for a in A} +ps={'H12a rx holds if upper < 1: {hi<1}") +print(f" H12d small xstate holds if upper < 1: False + H12d small xstate