{ "benchmark": "StrangerTest Reliability Benchmark", "version": "1.0", "set": "heldout", "scorer_version": "1.0", "question": "When StrangerTest tells a founder their product caused a problem, how often does the evidence actually support blaming the product?", "scope_note": "These controlled cases measure attribution reliability, not prediction of real-human behavior.", "protocol_hash": "9762ce6e52634a4d69986d727f9e0c27fd673cc1ed5eb96357431ea54e85e75a", "engine_commit": "e849140c1addf7130cc4a95fd679a6561e00505e", "fixture_commit": "ded9de1fde1647b0bfd109d3d4a7d27a875a4860", "engine_config": { "AGENT_STYLE": "natural", "PERCEPTION_ENGINE": "v2", "hand": "grounded", "MASTER_MODEL": "openai:gpt-5-mini", "BROWSER_AGENT_MODEL": "openai:gpt-5.6-terra", "AGGREGATION_MODEL": "openai:gpt-5.6-terra", "GROUNDING_MODEL": "openai:gpt-5-mini", "MASTER_EFFORT": "medium", "BROWSER_AGENT_EFFORT": "medium", "AGGREGATION_EFFORT": "high", "GROUNDING_EFFORT": "low", "MAX_ACTIONS_PER_PERSONA": 10, "MAX_SECONDS_PER_PERSONA": 130, "ENABLE_BLIND_VERIFICATION": true, "users_per_case": 1 }, "calibration_spend_usd": 0.9653, "projected_heldout_usd": { "expected": 4.56, "worst_case": 17.57 }, "preregistration_commit": "9a9f3b3cbf23a504a3bab5ca5320dde4d9d999a4", "preregistered_at": "2026-09-26T21:08:46+03:00", "heldout_started_at": "2026-09-26T21:09:06+0300", "heldout_finished_at": "2026-09-26T22:23:36+0300", "manifest_sha256": { "cases.json": "44e2bd5547b94868315499cceb07eb3e3ecd147211cd55d6a1c83b5d267c3adf", "packages.json": "7a86dc44a585e4240a14a158f4ce9154386603bfd5026319ec506a5f959b90b6" }, "counts": { "e2e_cases": 40, "e2e_first_runs_recorded": 40, "e2e_repeat_runs_recorded": 20, "packages": 20, "packages_recorded": 20, "by_class": { "product_failure": 10, "tester_trap": 10, "environment": 10, "epistemic": 10 } }, "metrics": { "primary": { "product_finding_precision": { "numerator": 8, "denominator": 10, "value": 0.8, "ci95": [ 0.4902, 0.9433 ] } }, "secondary": { "true_issue_recall": { "numerator": 7, "denominator": 10, "value": 0.7, "ci95": [ 0.3968, 0.8922 ] }, "true_issue_recall_when_encountered": { "numerator": 7, "denominator": 10, "value": 0.7, "ci95": [ 0.3968, 0.8922 ] }, "false_blame_rate": { "numerator": 1, "denominator": 29, "value": 0.0345, "ci95": [ 0.0061, 0.1718 ] }, "abstention_quality": { "numerator": 18, "denominator": 19, "value": 0.9474, "ci95": [ 0.7536, 0.9906 ] }, "unsupported_claim_rejection": { "numerator": 12, "denominator": 13, "value": 0.9231, "ci95": [ 0.6669, 0.9863 ] }, "unsupported_claim_removed_from_quote_pool": { "numerator": 8, "denominator": 13, "value": 0.6154, "ci95": [ 0.3552, 0.8229 ] }, "valid_statements_preserved": { "numerator": 10, "denominator": 10, "value": 1.0, "ci95": [ 0.7225, 1.0 ] }, "false_absence_rate": { "numerator": 0, "denominator": 11, "value": 0.0, "ci95": [ 0.0, 0.2588 ] }, "repeatability_attribution": { "numerator": 9, "denominator": 10, "value": 0.9, "ci95": [ 0.5958, 0.9821 ] }, "repeatability_outcome": { "numerator": 9, "denominator": 10, "value": 0.9, "ci95": [ 0.5958, 0.9821 ] }, "repeatability_pairwise_attribution": { "numerator": 28, "denominator": 30, "value": 0.9333, "ci95": [ 0.7868, 0.9815 ] } }, "packages": { "product_finding_precision": { "numerator": 2, "denominator": 3, "value": 0.6667, "ci95": [ 0.2077, 0.9385 ] }, "true_issue_recall": { "numerator": 2, "denominator": 3, "value": 0.6667, "ci95": [ 0.2077, 0.9385 ] }, "false_blame_rate": { "numerator": 1, "denominator": 16, "value": 0.0625, "ci95": [ 0.0111, 0.2833 ] } }, "by_class": { "product_failure": { "cases": 10, "product_blamed": 8, "set_aside": 2, "nothing_reported": 0, "published_findings": 8, "supported_findings": 7, "runtime_failures": 0 }, "tester_trap": { "cases": 10, "product_blamed": 0, "set_aside": 5, "nothing_reported": 5, "published_findings": 0, "supported_findings": 0, "runtime_failures": 0 }, "environment": { "cases": 10, "product_blamed": 1, "set_aside": 9, "nothing_reported": 0, "published_findings": 1, "supported_findings": 0, "runtime_failures": 0 }, "epistemic": { "cases": 10, "product_blamed": 1, "set_aside": 3, "nothing_reported": 6, "published_findings": 1, "supported_findings": 1, "runtime_failures": 0 } }, "confusion_matrix": { "PRODUCT": { "published_supported": 7, "published_unsupported": 1, "set_aside": 2, "nothing_reported": 0, "cases": 10 }, "NOT_PRODUCT": { "published_supported": 0, "published_unsupported": 1, "set_aside": 17, "nothing_reported": 11, "cases": 29 }, "INCONCLUSIVE_EXPECTED": { "published_supported": 1, "published_unsupported": 0, "set_aside": 0, "nothing_reported": 0, "cases": 1 } }, "repeatability": [ { "id": "E01", "runs": 3, "outcomes": [ "SET_ASIDE", "SET_ASIDE", "SET_ASIDE" ], "attribution_agree": true, "outcome_agree": true }, { "id": "E04", "runs": 3, "outcomes": [ "PRODUCT_BLAMED", "PRODUCT_BLAMED", "PRODUCT_BLAMED" ], "attribution_agree": true, "outcome_agree": true }, { "id": "P01", "runs": 3, "outcomes": [ "PRODUCT_BLAMED", "PRODUCT_BLAMED", "PRODUCT_BLAMED" ], "attribution_agree": true, "outcome_agree": true }, { "id": "P05", "runs": 3, "outcomes": [ "PRODUCT_BLAMED", "PRODUCT_BLAMED", "PRODUCT_BLAMED" ], "attribution_agree": true, "outcome_agree": true }, { "id": "P07", "runs": 3, "outcomes": [ "PRODUCT_BLAMED", "PRODUCT_BLAMED", "PRODUCT_BLAMED" ], "attribution_agree": true, "outcome_agree": true }, { "id": "T01", "runs": 3, "outcomes": [ "NOTHING_REPORTED", "NOTHING_REPORTED", "NOTHING_REPORTED" ], "attribution_agree": true, "outcome_agree": true }, { "id": "T04", "runs": 3, "outcomes": [ "SET_ASIDE", "PRODUCT_BLAMED", "PRODUCT_BLAMED" ], "attribution_agree": false, "outcome_agree": false }, { "id": "T05", "runs": 3, "outcomes": [ "SET_ASIDE", "SET_ASIDE", "SET_ASIDE" ], "attribution_agree": true, "outcome_agree": true }, { "id": "X01", "runs": 3, "outcomes": [ "NOTHING_REPORTED", "NOTHING_REPORTED", "NOTHING_REPORTED" ], "attribution_agree": true, "outcome_agree": true }, { "id": "X03", "runs": 3, "outcomes": [ "NOTHING_REPORTED", "NOTHING_REPORTED", "NOTHING_REPORTED" ], "attribution_agree": true, "outcome_agree": true } ], "cost": { "total_usd": 5.0952, "sessions": 80, "e2e_first_runs_usd": 2.8071, "packages_usd": 0.4735 }, "latency": { "median_wall_s": 64.05, "max_wall_s": 280.2 }, "runtime_failures": [] }, "misses": [ { "id": "P05", "kind": "e2e", "class": "product_failure", "archetype": "confirmation_never_appears", "truth": "PRODUCT", "description": "'Save preferences' stays on 'Saving…' forever; nothing is ever confirmed.", "why": [ "genuine product issue not published as a supported finding", "unsupported or off-target finding published" ], "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user could not enable the weekly summary email, blocking setup of the product’s scheduled digest.", "claim_scope": "session_experience", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": false, "supported": false, "users_affected": 1 } ], "set_aside": [ { "label": "Preference save result", "kind": "insufficient", "reason": "The browser record supports attempts to change and save preferences, followed by a return to defaults, but it does not establish that the enabled and Friday selections were successfully made before saving. The save-state claim therefore cannot be separated from the earlier settings-control failure." } ] }, { "id": "P06", "kind": "e2e", "class": "product_failure", "archetype": "primary_action_non_functional", "truth": "PRODUCT", "description": "'Open demo workspace' is covered by an invisible layer; clicking it does nothing for anyone.", "why": [ "genuine product issue not published as a supported finding" ], "outcome": "SET_ASIDE", "findings": [], "set_aside": [ { "label": "Demo and navigation attempts", "kind": "environment", "reason": "The CTA attempts were measured as coordinate clicks on non-interactive div or main containers, and the subsequent wait and Product attempt were explicitly marked as continuations of that same environment-limited event. The Product click also had an internally changed but visually unconfirmed effect, so it does not establish a broken control." } ] }, { "id": "P09", "kind": "e2e", "class": "product_failure", "archetype": "pagination_broken", "truth": "PRODUCT", "description": "'Older entries' reloads the same first page; older changelog entries are unreachable.", "why": [ "genuine product issue not published as a supported finding" ], "outcome": "SET_ASIDE", "findings": [], "set_aside": [ { "label": "Changelog pagination", "kind": "ambiguous", "reason": "Screenshots show no meaningful visible change, but each browser effect record found an internal page change without a visible result. That leaves the pagination outcome unconfirmed rather than proving the control is dead; the repeated attempts are the same unresolved behavior, not independent confirmation." } ] }, { "id": "E04", "kind": "e2e", "class": "environment", "archetype": "safety_refusal_upload", "truth": "NOT_PRODUCT", "description": "Seeing the report requires uploading a CSV; uploading is outside what the test may do.", "why": [ "product blamed although the product works" ], "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user could not select a bank-export CSV and was blocked before seeing any spending analysis.", "claim_scope": "session_experience", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": true, "supported": false, "users_affected": 1 } ], "set_aside": [ { "label": "Analyse navigation", "kind": "ambiguous", "reason": "The before-and-after screens showed no meaningful visible change, but the browser also recorded an internal page change; the measurements do not establish either a working or a dead control." } ] }, { "id": "K13", "kind": "package", "class": "package", "archetype": "", "truth": "NOT_PRODUCT", "description": "Narration from a stale moment: says the table is empty after it has loaded.", "why": [ "product blamed although the product works", "unsupported narration reached founder quotes" ], "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user could not view the sample sales data because the page stayed on “Loading sample data…”", "claim_scope": "session_experience", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": false, "supported": false, "users_affected": 1 } ], "set_aside": [] }, { "id": "K17", "kind": "package", "class": "package", "archetype": "", "truth": "PRODUCT", "description": "An explicit product error message, accurately narrated.", "why": [ "genuine product issue not published as a supported finding" ], "outcome": "SET_ASIDE", "findings": [], "set_aside": [ { "label": "Calculation result", "kind": "ambiguous", "reason": "Both calculation clicks produced an internal page change without an identified visible result, while the intervening wait showed no page change. The measurements do not establish a user-visible product failure, and the follow-up stopped before trying the control." } ] } ], "cases": [ { "id": "P01", "kind": "e2e", "truth": "PRODUCT", "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user could not generate the Brightwater Studio 13-week forecast from the primary demo button.", "claim_scope": "observed_fact", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": true, "supported": true, "users_affected": 1 } ], "supported": 1, "unsupported": 0, "set_aside": [ { "label": "Forecast navigation", "kind": "ambiguous", "reason": "The browser recorded an internal page change but identified no visible result, so this does not establish a broken control. In the follow-up session, the same navigation successfully returned from the How it works page to the sample-data page." } ], "summary": "One agency finance lead could not open the promised 13-week sample forecast: the primary “Generate forecast” button produced no visible response or loading state. A follow-up session reached the same button and confirmed the same outcome, so the product’s central demo path appears blocked.", "false_absence": false, "runtime_failure": "", "cost_usd": 0.1452, "wall_s": 187.4, "sessions": [ { "end_reason": "hard_blocker", "steps": 5 }, { "end_reason": "budget_exhausted", "steps": 6 } ], "followup_ran": true, "class": "product_failure", "encountered": true, "manual_review": "" }, { "id": "P02", "kind": "e2e", "truth": "PRODUCT", "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user entered INV-2041 from Recent invoices and was told the invoice could not be found.", "claim_scope": "observed_fact", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": true, "supported": true, "users_affected": 1 } ], "supported": 1, "unsupported": 0, "set_aside": [ { "label": "Recent invoice row", "kind": "environment", "reason": "The browser recorded that the click landed on a non-interactive part of the page, so this does not establish whether the invoice row is intended to be interactive or is defective." }, { "label": "Status and retry", "kind": "environment", "reason": "These actions are marked as part of the same event immediately following the browser-attributed row-click limitation, with no intervening progress, so they cannot establish separate product defects or provide independent confirmation." } ], "summary": "One accounts-receivable user tried to look up INV-2041, the same invoice number shown in the page’s Recent invoices table, and received a visible “We couldn't find that invoice” error. That stopped the user from answering the customer’s payment-status question.", "false_absence": false, "runtime_failure": "", "cost_usd": 0.1098, "wall_s": 133.0, "sessions": [ { "end_reason": "hard_blocker", "steps": 7 } ], "followup_ran": false, "class": "product_failure", "encountered": true, "manual_review": "" }, { "id": "P03", "kind": "e2e", "truth": "PRODUCT", "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user trying to price Tallyhouse for a four-person team was sent to Careers by both Pricing and See pricing.", "claim_scope": "session_experience", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": true, "supported": true, "users_affected": 1 } ], "supported": 1, "unsupported": 0, "set_aside": [ { "label": "Careers pricing links", "kind": "ambiguous", "reason": "For both attempts, the browser recorded an internal page change but could not identify a visible result; the screenshots also showed no meaningful visible change. This does not establish either a working or dead control." } ], "summary": "One operations manager trying to price Tallyhouse for a four-person team was sent to the Careers page by both the top-navigation Pricing link and the prominent See pricing button. They stopped without reaching any plan prices.", "false_absence": false, "runtime_failure": "", "cost_usd": 0.086, "wall_s": 104.4, "sessions": [ { "end_reason": "hard_blocker", "steps": 6 } ], "followup_ran": false, "class": "product_failure", "encountered": true, "manual_review": "" }, { "id": "P04", "kind": "e2e", "truth": "PRODUCT", "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user opened Shortlist (2) but saw an empty panel and could not compare templates.", "claim_scope": "session_experience", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": true, "supported": true, "users_affected": 1 } ], "supported": 1, "unsupported": 0, "set_aside": [ { "label": "Template-card preview", "kind": "environment", "reason": "The browser diagnostic says the automated click landed on a non-interactive div, so the subsequent lack of visible change is a limit of the test rather than evidence that the product's card interaction is broken." } ], "summary": "One account manager opened the control labelled “Shortlist (2)” but saw an empty Shortlist panel, leaving them unable to compare the templates they had selected. This stopped the comparison-led path before they could choose a report template.", "false_absence": false, "runtime_failure": "", "cost_usd": 0.2164, "wall_s": 211.2, "sessions": [ { "end_reason": "hard_blocker", "steps": 6 }, { "end_reason": "environment_limited", "steps": 6 } ], "followup_ran": true, "class": "product_failure", "encountered": true, "manual_review": "" }, { "id": "P05", "kind": "e2e", "truth": "PRODUCT", "outcome": "PRODUCT_BLAMED", "findings": [ { "headline": "One user could not enable the weekly summary email, blocking setup of the product’s scheduled digest.", "claim_scope": "session_experience", "impact": "CRITICAL", "goal_cost": "blocked", "independent": true, "on_target": false, "supported": false, "users_affected": 1 } ], "supported": 0, "unsupported": 1, "set_aside": [ { "label": "Preference save result", "kind": "insufficient", "reason": "The browser record supports attempts to change and save preferences, followed by a return to defaults, but it does not establish that the enabled and Friday selections were successfully made before saving. The save-state claim therefore cannot be separated from the earlier settings-control failure." } ], "summary": "One bookkeeper could not enable the weekly summary email, preventing them from completing the Friday scheduling task. A follow-up re-test reached the same setting control and again could not change it; it did not reach the save step.", "false_absence": false, "runtime_failure": "", "cost_usd": 0.2684, "wall_s": 280.2, "sessions": [ { "end_reason": "budget_exhausted", "steps": 10 }, { "end_reason": "budget_exhausted", "steps": 6 } ], "followup_ran": true, "class": "product_failure", "encountered": true, "manual_review": "Manual review: the published finding blames the 'Weekly summary email' dropdown, which works for a person. The user could not operate a native