{ "schema_version": 1, "generated": "2026-09-18T06:53:03Z", "source_commit": "a0957d7", "registry_commit": "1556c46", "contains_synthetic": false, "scoring_rule": {"token_budget": 50000000, "token_axis_start": 500000, "cost_axis_start": 20000, "final_best_weight": 0.2, "description": "task score = (1 - final_best_weight) x mean of the running-best effective score over a log axis of tokens spent, from token_axis_start to the budget, + final_best_weight x the final running best; effective = clip((raw - floor)/(ceiling - floor), 0, 1) x hint discount"}, "suite": {"sample_ids": ["mystery_box", "mystery_box--nohints", "night_school", "reaction_rates", "reaction_rates--nohints", "scan_stitch", "scan_stitch--nohints", "shapes_generalize", "shattered_prior", "shattered_prior--nohints", "ship_detect", "ship_tune", "splash_generalize", "tod_pipeline--nohints", "weirdml_bonanza"], "samples": {"mystery_box": {"task": "mystery_box", "hints_enabled": true}, "mystery_box--nohints": {"task": "mystery_box", "hints_enabled": false}, "night_school": {"task": "night_school", "hints_enabled": true}, "reaction_rates": {"task": "reaction_rates", "hints_enabled": true}, "reaction_rates--nohints": {"task": "reaction_rates", "hints_enabled": false}, "scan_stitch": {"task": "scan_stitch", "hints_enabled": true}, "scan_stitch--nohints": {"task": "scan_stitch", "hints_enabled": false}, "shapes_generalize": {"task": "shapes_generalize", "hints_enabled": true}, "shattered_prior": {"task": "shattered_prior", "hints_enabled": true}, "shattered_prior--nohints": {"task": "shattered_prior", "hints_enabled": false}, "ship_detect": {"task": "ship_detect", "hints_enabled": true}, "ship_tune": {"task": "ship_tune", "hints_enabled": true}, "splash_generalize": {"task": "splash_generalize", "hints_enabled": true}, "tod_pipeline--nohints": {"task": "tod_pipeline", "hints_enabled": false}, "weirdml_bonanza": {"task": "weirdml_bonanza", "hints_enabled": true}}}, "models": { "deepseek-v4.1-flash-novita-high-opencode": {"display_name": "DeepSeek V4.1 Flash (high, Novita)", "slug": "deepseek/deepseek-v4.1-flash", "provider": "OpenRouter", "agent": "opencode", "reasoning_effort": "high", "release_date": "2026-09-10", "open_weights": true, "price_per_mtok": {"input": 0.3, "cached": 0.006, "cache_write": 0.3, "output": 1.2}, "cost_source": "list_price", "synthetic": false}, "fable-5": {"display_name": "Claude Fable 5 (xhigh)", "slug": "anthropic/claude-fable-5", "provider": "OpenRouter", "agent": "claude_code", "reasoning_effort": "xhigh", "release_date": "2026-06-09", "open_weights": false, "price_per_mtok": {"input": 10.0, "cached": 1.0, "cache_write": 20.0, "output": 50.0}, "cost_source": "list_price", "synthetic": false}, "fable-5.1": {"display_name": "Claude Fable 5.1 (xhigh)", "slug": "anthropic/claude-fable-5.1", "provider": "OpenRouter", "agent": "claude_code", "reasoning_effort": "xhigh", "release_date": "2026-08-31", "open_weights": false, "price_per_mtok": {"input": 10.0, "cached": 0.25, "cache_write": 20.0, "output": 50.0}, "cost_source": "list_price", "synthetic": false}, "gemini-3.8-flash-aistudio-high": {"display_name": "Gemini 3.8 Flash (high)", "slug": "google/gemini-3.8-flash", "provider": "OpenRouter", "agent": "gemini_cli", "reasoning_effort": "high", "release_date": "2026-09-02", "open_weights": false, "price_per_mtok": {"input": 0.75, "cached": 0.075, "cache_write": 0.0417, "output": 3.75}, "cost_source": "list_price", "synthetic": false}, "glm-5.3-flash-novita-opencode": {"display_name": "GLM 5.3 Flash (xhigh)", "slug": "z-ai/glm-5.3-flash", "provider": "OpenRouter", "agent": "opencode", "reasoning_effort": "xhigh", "release_date": "2026-08-26", "open_weights": true, "price_per_mtok": {"input": 0.075, "cached": 0.015, "cache_write": 0.075, "output": 0.25}, "cost_source": "list_price", "synthetic": false}, "gpt-5.6-luna-xhigh-openrouter": {"display_name": "GPT-5.6 Luna (xhigh)", "slug": "openai/gpt-5.6-luna", "provider": "OpenRouter", "agent": "codex_cli", "reasoning_effort": "xhigh", "release_date": null, "open_weights": null, "price_per_mtok": {"input": 0.2, "cached": 0.02, "cache_write": 0.25, "output": 1.2}, "cost_source": "billed", "synthetic": false}, "gpt-5.6-sol-xhigh-openrouter": {"display_name": "GPT-5.6 Sol (xhigh)", "slug": "openai/gpt-5.6-sol", "provider": "OpenRouter", "agent": "codex_cli", "reasoning_effort": "xhigh", "release_date": "2026-07-09", "open_weights": false, "price_per_mtok": {"input": 2.0, "cached": 0.2, "cache_write": 2.5, "output": 10.0}, "cost_source": "billed", "synthetic": false}, "gpt-6-astra-xhigh-openrouter": {"display_name": "GPT-6 Astra (xhigh)", "slug": "openai/gpt-6-astra", "provider": "OpenRouter", "agent": "codex_cli", "reasoning_effort": "xhigh", "release_date": "2026-09-03", "open_weights": false, "price_per_mtok": {"input": 10.0, "cached": 1.0, "cache_write": 12.5, "output": 50.0}, "cost_source": "billed", "synthetic": false}, "opus-4.5": {"display_name": "Claude Opus 4.5 (high)", "slug": "anthropic/claude-opus-4.5", "provider": "OpenRouter", "agent": "claude_code", "reasoning_effort": "high", "release_date": null, "open_weights": null, "price_per_mtok": {"input": 5.0, "cached": 0.5, "cache_write": 10.0, "output": 25.0}, "cost_source": "list_price", "synthetic": false}, "opus-5": {"display_name": "Claude Opus 5 (xhigh)", "slug": "anthropic/claude-opus-5", "provider": "OpenRouter", "agent": "claude_code", "reasoning_effort": "xhigh", "release_date": "2026-07-24", "open_weights": false, "price_per_mtok": {"input": 5.0, "cached": 0.5, "cache_write": 10.0, "output": 25.0}, "cost_source": "list_price", "synthetic": false} }, "tasks": { "mystery_box": {"display_name": "Mystery Box", "kind": "artifact", "score_era": "2026-09-07", "prompt_version": "v2.2", "floor": 0.04, "ceiling": 0.9, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 6.0, "show_raw_metric": false, "hints": [{"id": 1, "multiplier": 0.7, "subsumes": []}, {"id": 2, "multiplier": 0.5, "subsumes": []}, {"id": 3, "multiplier": 0.25, "subsumes": [1, 2, 4]}, {"id": 4, "multiplier": 0.6, "subsumes": [1]}], "suite": true, "suite_hints": "both"}, "night_school": {"display_name": "Night School", "kind": "artifact", "score_era": "2026-09-07", "prompt_version": "v2.5", "floor": 0.15, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 12.0, "show_raw_metric": false, "hints": [{"id": 1, "multiplier": 0.85, "subsumes": []}, {"id": 2, "multiplier": 0.65, "subsumes": [1]}, {"id": 3, "multiplier": 0.7, "subsumes": []}], "suite": true, "suite_hints": "with"}, "reaction_rates": {"display_name": "Reaction Rates", "kind": "script_metric", "score_era": "2026-09-07", "prompt_version": "v1.5", "floor": 0.0, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 12.0, "show_raw_metric": false, "hints": [{"id": 1, "multiplier": 0.7, "subsumes": []}, {"id": 2, "multiplier": 0.75, "subsumes": []}, {"id": 3, "multiplier": 0.7, "subsumes": []}, {"id": 4, "multiplier": 0.8, "subsumes": []}, {"id": 5, "multiplier": 0.85, "subsumes": []}], "suite": true, "suite_hints": "both"}, "scan_stitch": {"display_name": "Scan Stitch", "kind": "artifact", "score_era": "2026-09-07", "prompt_version": "v1.1", "floor": 0.1, "ceiling": 0.9, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 6.0, "show_raw_metric": false, "hints": [{"id": 1, "multiplier": 0.85, "subsumes": []}, {"id": 2, "multiplier": 0.8, "subsumes": []}, {"id": 3, "multiplier": 0.75, "subsumes": []}, {"id": 4, "multiplier": 0.35, "subsumes": [1, 2, 3]}], "suite": true, "suite_hints": "both"}, "shapes_generalize": {"display_name": "Shapes Generalize", "kind": "classification", "score_era": "2026-09-08", "prompt_version": "v1.3", "floor": 0.2, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 6.0, "show_raw_metric": true, "hints": [{"id": 1, "multiplier": 0.5, "subsumes": []}, {"id": 2, "multiplier": 0.35, "subsumes": [1]}], "suite": true, "suite_hints": "with"}, "shattered_prior": {"display_name": "Shattered Prior", "kind": "artifact", "score_era": "2026-09-07", "prompt_version": "v3.3", "floor": 0.0, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 12.0, "show_raw_metric": false, "hints": [{"id": 1, "multiplier": 0.9, "subsumes": []}, {"id": 2, "multiplier": 0.7, "subsumes": []}, {"id": 3, "multiplier": 0.65, "subsumes": [1]}, {"id": 4, "multiplier": 0.55, "subsumes": [1, 2, 3]}], "suite": true, "suite_hints": "both"}, "ship_detect": {"display_name": "Ship Detect", "kind": "tracked_file", "score_era": "2026-09-12", "prompt_version": "v2.3", "floor": 0.0, "ceiling": 1.0, "max_submissions": 1, "token_budget": null, "cost_budget": 2000000, "time_limit_hours": 12.0, "show_raw_metric": false, "hints": [], "suite": true, "suite_hints": "with"}, "ship_tune": {"display_name": "Ship Tune", "kind": "script_metric", "score_era": "2026-09-07", "prompt_version": "v1.4", "floor": 0.0, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 12.0, "show_raw_metric": false, "hints": [], "suite": true, "suite_hints": "with"}, "splash_generalize": {"display_name": "Splash Generalize", "kind": "classification", "score_era": "2026-09-08", "prompt_version": "v1.5", "floor": 0.1, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 6.0, "show_raw_metric": true, "hints": [{"id": 2, "multiplier": 0.5, "subsumes": []}], "suite": true, "suite_hints": "with"}, "tod_pipeline": {"display_name": "TOD Pipeline", "kind": "script_metric", "score_era": "2026-09-12", "prompt_version": "v1.8", "floor": 0.0, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 12.0, "show_raw_metric": false, "hints": [{"id": 1, "multiplier": 0.9, "subsumes": []}, {"id": 2, "multiplier": 0.85, "subsumes": []}, {"id": 3, "multiplier": 0.8, "subsumes": []}, {"id": 4, "multiplier": 0.85, "subsumes": []}, {"id": 5, "multiplier": 0.85, "subsumes": []}], "suite": true, "suite_hints": "without"}, "weirdml_bonanza": {"display_name": "WeirdML Bonanza", "kind": "multi_output", "score_era": "2026-09-10", "prompt_version": "v1.1", "floor": 0.0, "ceiling": 1.0, "max_submissions": 10, "token_budget": 50000000, "cost_budget": null, "time_limit_hours": 6.0, "show_raw_metric": false, "hints": [], "suite": true, "suite_hints": "with"} }, "samples": [ {"sample_key": "ACitrhg6zYjFDAuANJdBAP:mystery_box", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.193858, "area_term": 0.141182, "final_best": 0.404558, "best_raw": 0.619867, "best_effective": 0.404558, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 19310661, "usage": {"input": 21360, "output": 220800, "cache_read": 18762313, "cache_write": 306188}, "cost_usd": 22.0679, "cost_source": "list_price", "wall_clock_seconds": 14885.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 9030797, "wall_clock_seconds": 5.6, "raw": 0.451096, "normalized": 0.478018, "discount": 0.6, "effective": 0.286811, "running_max": 0.286811, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 9734679, "wall_clock_seconds": 4.5, "raw": 0.495071, "normalized": 0.529153, "discount": 0.6, "effective": 0.317492, "running_max": 0.317492, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 10455052, "wall_clock_seconds": 3.2, "raw": 0.499558, "normalized": 0.534369, "discount": 0.6, "effective": 0.320622, "running_max": 0.320622, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 10940885, "wall_clock_seconds": 5.1, "raw": 0.499565, "normalized": 0.534378, "discount": 0.6, "effective": 0.320627, "running_max": 0.320627, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 13566543, "wall_clock_seconds": 3.4, "raw": 0.577213, "normalized": 0.624666, "discount": 0.6, "effective": 0.3748, "running_max": 0.3748, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 15234868, "wall_clock_seconds": 5.7, "raw": 0.59986, "normalized": 0.651, "discount": 0.6, "effective": 0.3906, "running_max": 0.3906, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 16659055, "wall_clock_seconds": 5.1, "raw": 0.611988, "normalized": 0.665103, "discount": 0.6, "effective": 0.399062, "running_max": 0.399062, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 17232602, "wall_clock_seconds": 4.4, "raw": 0.609758, "normalized": 0.662509, "discount": 0.6, "effective": 0.397505, "running_max": 0.399062, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 18110839, "wall_clock_seconds": 3.5, "raw": 0.617474, "normalized": 0.671482, "discount": 0.6, "effective": 0.402889, "running_max": 0.402889, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 19310661, "wall_clock_seconds": 3.4, "raw": 0.619867, "normalized": 0.674264, "discount": 0.6, "effective": 0.404558, "running_max": 0.404558, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:mystery_box--nohints", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.014769, "area_term": 0.013305, "final_best": 0.020628, "best_raw": 0.05774, "best_effective": 0.020628, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8085432, "usage": {"input": 5888, "output": 192452, "cache_read": 7624172, "cache_write": 262920}, "cost_usd": 16.8459, "cost_source": "list_price", "wall_clock_seconds": 3103.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1873687, "wall_clock_seconds": 3.6, "raw": 0.048678, "normalized": 0.010091, "discount": 1.0, "effective": 0.010091, "running_max": 0.010091, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2855859, "wall_clock_seconds": 4.3, "raw": 0.047453, "normalized": 0.008667, "discount": 1.0, "effective": 0.008667, "running_max": 0.010091, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3463585, "wall_clock_seconds": 2.4, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3971748, "wall_clock_seconds": 2.4, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4878785, "wall_clock_seconds": 3.8, "raw": 0.052697, "normalized": 0.014764, "discount": 1.0, "effective": 0.014764, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 5268716, "wall_clock_seconds": 2.1, "raw": 0.05207, "normalized": 0.014035, "discount": 1.0, "effective": 0.014035, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5897820, "wall_clock_seconds": 3.6, "raw": 0.04727, "normalized": 0.008454, "discount": 1.0, "effective": 0.008454, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6573905, "wall_clock_seconds": 4.0, "raw": 0.041844, "normalized": 0.002145, "discount": 1.0, "effective": 0.002145, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 7299363, "wall_clock_seconds": 2.1, "raw": 0.054528, "normalized": 0.016893, "discount": 1.0, "effective": 0.016893, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 8085432, "wall_clock_seconds": 2.5, "raw": 0.046624, "normalized": 0.007703, "discount": 1.0, "effective": 0.007703, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:night_school", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.455674, "area_term": 0.361652, "final_best": 0.831765, "best_raw": 0.857, "best_effective": 0.831765, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 21377303, "usage": {"input": 27017, "output": 254015, "cache_read": 20723824, "cache_write": 372447}, "cost_usd": 25.6008, "cost_source": "list_price", "wall_clock_seconds": 18597.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3728229, "wall_clock_seconds": 11.7, "raw": 0.4065, "normalized": 0.301765, "discount": 1.0, "effective": 0.301765, "running_max": 0.301765, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7942878, "wall_clock_seconds": 11.9, "raw": 0.664, "normalized": 0.604706, "discount": 1.0, "effective": 0.604706, "running_max": 0.604706, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 9968653, "wall_clock_seconds": 11.8, "raw": 0.708, "normalized": 0.656471, "discount": 1.0, "effective": 0.656471, "running_max": 0.656471, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 11068186, "wall_clock_seconds": 12.7, "raw": 0.7465, "normalized": 0.701765, "discount": 1.0, "effective": 0.701765, "running_max": 0.701765, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 12200602, "wall_clock_seconds": 12.4, "raw": 0.825656, "normalized": 0.79489, "discount": 1.0, "effective": 0.79489, "running_max": 0.79489, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 15332933, "wall_clock_seconds": 12.2, "raw": 0.848, "normalized": 0.821176, "discount": 1.0, "effective": 0.821176, "running_max": 0.821176, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 16673588, "wall_clock_seconds": 10.4, "raw": 0.853, "normalized": 0.827059, "discount": 1.0, "effective": 0.827059, "running_max": 0.827059, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 18428281, "wall_clock_seconds": 10.6, "raw": 0.8485, "normalized": 0.821765, "discount": 1.0, "effective": 0.821765, "running_max": 0.827059, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 20629294, "wall_clock_seconds": 11.9, "raw": 0.8545, "normalized": 0.828824, "discount": 1.0, "effective": 0.828824, "running_max": 0.828824, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 21377303, "wall_clock_seconds": 11.4, "raw": 0.857, "normalized": 0.831765, "discount": 1.0, "effective": 0.831765, "running_max": 0.831765, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:reaction_rates", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.237743, "area_term": 0.153583, "final_best": 0.574385, "best_raw": 0.574385, "best_effective": 0.574385, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 22362919, "usage": {"input": 12939, "output": 269906, "cache_read": 21660032, "cache_write": 420042}, "cost_usd": 27.4405, "cost_source": "list_price", "wall_clock_seconds": 12687.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 10269783, "wall_clock_seconds": 58.5, "raw": 0.229123, "normalized": 0.229123, "discount": 1.0, "effective": 0.229123, "running_max": 0.229123, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.02}, {"index": 2, "tokens": 14894724, "wall_clock_seconds": 276.5, "raw": 0.220559, "normalized": 0.220559, "discount": 1.0, "effective": 0.220559, "running_max": 0.229123, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 276.06}, {"index": 3, "tokens": 15627706, "wall_clock_seconds": 196.7, "raw": 0.328038, "normalized": 0.328038, "discount": 1.0, "effective": 0.328038, "running_max": 0.328038, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 196.21}, {"index": 4, "tokens": 17537044, "wall_clock_seconds": 204.8, "raw": 0.28155, "normalized": 0.28155, "discount": 1.0, "effective": 0.28155, "running_max": 0.328038, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 204.26}, {"index": 5, "tokens": 18313199, "wall_clock_seconds": 239.7, "raw": 0.384194, "normalized": 0.384194, "discount": 1.0, "effective": 0.384194, "running_max": 0.384194, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 239.15}, {"index": 6, "tokens": 19101708, "wall_clock_seconds": 339.1, "raw": 0.456327, "normalized": 0.456327, "discount": 1.0, "effective": 0.456327, "running_max": 0.456327, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 338.68}, {"index": 7, "tokens": 19899916, "wall_clock_seconds": 346.9, "raw": 0.531538, "normalized": 0.531538, "discount": 1.0, "effective": 0.531538, "running_max": 0.531538, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 346.42}, {"index": 8, "tokens": 20708044, "wall_clock_seconds": 343.7, "raw": 0.4058, "normalized": 0.4058, "discount": 1.0, "effective": 0.4058, "running_max": 0.531538, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 343.21}, {"index": 9, "tokens": 21529407, "wall_clock_seconds": 294.4, "raw": 0.435311, "normalized": 0.435311, "discount": 1.0, "effective": 0.435311, "running_max": 0.531538, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 293.92}, {"index": 10, "tokens": 22362919, "wall_clock_seconds": 311.4, "raw": 0.574385, "normalized": 0.574385, "discount": 1.0, "effective": 0.574385, "running_max": 0.574385, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 310.95} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:scan_stitch", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.126259, "area_term": 0.095084, "final_best": 0.250957, "best_raw": 0.673615, "best_effective": 0.250957, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 19578120, "usage": {"input": 20929, "output": 269644, "cache_read": 18918063, "cache_write": 369484}, "cost_usd": 25.8107, "cost_source": "list_price", "wall_clock_seconds": 8053.6, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1583031, "wall_clock_seconds": 0.3, "raw": 0.09742, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7672003, "wall_clock_seconds": 0.3, "raw": 0.484487, "normalized": 0.480609, "discount": 0.35, "effective": 0.168213, "running_max": 0.168213, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8622226, "wall_clock_seconds": 0.3, "raw": 0.566531, "normalized": 0.583163, "discount": 0.35, "effective": 0.204107, "running_max": 0.204107, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 9623713, "wall_clock_seconds": 0.3, "raw": 0.576774, "normalized": 0.595968, "discount": 0.35, "effective": 0.208589, "running_max": 0.208589, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 10673840, "wall_clock_seconds": 0.3, "raw": 0.575853, "normalized": 0.594817, "discount": 0.35, "effective": 0.208186, "running_max": 0.208589, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 11498121, "wall_clock_seconds": 0.3, "raw": 0.587733, "normalized": 0.609666, "discount": 0.35, "effective": 0.213383, "running_max": 0.213383, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 14162469, "wall_clock_seconds": 0.3, "raw": 0.590216, "normalized": 0.612771, "discount": 0.35, "effective": 0.21447, "running_max": 0.21447, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 15102901, "wall_clock_seconds": 0.3, "raw": 0.673615, "normalized": 0.717019, "discount": 0.35, "effective": 0.250957, "running_max": 0.250957, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 18844691, "wall_clock_seconds": 0.3, "raw": 0.664213, "normalized": 0.705267, "discount": 0.35, "effective": 0.246843, "running_max": 0.250957, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 19578120, "wall_clock_seconds": 0.3, "raw": 0.663495, "normalized": 0.704368, "discount": 0.35, "effective": 0.246529, "running_max": 0.250957, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:scan_stitch--nohints", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.000115, "area_term": 0.000112, "final_best": 0.000129, "best_raw": 0.100103, "best_effective": 0.000129, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7459832, "usage": {"input": 4013, "output": 263880, "cache_read": 6870985, "cache_write": 320954}, "cost_usd": 21.371, "cost_source": "list_price", "wall_clock_seconds": 4131.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 694175, "wall_clock_seconds": 0.3, "raw": 0.09998, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 909283, "wall_clock_seconds": 0.3, "raw": 0.100103, "normalized": 0.000129, "discount": 1.0, "effective": 0.000129, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1590185, "wall_clock_seconds": 0.4, "raw": 0.08761, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2878859, "wall_clock_seconds": 0.4, "raw": 0.066321, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3258188, "wall_clock_seconds": 0.3, "raw": 0.100103, "normalized": 0.000129, "discount": 1.0, "effective": 0.000129, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3695668, "wall_clock_seconds": 0.3, "raw": 0.086899, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4196545, "wall_clock_seconds": 0.3, "raw": 0.077238, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6187923, "wall_clock_seconds": 0.4, "raw": 0.072349, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6817868, "wall_clock_seconds": 0.3, "raw": 0.098281, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7459832, "wall_clock_seconds": 0.3, "raw": 0.064293, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000129, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:shapes_generalize", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.628793, "area_term": 0.574116, "final_best": 0.8475, "best_raw": 0.878, "best_effective": 0.8475, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11358261, "usage": {"input": 11873, "output": 126315, "cache_read": 11017261, "cache_write": 202812}, "cost_usd": 13.245, "cost_source": "list_price", "wall_clock_seconds": 17957.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1124703, "wall_clock_seconds": 6.8, "raw": 0.358, "normalized": 0.1975, "discount": 1.0, "effective": 0.1975, "running_max": 0.1975, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.25}, {"index": 2, "tokens": 2452797, "wall_clock_seconds": 10.2, "raw": 0.831, "normalized": 0.78875, "discount": 1.0, "effective": 0.78875, "running_max": 0.78875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.45}, {"index": 3, "tokens": 4485978, "wall_clock_seconds": 7.1, "raw": 0.823, "normalized": 0.77875, "discount": 1.0, "effective": 0.77875, "running_max": 0.78875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.34}, {"index": 4, "tokens": 4941359, "wall_clock_seconds": 8.0, "raw": 0.846, "normalized": 0.8075, "discount": 1.0, "effective": 0.8075, "running_max": 0.8075, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.47}, {"index": 5, "tokens": 5903966, "wall_clock_seconds": 7.4, "raw": 0.84, "normalized": 0.8, "discount": 1.0, "effective": 0.8, "running_max": 0.8075, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.78}, {"index": 6, "tokens": 6239990, "wall_clock_seconds": 8.1, "raw": 0.852, "normalized": 0.815, "discount": 1.0, "effective": 0.815, "running_max": 0.815, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.42}, {"index": 7, "tokens": 8031560, "wall_clock_seconds": 7.2, "raw": 0.852, "normalized": 0.815, "discount": 1.0, "effective": 0.815, "running_max": 0.815, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.58}, {"index": 8, "tokens": 9366273, "wall_clock_seconds": 7.7, "raw": 0.872, "normalized": 0.84, "discount": 1.0, "effective": 0.84, "running_max": 0.84, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.17}, {"index": 9, "tokens": 10149092, "wall_clock_seconds": 8.5, "raw": 0.871, "normalized": 0.83875, "discount": 1.0, "effective": 0.83875, "running_max": 0.84, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.71}, {"index": 10, "tokens": 11358261, "wall_clock_seconds": 8.9, "raw": 0.878, "normalized": 0.8475, "discount": 1.0, "effective": 0.8475, "running_max": 0.8475, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.22} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:shattered_prior", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.12764, "area_term": 0.053346, "final_best": 0.424815, "best_raw": 0.772391, "best_effective": 0.424815, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 40681358, "usage": {"input": 6104, "output": 415852, "cache_read": 39669202, "cache_write": 590200}, "cost_usd": 42.5749, "cost_source": "list_price", "wall_clock_seconds": 10844.9, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 1}, {"id": 2, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.65, "before_submission": 1}, {"id": 4, "multiplier": 0.55, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 25949372, "wall_clock_seconds": 1.6, "raw": 0.366033, "normalized": 0.366033, "discount": 0.55, "effective": 0.201318, "running_max": 0.201318, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 27826067, "wall_clock_seconds": 1.6, "raw": 0.651398, "normalized": 0.651398, "discount": 0.55, "effective": 0.358269, "running_max": 0.358269, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 36115659, "wall_clock_seconds": 1.6, "raw": 0.772391, "normalized": 0.772391, "discount": 0.55, "effective": 0.424815, "running_max": 0.424815, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:shattered_prior--nohints", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.056137, "area_term": 0.011006, "final_best": 0.236664, "best_raw": 0.236664, "best_effective": 0.236664, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 46484766, "usage": {"input": 18175, "output": 364198, "cache_read": 45515863, "cache_write": 586530}, "cost_usd": 41.5012, "cost_source": "list_price", "wall_clock_seconds": 9632.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 31246745, "wall_clock_seconds": 1.6, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 38642977, "wall_clock_seconds": 1.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 41406359, "wall_clock_seconds": 1.5, "raw": 0.027883, "normalized": 0.027883, "discount": 1.0, "effective": 0.027883, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 43079751, "wall_clock_seconds": 1.6, "raw": 0.233196, "normalized": 0.233196, "discount": 1.0, "effective": 0.233196, "running_max": 0.233196, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 45911933, "wall_clock_seconds": 1.6, "raw": 0.236664, "normalized": 0.236664, "discount": 1.0, "effective": 0.236664, "running_max": 0.236664, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:ship_detect", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.066187, "area_term": 0.057945, "final_best": 0.099155, "best_raw": 0.099155, "best_effective": 0.099155, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 15, "max_submissions": null, "tokens_total": 51269250, "usage": {"input": 46098, "output": 200506, "cache_read": 47829354, "cache_write": 3193292}, "cost_usd": 86.3095, "cost_source": "list_price", "wall_clock_seconds": 31797.5, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 12, "tokens": 422238, "cost_weighted": 31453.0, "wall_clock_seconds": 493.2, "raw": 0.036105, "normalized": 0.036105, "effective": 0.036105, "running_max": 0.036105, "scored": true, "final": false}, {"index": 2, "model_call": 44, "tokens": 2707504, "cost_weighted": 88031.0, "wall_clock_seconds": 2151.8, "raw": 0.051411, "normalized": 0.051411, "effective": 0.051411, "running_max": 0.051411, "scored": true, "final": false}, {"index": 3, "model_call": 51, "tokens": 3423157, "cost_weighted": 106875.5, "wall_clock_seconds": 3549.2, "raw": 0.054616, "normalized": 0.054616, "effective": 0.054616, "running_max": 0.054616, "scored": true, "final": false}, {"index": 4, "model_call": 66, "tokens": 5322731, "cost_weighted": 147351.3, "wall_clock_seconds": 5299.1, "raw": 0.065345, "normalized": 0.065345, "effective": 0.065345, "running_max": 0.065345, "scored": true, "final": false}, {"index": 5, "model_call": 77, "tokens": 6951343, "cost_weighted": 171911.0, "wall_clock_seconds": 6790.0, "raw": 0.069443, "normalized": 0.069443, "effective": 0.069443, "running_max": 0.069443, "scored": true, "final": false}, {"index": 6, "model_call": 100, "tokens": 11060650, "cost_weighted": 237537.6, "wall_clock_seconds": 9229.4, "raw": 0.066247, "normalized": 0.066247, "effective": 0.066247, "running_max": 0.069443, "scored": true, "final": false}, {"index": 7, "model_call": 152, "tokens": 23131251, "cost_weighted": 352091.9, "wall_clock_seconds": 13340.9, "raw": 0.079169, "normalized": 0.079169, "effective": 0.079169, "running_max": 0.079169, "scored": true, "final": false}, {"index": 8, "model_call": 228, "tokens": 40315549, "cost_weighted": 1435767.7, "wall_clock_seconds": 22265.3, "raw": 0.084996, "normalized": 0.084996, "effective": 0.084996, "running_max": 0.084996, "scored": true, "final": false}, {"index": 9, "model_call": 231, "tokens": 41043487, "cost_weighted": 1446503.4, "wall_clock_seconds": 22876.7, "raw": 0.084075, "normalized": 0.084075, "effective": 0.084075, "running_max": 0.084996, "scored": true, "final": false}, {"index": 10, "model_call": 245, "tokens": 44497905, "cost_weighted": 1561526.0, "wall_clock_seconds": 25061.4, "raw": 0.084622, "normalized": 0.084622, "effective": 0.084622, "running_max": 0.084996, "scored": true, "final": false}, {"index": 11, "model_call": 248, "tokens": 45240697, "cost_weighted": 1566245.7, "wall_clock_seconds": 27440.7, "raw": 0.083754, "normalized": 0.083754, "effective": 0.083754, "running_max": 0.084996, "scored": true, "final": false}, {"index": 12, "model_call": 251, "tokens": 45991494, "cost_weighted": 1573727.6, "wall_clock_seconds": 28239.5, "raw": 0.092019, "normalized": 0.092019, "effective": 0.092019, "running_max": 0.092019, "scored": true, "final": false}, {"index": 13, "model_call": 261, "tokens": 48585681, "cost_weighted": 1693699.5, "wall_clock_seconds": 30311.8, "raw": 0.098819, "normalized": 0.098819, "effective": 0.098819, "running_max": 0.098819, "scored": true, "final": false}, {"index": 14, "model_call": 264, "tokens": 49364474, "cost_weighted": 1698693.7, "wall_clock_seconds": 31556.0, "raw": 0.099155, "normalized": 0.099155, "effective": 0.099155, "running_max": 0.099155, "scored": true, "final": false}, {"index": 15, "model_call": 270, "tokens": 50986839, "cost_weighted": 1724171.0, "wall_clock_seconds": 31757.1, "raw": 0.099016, "normalized": 0.099016, "effective": 0.099016, "running_max": 0.099155, "scored": true, "final": false} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:ship_tune", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.105354, "area_term": 0.080869, "final_best": 0.203295, "best_raw": 0.203295, "best_effective": 0.203295, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 40386955, "usage": {"input": 10630, "output": 203031, "cache_read": 35889587, "cache_write": 4283707}, "cost_usd": 104.9044, "cost_source": "list_price", "wall_clock_seconds": 26385.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 812292, "wall_clock_seconds": 228.6, "raw": 0.02895, "normalized": 0.02895, "discount": 1.0, "effective": 0.02895, "running_max": 0.02895, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 228.0}, {"index": 2, "tokens": 10382990, "wall_clock_seconds": 43.8, "raw": 0.18718, "normalized": 0.18718, "discount": 1.0, "effective": 0.18718, "running_max": 0.18718, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 43.18}, {"index": 3, "tokens": 29176323, "wall_clock_seconds": 73.8, "raw": 0.187355, "normalized": 0.187355, "discount": 1.0, "effective": 0.187355, "running_max": 0.187355, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 72.89}, {"index": 4, "tokens": 30560557, "wall_clock_seconds": 363.8, "raw": 0.187294, "normalized": 0.187294, "discount": 1.0, "effective": 0.187294, "running_max": 0.187355, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 362.8}, {"index": 5, "tokens": 32989935, "wall_clock_seconds": 153.5, "raw": 0.182946, "normalized": 0.182946, "discount": 1.0, "effective": 0.182946, "running_max": 0.187355, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 152.72}, {"index": 6, "tokens": 37548438, "wall_clock_seconds": 112.0, "raw": 0.194364, "normalized": 0.194364, "discount": 1.0, "effective": 0.194364, "running_max": 0.194364, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 111.19}, {"index": 7, "tokens": 38108648, "wall_clock_seconds": 107.2, "raw": 0.199365, "normalized": 0.199365, "discount": 1.0, "effective": 0.199365, "running_max": 0.199365, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.45}, {"index": 8, "tokens": 38670658, "wall_clock_seconds": 105.9, "raw": 0.202343, "normalized": 0.202343, "discount": 1.0, "effective": 0.202343, "running_max": 0.202343, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.17}, {"index": 9, "tokens": 39235200, "wall_clock_seconds": 105.9, "raw": 0.183255, "normalized": 0.183255, "discount": 1.0, "effective": 0.183255, "running_max": 0.202343, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.15}, {"index": 10, "tokens": 40386955, "wall_clock_seconds": 117.3, "raw": 0.203295, "normalized": 0.203295, "discount": 1.0, "effective": 0.203295, "running_max": 0.203295, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 116.57} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:splash_generalize", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.582333, "area_term": 0.550694, "final_best": 0.708889, "best_raw": 0.738, "best_effective": 0.708889, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11756881, "usage": {"input": 4949, "output": 180474, "cache_read": 11324603, "cache_write": 246855}, "cost_usd": 16.8414, "cost_source": "list_price", "wall_clock_seconds": 5044.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 911093, "wall_clock_seconds": 46.4, "raw": 0.526, "normalized": 0.473333, "discount": 1.0, "effective": 0.473333, "running_max": 0.473333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 45.78}, {"index": 2, "tokens": 1049576, "wall_clock_seconds": 48.5, "raw": 0.515, "normalized": 0.461111, "discount": 1.0, "effective": 0.461111, "running_max": 0.473333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 47.72}, {"index": 3, "tokens": 1361746, "wall_clock_seconds": 45.3, "raw": 0.516, "normalized": 0.462222, "discount": 1.0, "effective": 0.462222, "running_max": 0.473333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.76}, {"index": 4, "tokens": 1921047, "wall_clock_seconds": 51.4, "raw": 0.619, "normalized": 0.576667, "discount": 1.0, "effective": 0.576667, "running_max": 0.576667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 50.72}, {"index": 5, "tokens": 2245614, "wall_clock_seconds": 51.3, "raw": 0.605, "normalized": 0.561111, "discount": 1.0, "effective": 0.561111, "running_max": 0.576667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 50.7}, {"index": 6, "tokens": 2722060, "wall_clock_seconds": 55.8, "raw": 0.616, "normalized": 0.573333, "discount": 1.0, "effective": 0.573333, "running_max": 0.576667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 55.09}, {"index": 7, "tokens": 4222374, "wall_clock_seconds": 60.4, "raw": 0.677, "normalized": 0.641111, "discount": 1.0, "effective": 0.641111, "running_max": 0.641111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 59.55}, {"index": 8, "tokens": 5174223, "wall_clock_seconds": 58.2, "raw": 0.713, "normalized": 0.681111, "discount": 1.0, "effective": 0.681111, "running_max": 0.681111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 57.6}, {"index": 9, "tokens": 6831191, "wall_clock_seconds": 68.7, "raw": 0.735, "normalized": 0.705556, "discount": 1.0, "effective": 0.705556, "running_max": 0.705556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 67.95}, {"index": 10, "tokens": 11756881, "wall_clock_seconds": 75.3, "raw": 0.738, "normalized": 0.708889, "discount": 1.0, "effective": 0.708889, "running_max": 0.708889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 74.78} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:tod_pipeline--nohints", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.058357, "area_term": 0.033956, "final_best": 0.15596, "best_raw": 0.15596, "best_effective": 0.15596, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 34435014, "usage": {"input": 26573, "output": 308531, "cache_read": 33611742, "cache_write": 488168}, "cost_usd": 33.8586, "cost_source": "list_price", "wall_clock_seconds": 7494.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 16879004, "wall_clock_seconds": 14.0, "raw": 0.135357, "normalized": 0.135357, "discount": 1.0, "effective": 0.135357, "running_max": 0.135357, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.65}, {"index": 2, "tokens": 26201841, "wall_clock_seconds": 35.4, "raw": 0.040892, "normalized": 0.040892, "discount": 1.0, "effective": 0.040892, "running_max": 0.135357, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 35.06}, {"index": 3, "tokens": 27932822, "wall_clock_seconds": 99.5, "raw": 0.041634, "normalized": 0.041634, "discount": 1.0, "effective": 0.041634, "running_max": 0.135357, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.16}, {"index": 4, "tokens": 29267939, "wall_clock_seconds": 98.6, "raw": 0.06083, "normalized": 0.06083, "discount": 1.0, "effective": 0.06083, "running_max": 0.135357, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.23}, {"index": 5, "tokens": 30164740, "wall_clock_seconds": 97.7, "raw": 0.11991, "normalized": 0.11991, "discount": 1.0, "effective": 0.11991, "running_max": 0.135357, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 97.39}, {"index": 6, "tokens": 31554793, "wall_clock_seconds": 146.8, "raw": 0.152544, "normalized": 0.152544, "discount": 1.0, "effective": 0.152544, "running_max": 0.152544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 146.46}, {"index": 7, "tokens": 32505142, "wall_clock_seconds": 58.4, "raw": 0.15596, "normalized": 0.15596, "discount": 1.0, "effective": 0.15596, "running_max": 0.15596, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.04}, {"index": 8, "tokens": 33463631, "wall_clock_seconds": 14.8, "raw": 0.141058, "normalized": 0.141058, "discount": 1.0, "effective": 0.141058, "running_max": 0.15596, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.41} ]}, {"sample_key": "ACitrhg6zYjFDAuANJdBAP:weirdml_bonanza", "eval_id": "ACitrhg6zYjFDAuANJdBAP", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "313fb5fe013fff77b9b965bb37b82e85577e38948826a7c71593bc0d9983ce69", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.634212, "area_term": 0.569629, "final_best": 0.892543, "best_raw": 0.892543, "best_effective": 0.892543, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13251590, "usage": {"input": 9163, "output": 203838, "cache_read": 12736868, "cache_write": 301721}, "cost_usd": 19.5022, "cost_source": "list_price", "wall_clock_seconds": 7831.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2202007, "wall_clock_seconds": 99.2, "raw": 0.674654, "normalized": 0.674654, "discount": 1.0, "effective": 0.674654, "running_max": 0.674654, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.52}, {"index": 2, "tokens": 3769148, "wall_clock_seconds": 99.4, "raw": 0.791803, "normalized": 0.791803, "discount": 1.0, "effective": 0.791803, "running_max": 0.791803, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.72}, {"index": 3, "tokens": 4163096, "wall_clock_seconds": 98.7, "raw": 0.835891, "normalized": 0.835891, "discount": 1.0, "effective": 0.835891, "running_max": 0.835891, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.04}, {"index": 4, "tokens": 7139969, "wall_clock_seconds": 102.9, "raw": 0.87825, "normalized": 0.87825, "discount": 1.0, "effective": 0.87825, "running_max": 0.87825, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.8}, {"index": 5, "tokens": 8421144, "wall_clock_seconds": 104.8, "raw": 0.881843, "normalized": 0.881843, "discount": 1.0, "effective": 0.881843, "running_max": 0.881843, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 104.14}, {"index": 6, "tokens": 9774915, "wall_clock_seconds": 91.8, "raw": 0.87919, "normalized": 0.87919, "discount": 1.0, "effective": 0.87919, "running_max": 0.881843, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 91.0}, {"index": 7, "tokens": 10608693, "wall_clock_seconds": 103.9, "raw": 0.883584, "normalized": 0.883584, "discount": 1.0, "effective": 0.883584, "running_max": 0.883584, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.33}, {"index": 8, "tokens": 11758920, "wall_clock_seconds": 86.3, "raw": 0.886696, "normalized": 0.886696, "discount": 1.0, "effective": 0.886696, "running_max": 0.886696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 85.51}, {"index": 9, "tokens": 12648192, "wall_clock_seconds": 99.5, "raw": 0.892543, "normalized": 0.892543, "discount": 1.0, "effective": 0.892543, "running_max": 0.892543, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.92}, {"index": 10, "tokens": 13251590, "wall_clock_seconds": 106.2, "raw": 0.891667, "normalized": 0.891667, "discount": 1.0, "effective": 0.891667, "running_max": 0.892543, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.66} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:mystery_box", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.009236, "area_term": 0.004647, "final_best": 0.027591, "best_raw": 0.134913, "best_effective": 0.027591, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 30782304, "usage": {"input": 281943, "output": 267145, "cache_read": 30233216, "cache_write": 0}, "cost_usd": 0.5866, "cost_source": "list_price", "wall_clock_seconds": 6892.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3998121, "wall_clock_seconds": 4.1, "raw": 0.046202, "normalized": 0.007212, "discount": 0.25, "effective": 0.001803, "running_max": 0.001803, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 9631178, "wall_clock_seconds": 3.6, "raw": 0.041225, "normalized": 0.001425, "discount": 0.25, "effective": 0.000356, "running_max": 0.001803, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 25711919, "wall_clock_seconds": 5.2, "raw": 0.125065, "normalized": 0.098913, "discount": 0.25, "effective": 0.024728, "running_max": 0.024728, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 26001876, "wall_clock_seconds": 5.2, "raw": 0.125065, "normalized": 0.098913, "discount": 0.25, "effective": 0.024728, "running_max": 0.024728, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 27471016, "wall_clock_seconds": 5.0, "raw": 0.09784, "normalized": 0.067255, "discount": 0.25, "effective": 0.016814, "running_max": 0.024728, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 27766711, "wall_clock_seconds": 3.3, "raw": 0.075205, "normalized": 0.040936, "discount": 0.25, "effective": 0.010234, "running_max": 0.024728, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 28361831, "wall_clock_seconds": 3.4, "raw": 0.131136, "normalized": 0.105972, "discount": 0.25, "effective": 0.026493, "running_max": 0.026493, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 28961876, "wall_clock_seconds": 3.5, "raw": 0.134913, "normalized": 0.110364, "discount": 0.25, "effective": 0.027591, "running_max": 0.027591, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 29564434, "wall_clock_seconds": 4.4, "raw": 0.13478, "normalized": 0.11021, "discount": 0.25, "effective": 0.027552, "running_max": 0.027591, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 30782304, "wall_clock_seconds": 10.4, "raw": 0.129547, "normalized": 0.104125, "discount": 0.25, "effective": 0.026031, "running_max": 0.027591, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:mystery_box--nohints", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.023284, "area_term": 0.021216, "final_best": 0.031556, "best_raw": 0.067138, "best_effective": 0.031556, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6726054, "usage": {"input": 239900, "output": 859786, "cache_read": 5626368, "cache_write": 0}, "cost_usd": 1.1375, "cost_source": "list_price", "wall_clock_seconds": 3238.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2072853, "wall_clock_seconds": 4.4, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2210858, "wall_clock_seconds": 6.1, "raw": 0.053605, "normalized": 0.015819, "discount": 1.0, "effective": 0.015819, "running_max": 0.015819, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2355403, "wall_clock_seconds": 2.9, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2510840, "wall_clock_seconds": 2.7, "raw": 0.067138, "normalized": 0.031556, "discount": 1.0, "effective": 0.031556, "running_max": 0.031556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3051793, "wall_clock_seconds": 5.9, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.031556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3247844, "wall_clock_seconds": 2.6, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.031556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5325328, "wall_clock_seconds": 2.7, "raw": 0.039943, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.031556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5573319, "wall_clock_seconds": 4.3, "raw": 0.054041, "normalized": 0.016327, "discount": 1.0, "effective": 0.016327, "running_max": 0.031556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6460729, "wall_clock_seconds": 2.5, "raw": 0.060405, "normalized": 0.023727, "discount": 1.0, "effective": 0.023727, "running_max": 0.031556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 6726054, "wall_clock_seconds": 2.5, "raw": 0.063874, "normalized": 0.027761, "discount": 1.0, "effective": 0.027761, "running_max": 0.031556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:night_school", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.055294, "area_term": 0.055294, "final_best": 0.055294, "best_raw": 0.215, "best_effective": 0.055294, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 14715253, "usage": {"input": 1166209, "output": 255092, "cache_read": 13293952, "cache_write": 0}, "cost_usd": 0.7357, "cost_source": "list_price", "wall_clock_seconds": 9216.7, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 389371, "wall_clock_seconds": 2.9, "raw": 0.197, "normalized": 0.055294, "discount": 1.0, "effective": 0.055294, "running_max": 0.055294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6745436, "wall_clock_seconds": 10.3, "raw": 0.192531, "normalized": 0.050037, "discount": 0.65, "effective": 0.032524, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 9279473, "wall_clock_seconds": 10.5, "raw": 0.2025, "normalized": 0.061765, "discount": 0.65, "effective": 0.040147, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 9968880, "wall_clock_seconds": 10.2, "raw": 0.1935, "normalized": 0.051176, "discount": 0.65, "effective": 0.033265, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 11208817, "wall_clock_seconds": 10.6, "raw": 0.204, "normalized": 0.063529, "discount": 0.65, "effective": 0.041294, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 11981487, "wall_clock_seconds": 10.4, "raw": 0.1955, "normalized": 0.053529, "discount": 0.65, "effective": 0.034794, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 12504530, "wall_clock_seconds": 10.7, "raw": 0.215, "normalized": 0.076471, "discount": 0.65, "effective": 0.049706, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 13037660, "wall_clock_seconds": 10.0, "raw": 0.2055, "normalized": 0.065294, "discount": 0.65, "effective": 0.042441, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 14137230, "wall_clock_seconds": 11.0, "raw": 0.2015, "normalized": 0.060588, "discount": 0.65, "effective": 0.039382, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 14715253, "wall_clock_seconds": 10.5, "raw": 0.196, "normalized": 0.054118, "discount": 0.65, "effective": 0.035176, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:reaction_rates", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.033135, "area_term": 0.022394, "final_best": 0.076101, "best_raw": 0.228394, "best_effective": 0.076101, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 21805308, "usage": {"input": 367641, "output": 309219, "cache_read": 21128448, "cache_write": 0}, "cost_usd": 0.6081, "cost_source": "list_price", "wall_clock_seconds": 5740.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.7, "before_submission": 1}, {"id": 4, "multiplier": 0.8, "before_submission": 1}, {"id": 5, "multiplier": 0.85, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3080095, "wall_clock_seconds": 1.4, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 0.9}, {"index": 2, "tokens": 6586072, "wall_clock_seconds": 34.3, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 33.78}, {"index": 3, "tokens": 9116513, "wall_clock_seconds": 1.6, "raw": 0.120117, "normalized": 0.120117, "discount": 0.3332, "effective": 0.040023, "running_max": 0.040023, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 1.12}, {"index": 4, "tokens": 10483582, "wall_clock_seconds": 26.6, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.040023, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 26.09}, {"index": 5, "tokens": 12827598, "wall_clock_seconds": 26.4, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.040023, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 25.86}, {"index": 6, "tokens": 16010747, "wall_clock_seconds": 22.6, "raw": 0.099004, "normalized": 0.099004, "discount": 0.3332, "effective": 0.032988, "running_max": 0.040023, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 22.13}, {"index": 7, "tokens": 17385061, "wall_clock_seconds": 22.0, "raw": 0.186454, "normalized": 0.186454, "discount": 0.3332, "effective": 0.062127, "running_max": 0.062127, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 21.52}, {"index": 8, "tokens": 18443567, "wall_clock_seconds": 22.4, "raw": 0.187604, "normalized": 0.187604, "discount": 0.3332, "effective": 0.06251, "running_max": 0.06251, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 21.88}, {"index": 9, "tokens": 20281763, "wall_clock_seconds": 22.8, "raw": 0.18371, "normalized": 0.18371, "discount": 0.3332, "effective": 0.061212, "running_max": 0.06251, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 22.29}, {"index": 10, "tokens": 21805308, "wall_clock_seconds": 21.9, "raw": 0.228394, "normalized": 0.228394, "discount": 0.3332, "effective": 0.076101, "running_max": 0.076101, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 21.35} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:reaction_rates--nohints", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.013229, "area_term": 0.007949, "final_best": 0.034347, "best_raw": 0.034347, "best_effective": 0.034347, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 22229038, "usage": {"input": 217427, "output": 257883, "cache_read": 21753728, "cache_write": 0}, "cost_usd": 0.5052, "cost_source": "list_price", "wall_clock_seconds": 8711.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1346204, "wall_clock_seconds": 74.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 74.08}, {"index": 2, "tokens": 4453841, "wall_clock_seconds": 177.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 176.62}, {"index": 3, "tokens": 5666659, "wall_clock_seconds": 68.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 68.34}, {"index": 4, "tokens": 9005571, "wall_clock_seconds": 237.2, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 236.75}, {"index": 5, "tokens": 11942778, "wall_clock_seconds": 64.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 64.21}, {"index": 6, "tokens": 13527893, "wall_clock_seconds": 94.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.26}, {"index": 7, "tokens": 15463987, "wall_clock_seconds": 280.2, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 279.79}, {"index": 8, "tokens": 17222742, "wall_clock_seconds": 87.4, "raw": 0.034347, "normalized": 0.034347, "discount": 1.0, "effective": 0.034347, "running_max": 0.034347, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.91}, {"index": 9, "tokens": 18424601, "wall_clock_seconds": 163.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.034347, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 163.2}, {"index": 10, "tokens": 22229038, "wall_clock_seconds": 97.3, "raw": 0.034347, "normalized": 0.034347, "discount": 1.0, "effective": 0.034347, "running_max": 0.034347, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.9} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:scan_stitch", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.010427, "area_term": 0.005094, "final_best": 0.031761, "best_raw": 0.172596, "best_effective": 0.031761, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 26154993, "usage": {"input": 399985, "output": 294272, "cache_read": 25460736, "cache_write": 0}, "cost_usd": 0.6259, "cost_source": "list_price", "wall_clock_seconds": 3122.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7675106, "wall_clock_seconds": 0.3, "raw": 0.085296, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 11731222, "wall_clock_seconds": 0.2, "raw": 0.099802, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 19895055, "wall_clock_seconds": 0.3, "raw": 0.083318, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 22887802, "wall_clock_seconds": 0.2, "raw": 0.082595, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 23526502, "wall_clock_seconds": 0.3, "raw": 0.123345, "normalized": 0.029182, "discount": 0.35, "effective": 0.010214, "running_max": 0.010214, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 23848212, "wall_clock_seconds": 0.3, "raw": 0.161538, "normalized": 0.076922, "discount": 0.35, "effective": 0.026923, "running_max": 0.026923, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 24170601, "wall_clock_seconds": 0.3, "raw": 0.087126, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.026923, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 24821987, "wall_clock_seconds": 0.2, "raw": 0.172596, "normalized": 0.090744, "discount": 0.35, "effective": 0.031761, "running_max": 0.031761, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 25478167, "wall_clock_seconds": 0.3, "raw": 0.148226, "normalized": 0.060282, "discount": 0.35, "effective": 0.021099, "running_max": 0.031761, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 26154993, "wall_clock_seconds": 0.3, "raw": 0.163157, "normalized": 0.078946, "discount": 0.35, "effective": 0.027631, "running_max": 0.031761, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:scan_stitch--nohints", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.09917, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11461090, "usage": {"input": 447725, "output": 615029, "cache_read": 10398336, "cache_write": 0}, "cost_usd": 0.9347, "cost_source": "list_price", "wall_clock_seconds": 4206.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1561737, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1768376, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2330227, "wall_clock_seconds": 0.2, "raw": 0.098552, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3720354, "wall_clock_seconds": 0.3, "raw": 0.082464, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4756407, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 7939412, "wall_clock_seconds": 0.6, "raw": 0.054494, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 8554189, "wall_clock_seconds": 0.3, "raw": 0.055325, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 9447814, "wall_clock_seconds": 0.2, "raw": 0.033183, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 10178948, "wall_clock_seconds": 0.3, "raw": 0.08251, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 11461090, "wall_clock_seconds": 0.5, "raw": 0.064686, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:shattered_prior", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.060714, "area_term": 0.048564, "final_best": 0.109311, "best_raw": 0.198748, "best_effective": 0.109311, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 7, "max_submissions": 10, "tokens_total": 43911201, "usage": {"input": 461666, "output": 578239, "cache_read": 42871296, "cache_write": 0}, "cost_usd": 1.0896, "cost_source": "list_price", "wall_clock_seconds": 5385.0, "stop_reason": "agent_finished", "hints": [{"id": 3, "multiplier": 0.65, "before_submission": 1}, {"id": 4, "multiplier": 0.55, "before_submission": 1}, {"id": 1, "multiplier": 0.9, "before_submission": 1}, {"id": 2, "multiplier": 0.7, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4912907, "wall_clock_seconds": 1.9, "raw": 0.171757, "normalized": 0.171757, "discount": 0.55, "effective": 0.094466, "running_max": 0.094466, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 17670810, "wall_clock_seconds": 1.9, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.094466, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 20942128, "wall_clock_seconds": 1.6, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.094466, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 22518265, "wall_clock_seconds": 2.1, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.094466, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 25833568, "wall_clock_seconds": 2.0, "raw": 0.172083, "normalized": 0.172083, "discount": 0.55, "effective": 0.094646, "running_max": 0.094646, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 35950140, "wall_clock_seconds": 2.1, "raw": 0.181971, "normalized": 0.181971, "discount": 0.55, "effective": 0.100084, "running_max": 0.100084, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 37895438, "wall_clock_seconds": 1.8, "raw": 0.198748, "normalized": 0.198748, "discount": 0.55, "effective": 0.109311, "running_max": 0.109311, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:shattered_prior--nohints", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.045965, "area_term": 0.042621, "final_best": 0.059342, "best_raw": 0.059342, "best_effective": 0.059342, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17808348, "usage": {"input": 368260, "output": 307800, "cache_read": 17132288, "cache_write": 0}, "cost_usd": 0.5826, "cost_source": "list_price", "wall_clock_seconds": 2277.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1144463, "wall_clock_seconds": 2.2, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 9344190, "wall_clock_seconds": 2.0, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 9668564, "wall_clock_seconds": 1.8, "raw": 0.019857, "normalized": 0.019857, "discount": 1.0, "effective": 0.019857, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 11381740, "wall_clock_seconds": 2.2, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 12852523, "wall_clock_seconds": 2.0, "raw": 0.048689, "normalized": 0.048689, "discount": 1.0, "effective": 0.048689, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 13602411, "wall_clock_seconds": 1.9, "raw": 0.05167, "normalized": 0.05167, "discount": 1.0, "effective": 0.05167, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 14358062, "wall_clock_seconds": 3.3, "raw": 0.040511, "normalized": 0.040511, "discount": 1.0, "effective": 0.040511, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 15115331, "wall_clock_seconds": 1.9, "raw": 0.047692, "normalized": 0.047692, "discount": 1.0, "effective": 0.047692, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 15877119, "wall_clock_seconds": 1.8, "raw": 0.055777, "normalized": 0.055777, "discount": 1.0, "effective": 0.055777, "running_max": 0.055777, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 17808348, "wall_clock_seconds": 1.9, "raw": 0.059342, "normalized": 0.059342, "discount": 1.0, "effective": 0.059342, "running_max": 0.059342, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:ship_tune", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.055079, "area_term": 0.045113, "final_best": 0.094941, "best_raw": 0.094941, "best_effective": 0.094941, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 15503940, "usage": {"input": 843896, "output": 207564, "cache_read": 14452480, "cache_write": 0}, "cost_usd": 0.589, "cost_source": "list_price", "wall_clock_seconds": 7654.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4636129, "wall_clock_seconds": 220.8, "raw": 0.074646, "normalized": 0.074646, "discount": 1.0, "effective": 0.074646, "running_max": 0.074646, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 220.14}, {"index": 2, "tokens": 6083334, "wall_clock_seconds": 267.1, "raw": 0.056937, "normalized": 0.056937, "discount": 1.0, "effective": 0.056937, "running_max": 0.074646, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 266.45}, {"index": 3, "tokens": 8962272, "wall_clock_seconds": 213.4, "raw": 0.036068, "normalized": 0.036068, "discount": 1.0, "effective": 0.036068, "running_max": 0.074646, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 212.76}, {"index": 4, "tokens": 9552160, "wall_clock_seconds": 303.6, "raw": 0.058886, "normalized": 0.058886, "discount": 1.0, "effective": 0.058886, "running_max": 0.074646, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 303.04}, {"index": 5, "tokens": 10820438, "wall_clock_seconds": 247.1, "raw": 0.092157, "normalized": 0.092157, "discount": 1.0, "effective": 0.092157, "running_max": 0.092157, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 246.31}, {"index": 6, "tokens": 12142672, "wall_clock_seconds": 265.9, "raw": 0.092322, "normalized": 0.092322, "discount": 1.0, "effective": 0.092322, "running_max": 0.092322, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 265.12}, {"index": 7, "tokens": 13515653, "wall_clock_seconds": 253.2, "raw": 0.092291, "normalized": 0.092291, "discount": 1.0, "effective": 0.092291, "running_max": 0.092322, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 252.46}, {"index": 8, "tokens": 14244000, "wall_clock_seconds": 219.6, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.092322, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 218.63}, {"index": 9, "tokens": 14742923, "wall_clock_seconds": 273.1, "raw": 0.094941, "normalized": 0.094941, "discount": 1.0, "effective": 0.094941, "running_max": 0.094941, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 271.97}, {"index": 10, "tokens": 15503940, "wall_clock_seconds": 480.3, "raw": 0.094739, "normalized": 0.094739, "discount": 1.0, "effective": 0.094739, "running_max": 0.094941, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 479.46} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:tod_pipeline--nohints", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.00779, "area_term": 0.005313, "final_best": 0.017696, "best_raw": 0.017696, "best_effective": 0.017696, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 22556641, "usage": {"input": 250414, "output": 385843, "cache_read": 21920384, "cache_write": 0}, "cost_usd": 0.6697, "cost_source": "list_price", "wall_clock_seconds": 3846.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7477633, "wall_clock_seconds": 49.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 49.11}, {"index": 2, "tokens": 10742255, "wall_clock_seconds": 48.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 47.98}, {"index": 3, "tokens": 12544194, "wall_clock_seconds": 17.0, "raw": 0.017696, "normalized": 0.017696, "discount": 1.0, "effective": 0.017696, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 16.73}, {"index": 4, "tokens": 14267703, "wall_clock_seconds": 37.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 37.14}, {"index": 5, "tokens": 15352188, "wall_clock_seconds": 37.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 37.26}, {"index": 6, "tokens": 18246772, "wall_clock_seconds": 53.8, "raw": 0.00031, "normalized": 0.00031, "discount": 1.0, "effective": 0.00031, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 53.52}, {"index": 7, "tokens": 19143198, "wall_clock_seconds": 24.6, "raw": 0.000645, "normalized": 0.000645, "discount": 1.0, "effective": 0.000645, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 24.31}, {"index": 8, "tokens": 20050282, "wall_clock_seconds": 41.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 41.3}, {"index": 9, "tokens": 21281254, "wall_clock_seconds": 43.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 43.4}, {"index": 10, "tokens": 22556641, "wall_clock_seconds": 42.9, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.017696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 42.37} ]}, {"sample_key": "JYYnvgkFb2DthfqnBbwP7H:weirdml_bonanza", "eval_id": "JYYnvgkFb2DthfqnBbwP7H", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "a842a3e6b0f3dae757efe716247d214ed1e8551ca102c3f384005f5c9379294a", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.263003, "area_term": 0.215404, "final_best": 0.453397, "best_raw": 0.453397, "best_effective": 0.453397, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 29300549, "usage": {"input": 282905, "output": 354988, "cache_read": 28662656, "cache_write": 0}, "cost_usd": 0.6828, "cost_source": "list_price", "wall_clock_seconds": 9505.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4808294, "wall_clock_seconds": 72.8, "raw": 0.365543, "normalized": 0.365543, "discount": 1.0, "effective": 0.365543, "running_max": 0.365543, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 72.26}, {"index": 2, "tokens": 9693483, "wall_clock_seconds": 104.2, "raw": 0.430522, "normalized": 0.430522, "discount": 1.0, "effective": 0.430522, "running_max": 0.430522, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.57}, {"index": 3, "tokens": 13411049, "wall_clock_seconds": 107.3, "raw": 0.451836, "normalized": 0.451836, "discount": 1.0, "effective": 0.451836, "running_max": 0.451836, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.35}, {"index": 4, "tokens": 16523205, "wall_clock_seconds": 101.5, "raw": 0.451472, "normalized": 0.451472, "discount": 1.0, "effective": 0.451472, "running_max": 0.451836, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.85}, {"index": 5, "tokens": 19615028, "wall_clock_seconds": 102.8, "raw": 0.438061, "normalized": 0.438061, "discount": 1.0, "effective": 0.438061, "running_max": 0.451836, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 102.1}, {"index": 6, "tokens": 20410807, "wall_clock_seconds": 100.3, "raw": 0.437958, "normalized": 0.437958, "discount": 1.0, "effective": 0.437958, "running_max": 0.451836, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.64}, {"index": 7, "tokens": 21220149, "wall_clock_seconds": 99.5, "raw": 0.453397, "normalized": 0.453397, "discount": 1.0, "effective": 0.453397, "running_max": 0.453397, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.8}, {"index": 8, "tokens": 24286028, "wall_clock_seconds": 104.3, "raw": 0.440712, "normalized": 0.440712, "discount": 1.0, "effective": 0.440712, "running_max": 0.453397, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.74}, {"index": 9, "tokens": 26607699, "wall_clock_seconds": 104.1, "raw": 0.446946, "normalized": 0.446946, "discount": 1.0, "effective": 0.446946, "running_max": 0.453397, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.57}, {"index": 10, "tokens": 29300549, "wall_clock_seconds": 103.0, "raw": 0.437549, "normalized": 0.437549, "discount": 1.0, "effective": 0.437549, "running_max": 0.453397, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 102.43} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:mystery_box", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.110963, "area_term": 0.094599, "final_best": 0.176417, "best_raw": 0.646873, "best_effective": 0.176417, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 19899209, "usage": {"input": 24019, "output": 275942, "cache_read": 19233571, "cache_write": 365677}, "cost_usd": 26.1592, "cost_source": "list_price", "wall_clock_seconds": 5980.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3468586, "wall_clock_seconds": 4.8, "raw": 0.474217, "normalized": 0.504904, "discount": 0.25, "effective": 0.126226, "running_max": 0.126226, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5451746, "wall_clock_seconds": 9.4, "raw": 0.557077, "normalized": 0.601253, "discount": 0.25, "effective": 0.150313, "running_max": 0.150313, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 7002278, "wall_clock_seconds": 6.6, "raw": 0.622934, "normalized": 0.67783, "discount": 0.25, "effective": 0.169458, "running_max": 0.169458, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 7835567, "wall_clock_seconds": 3.4, "raw": 0.57432, "normalized": 0.621302, "discount": 0.25, "effective": 0.155326, "running_max": 0.169458, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 8960120, "wall_clock_seconds": 3.4, "raw": 0.543519, "normalized": 0.585487, "discount": 0.25, "effective": 0.146372, "running_max": 0.169458, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 11068644, "wall_clock_seconds": 3.4, "raw": 0.433612, "normalized": 0.457688, "discount": 0.25, "effective": 0.114422, "running_max": 0.169458, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 13777631, "wall_clock_seconds": 3.5, "raw": 0.625985, "normalized": 0.681378, "discount": 0.25, "effective": 0.170344, "running_max": 0.170344, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 15713503, "wall_clock_seconds": 3.7, "raw": 0.638493, "normalized": 0.695922, "discount": 0.25, "effective": 0.173981, "running_max": 0.173981, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 18105011, "wall_clock_seconds": 3.6, "raw": 0.646873, "normalized": 0.705666, "discount": 0.25, "effective": 0.176417, "running_max": 0.176417, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 19899209, "wall_clock_seconds": 3.5, "raw": 0.646639, "normalized": 0.705394, "discount": 0.25, "effective": 0.176348, "running_max": 0.176417, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:mystery_box--nohints", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.024487, "area_term": 0.023567, "final_best": 0.028171, "best_raw": 0.064227, "best_effective": 0.028171, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7207054, "usage": {"input": 4091, "output": 214319, "cache_read": 6716888, "cache_write": 271756}, "cost_usd": 17.8712, "cost_source": "list_price", "wall_clock_seconds": 3612.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1061351, "wall_clock_seconds": 2.7, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1252791, "wall_clock_seconds": 2.8, "raw": 0.059071, "normalized": 0.022175, "discount": 1.0, "effective": 0.022175, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2210522, "wall_clock_seconds": 3.9, "raw": 0.062658, "normalized": 0.026347, "discount": 1.0, "effective": 0.026347, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2351710, "wall_clock_seconds": 4.1, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3021833, "wall_clock_seconds": 4.9, "raw": 0.049979, "normalized": 0.011603, "discount": 1.0, "effective": 0.011603, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3607799, "wall_clock_seconds": 4.0, "raw": 0.053848, "normalized": 0.016102, "discount": 1.0, "effective": 0.016102, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4909445, "wall_clock_seconds": 5.5, "raw": 0.047983, "normalized": 0.009282, "discount": 1.0, "effective": 0.009282, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6143549, "wall_clock_seconds": 2.8, "raw": 0.053558, "normalized": 0.015765, "discount": 1.0, "effective": 0.015765, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6663493, "wall_clock_seconds": 3.9, "raw": 0.048412, "normalized": 0.009782, "discount": 1.0, "effective": 0.009782, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7207054, "wall_clock_seconds": 2.7, "raw": 0.043637, "normalized": 0.004229, "discount": 1.0, "effective": 0.004229, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:reaction_rates", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.241586, "area_term": 0.13108, "final_best": 0.683608, "best_raw": 0.854509, "best_effective": 0.683608, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 31417593, "usage": {"input": 25926, "output": 266221, "cache_read": 30669550, "cache_write": 455896}, "cost_usd": 30.3556, "cost_source": "list_price", "wall_clock_seconds": 27114.6, "stop_reason": "agent_finished", "hints": [{"id": 4, "multiplier": 0.8, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 10440042, "wall_clock_seconds": 32.7, "raw": 0.158872, "normalized": 0.158872, "discount": 0.8, "effective": 0.127097, "running_max": 0.127097, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 32.21}, {"index": 2, "tokens": 20515073, "wall_clock_seconds": 26.3, "raw": 0.329129, "normalized": 0.329129, "discount": 0.8, "effective": 0.263303, "running_max": 0.263303, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 25.76}, {"index": 3, "tokens": 25083427, "wall_clock_seconds": 250.6, "raw": 0.778023, "normalized": 0.778023, "discount": 0.8, "effective": 0.622418, "running_max": 0.622418, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 250.06}, {"index": 4, "tokens": 27116659, "wall_clock_seconds": 247.0, "raw": 0.800502, "normalized": 0.800502, "discount": 0.8, "effective": 0.640402, "running_max": 0.640402, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 246.48}, {"index": 5, "tokens": 28361594, "wall_clock_seconds": 297.1, "raw": 0.854509, "normalized": 0.854509, "discount": 0.8, "effective": 0.683608, "running_max": 0.683608, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 296.32} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:reaction_rates--nohints", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.282247, "area_term": 0.182817, "final_best": 0.679966, "best_raw": 0.679966, "best_effective": 0.679966, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 24423778, "usage": {"input": 9287, "output": 254630, "cache_read": 23753920, "cache_write": 405941}, "cost_usd": 26.8817, "cost_source": "list_price", "wall_clock_seconds": 16004.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 14140576, "wall_clock_seconds": 44.6, "raw": 0.595351, "normalized": 0.595351, "discount": 1.0, "effective": 0.595351, "running_max": 0.595351, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.09}, {"index": 2, "tokens": 15102877, "wall_clock_seconds": 36.8, "raw": 0.406753, "normalized": 0.406753, "discount": 1.0, "effective": 0.406753, "running_max": 0.595351, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 36.23}, {"index": 3, "tokens": 15755276, "wall_clock_seconds": 61.9, "raw": 0.644487, "normalized": 0.644487, "discount": 1.0, "effective": 0.644487, "running_max": 0.644487, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 61.36}, {"index": 4, "tokens": 16415755, "wall_clock_seconds": 96.7, "raw": 0.350925, "normalized": 0.350925, "discount": 1.0, "effective": 0.350925, "running_max": 0.644487, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.21}, {"index": 5, "tokens": 17104077, "wall_clock_seconds": 48.8, "raw": 0.526463, "normalized": 0.526463, "discount": 1.0, "effective": 0.526463, "running_max": 0.644487, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 48.26}, {"index": 6, "tokens": 17813858, "wall_clock_seconds": 47.8, "raw": 0.667104, "normalized": 0.667104, "discount": 1.0, "effective": 0.667104, "running_max": 0.667104, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 47.27}, {"index": 7, "tokens": 20832365, "wall_clock_seconds": 548.0, "raw": 0.47404, "normalized": 0.47404, "discount": 1.0, "effective": 0.47404, "running_max": 0.667104, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 547.48}, {"index": 8, "tokens": 22804630, "wall_clock_seconds": 60.8, "raw": 0.677181, "normalized": 0.677181, "discount": 1.0, "effective": 0.677181, "running_max": 0.677181, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 60.27}, {"index": 9, "tokens": 23611818, "wall_clock_seconds": 62.3, "raw": 0.655339, "normalized": 0.655339, "discount": 1.0, "effective": 0.655339, "running_max": 0.677181, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 61.83}, {"index": 10, "tokens": 24423778, "wall_clock_seconds": 53.4, "raw": 0.679966, "normalized": 0.679966, "discount": 1.0, "effective": 0.679966, "running_max": 0.679966, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 52.93} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:scan_stitch", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.185389, "area_term": 0.159787, "final_best": 0.287797, "best_raw": 0.757822, "best_effective": 0.287797, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11111008, "usage": {"input": 4744, "output": 179039, "cache_read": 10689652, "cache_write": 237573}, "cost_usd": 16.4233, "cost_source": "list_price", "wall_clock_seconds": 3532.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2446632, "wall_clock_seconds": 0.3, "raw": 0.089005, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3530200, "wall_clock_seconds": 0.3, "raw": 0.620004, "normalized": 0.650006, "discount": 0.35, "effective": 0.227502, "running_max": 0.227502, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4164792, "wall_clock_seconds": 0.3, "raw": 0.673782, "normalized": 0.717228, "discount": 0.35, "effective": 0.25103, "running_max": 0.25103, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 4681681, "wall_clock_seconds": 0.3, "raw": 0.69368, "normalized": 0.742101, "discount": 0.35, "effective": 0.259735, "running_max": 0.259735, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 5800068, "wall_clock_seconds": 0.3, "raw": 0.716966, "normalized": 0.771207, "discount": 0.35, "effective": 0.269922, "running_max": 0.269922, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 6790772, "wall_clock_seconds": 0.3, "raw": 0.739676, "normalized": 0.799595, "discount": 0.35, "effective": 0.279858, "running_max": 0.279858, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 7412470, "wall_clock_seconds": 0.3, "raw": 0.736793, "normalized": 0.795992, "discount": 0.35, "effective": 0.278597, "running_max": 0.279858, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 8501550, "wall_clock_seconds": 0.3, "raw": 0.739336, "normalized": 0.79917, "discount": 0.35, "effective": 0.27971, "running_max": 0.279858, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8948700, "wall_clock_seconds": 0.3, "raw": 0.738751, "normalized": 0.798439, "discount": 0.35, "effective": 0.279454, "running_max": 0.279858, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 11111008, "wall_clock_seconds": 0.3, "raw": 0.757822, "normalized": 0.822277, "discount": 0.35, "effective": 0.287797, "running_max": 0.287797, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:scan_stitch--nohints", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.09917, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7319035, "usage": {"input": 4065, "output": 241612, "cache_read": 6792560, "cache_write": 280798}, "cost_usd": 19.4353, "cost_source": "list_price", "wall_clock_seconds": 4019.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 950053, "wall_clock_seconds": 0.3, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1697105, "wall_clock_seconds": 0.3, "raw": 0.086927, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1904428, "wall_clock_seconds": 0.3, "raw": 0.097422, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2569980, "wall_clock_seconds": 0.3, "raw": 0.087653, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2903850, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3469736, "wall_clock_seconds": 0.4, "raw": 0.097422, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4122332, "wall_clock_seconds": 1.5, "raw": 0.071386, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4584730, "wall_clock_seconds": 0.5, "raw": 0.098414, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5310221, "wall_clock_seconds": 0.4, "raw": 0.086927, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7319035, "wall_clock_seconds": 1.0, "raw": 0.064567, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:shapes_generalize", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.744391, "area_term": 0.709863, "final_best": 0.8825, "best_raw": 0.906, "best_effective": 0.8825, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5841893, "usage": {"input": 8820, "output": 90753, "cache_read": 5612998, "cache_write": 129322}, "cost_usd": 8.6155, "cost_source": "list_price", "wall_clock_seconds": 11135.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1023800, "wall_clock_seconds": 6.1, "raw": 0.358, "normalized": 0.1975, "discount": 1.0, "effective": 0.1975, "running_max": 0.1975, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.49}, {"index": 2, "tokens": 1163237, "wall_clock_seconds": 7.5, "raw": 0.8, "normalized": 0.75, "discount": 1.0, "effective": 0.75, "running_max": 0.75, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.83}, {"index": 3, "tokens": 1619115, "wall_clock_seconds": 13.6, "raw": 0.844, "normalized": 0.805, "discount": 1.0, "effective": 0.805, "running_max": 0.805, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.68}, {"index": 4, "tokens": 1945701, "wall_clock_seconds": 8.3, "raw": 0.837, "normalized": 0.79625, "discount": 1.0, "effective": 0.79625, "running_max": 0.805, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.59}, {"index": 5, "tokens": 2386959, "wall_clock_seconds": 7.0, "raw": 0.903, "normalized": 0.87875, "discount": 1.0, "effective": 0.87875, "running_max": 0.87875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.27}, {"index": 6, "tokens": 3289262, "wall_clock_seconds": 6.9, "raw": 0.906, "normalized": 0.8825, "discount": 1.0, "effective": 0.8825, "running_max": 0.8825, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.27}, {"index": 7, "tokens": 4341060, "wall_clock_seconds": 7.7, "raw": 0.897, "normalized": 0.87125, "discount": 1.0, "effective": 0.87125, "running_max": 0.8825, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.96}, {"index": 8, "tokens": 4857531, "wall_clock_seconds": 7.1, "raw": 0.904, "normalized": 0.88, "discount": 1.0, "effective": 0.88, "running_max": 0.8825, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.49}, {"index": 9, "tokens": 5261103, "wall_clock_seconds": 6.5, "raw": 0.905, "normalized": 0.88125, "discount": 1.0, "effective": 0.88125, "running_max": 0.8825, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.95}, {"index": 10, "tokens": 5841893, "wall_clock_seconds": 6.2, "raw": 0.89, "normalized": 0.8625, "discount": 1.0, "effective": 0.8625, "running_max": 0.8825, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.65} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:shattered_prior", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.194103, "area_term": 0.080187, "final_best": 0.649768, "best_raw": 0.999642, "best_effective": 0.649768, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 38565271, "usage": {"input": 8785, "output": 352778, "cache_read": 37684367, "cache_write": 519341}, "cost_usd": 37.5347, "cost_source": "list_price", "wall_clock_seconds": 6792.5, "stop_reason": "agent_finished", "hints": [{"id": 3, "multiplier": 0.65, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 26503080, "wall_clock_seconds": 1.9, "raw": 0.776757, "normalized": 0.776757, "discount": 0.65, "effective": 0.504892, "running_max": 0.504892, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 33531966, "wall_clock_seconds": 1.8, "raw": 0.875671, "normalized": 0.875671, "discount": 0.65, "effective": 0.569186, "running_max": 0.569186, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 37537183, "wall_clock_seconds": 1.9, "raw": 0.999642, "normalized": 0.999642, "discount": 0.65, "effective": 0.649768, "running_max": 0.649768, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:shattered_prior--nohints", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.283837, "area_term": 0.121594, "final_best": 0.932808, "best_raw": 0.932808, "best_effective": 0.932808, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 43749532, "usage": {"input": 6590, "output": 372596, "cache_read": 42864433, "cache_write": 505913}, "cost_usd": 39.5301, "cost_source": "list_price", "wall_clock_seconds": 16666.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 18092951, "wall_clock_seconds": 1.8, "raw": 0.182708, "normalized": 0.182708, "discount": 1.0, "effective": 0.182708, "running_max": 0.182708, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 29947796, "wall_clock_seconds": 1.7, "raw": 0.877322, "normalized": 0.877322, "discount": 1.0, "effective": 0.877322, "running_max": 0.877322, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 35929797, "wall_clock_seconds": 2.6, "raw": 0.930771, "normalized": 0.930771, "discount": 1.0, "effective": 0.930771, "running_max": 0.930771, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 38327581, "wall_clock_seconds": 1.9, "raw": 0.932808, "normalized": 0.932808, "discount": 1.0, "effective": 0.932808, "running_max": 0.932808, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 41263982, "wall_clock_seconds": 2.2, "raw": 0.88696, "normalized": 0.88696, "discount": 1.0, "effective": 0.88696, "running_max": 0.932808, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 43250070, "wall_clock_seconds": 1.6, "raw": 0.902745, "normalized": 0.902745, "discount": 1.0, "effective": 0.902745, "running_max": 0.932808, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:ship_detect", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.063068, "area_term": 0.05815, "final_best": 0.08274, "best_raw": 0.08274, "best_effective": 0.08274, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 18, "max_submissions": null, "tokens_total": 19897011, "usage": {"input": 22814, "output": 170300, "cache_read": 16311997, "cache_write": 3391900}, "cost_usd": 80.6591, "cost_source": "list_price", "wall_clock_seconds": 34419.8, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 15, "tokens": 593249, "cost_weighted": 42920.8, "wall_clock_seconds": 818.7, "raw": 0.043165, "normalized": 0.043165, "effective": 0.043165, "running_max": 0.043165, "scored": true, "final": false}, {"index": 2, "model_call": 22, "tokens": 1088947, "cost_weighted": 69037.5, "wall_clock_seconds": 1721.3, "raw": 0.050791, "normalized": 0.050791, "effective": 0.050791, "running_max": 0.050791, "scored": true, "final": false}, {"index": 3, "model_call": 23, "tokens": 1170599, "cost_weighted": 70803.3, "wall_clock_seconds": 2530.7, "raw": 0.053615, "normalized": 0.053615, "effective": 0.053615, "running_max": 0.053615, "scored": true, "final": false}, {"index": 4, "model_call": 37, "tokens": 2720733, "cost_weighted": 125137.4, "wall_clock_seconds": 6824.5, "raw": 0.067067, "normalized": 0.067067, "effective": 0.067067, "running_max": 0.067067, "scored": true, "final": false}, {"index": 5, "model_call": 46, "tokens": 4115291, "cost_weighted": 158713.2, "wall_clock_seconds": 9658.9, "raw": 0.063577, "normalized": 0.063577, "effective": 0.063577, "running_max": 0.067067, "scored": true, "final": false}, {"index": 6, "model_call": 52, "tokens": 5174903, "cost_weighted": 180640.6, "wall_clock_seconds": 13215.6, "raw": 0.065376, "normalized": 0.065376, "effective": 0.065376, "running_max": 0.067067, "scored": true, "final": false}, {"index": 7, "model_call": 55, "tokens": 5733103, "cost_weighted": 190172.9, "wall_clock_seconds": 15527.0, "raw": 0.073463, "normalized": 0.073463, "effective": 0.073463, "running_max": 0.073463, "scored": true, "final": false}, {"index": 8, "model_call": 58, "tokens": 6315775, "cost_weighted": 199984.6, "wall_clock_seconds": 17814.8, "raw": 0.071804, "normalized": 0.071804, "effective": 0.071804, "running_max": 0.073463, "scored": true, "final": false}, {"index": 9, "model_call": 66, "tokens": 7988028, "cost_weighted": 227689.1, "wall_clock_seconds": 20427.0, "raw": 0.075267, "normalized": 0.075267, "effective": 0.075267, "running_max": 0.075267, "scored": true, "final": false}, {"index": 10, "model_call": 70, "tokens": 8893645, "cost_weighted": 243959.2, "wall_clock_seconds": 20879.7, "raw": 0.074049, "normalized": 0.074049, "effective": 0.074049, "running_max": 0.075267, "scored": true, "final": false}, {"index": 11, "model_call": 74, "tokens": 9832781, "cost_weighted": 255270.3, "wall_clock_seconds": 23496.1, "raw": 0.076201, "normalized": 0.076201, "effective": 0.076201, "running_max": 0.076201, "scored": true, "final": false}, {"index": 12, "model_call": 79, "tokens": 10805327, "cost_weighted": 335050.0, "wall_clock_seconds": 23853.3, "raw": 0.080516, "normalized": 0.080516, "effective": 0.080516, "running_max": 0.080516, "scored": true, "final": false}, {"index": 13, "model_call": 96, "tokens": 14062341, "cost_weighted": 911250.2, "wall_clock_seconds": 24742.1, "raw": 0.080298, "normalized": 0.080298, "effective": 0.080298, "running_max": 0.080516, "scored": true, "final": false}, {"index": 14, "model_call": 109, "tokens": 16625357, "cost_weighted": 1286663.8, "wall_clock_seconds": 29065.4, "raw": 0.08274, "normalized": 0.08274, "effective": 0.08274, "running_max": 0.08274, "scored": true, "final": false}, {"index": 15, "model_call": 110, "tokens": 16822861, "cost_weighted": 1288131.7, "wall_clock_seconds": 29306.3, "raw": 0.082142, "normalized": 0.082142, "effective": 0.082142, "running_max": 0.08274, "scored": true, "final": false}, {"index": 16, "model_call": 116, "tokens": 18033891, "cost_weighted": 1442996.6, "wall_clock_seconds": 32623.7, "raw": 0.082357, "normalized": 0.082357, "effective": 0.082357, "running_max": 0.08274, "scored": true, "final": false}, {"index": 17, "model_call": 119, "tokens": 18648262, "cost_weighted": 1521877.1, "wall_clock_seconds": 32940.5, "raw": 0.08267, "normalized": 0.08267, "effective": 0.08267, "running_max": 0.08274, "scored": true, "final": false}, {"index": 18, "model_call": 122, "tokens": 19264535, "cost_weighted": 1603022.5, "wall_clock_seconds": 34270.5, "raw": 0.081859, "normalized": 0.081859, "effective": 0.081859, "running_max": 0.08274, "scored": true, "final": false} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:ship_tune", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.122871, "area_term": 0.091055, "final_best": 0.250136, "best_raw": 0.250136, "best_effective": 0.250136, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 34179244, "usage": {"input": 123557, "output": 962781, "cache_read": 31068411, "cache_write": 2024495}, "cost_usd": 97.6316, "cost_source": "list_price", "wall_clock_seconds": 36804.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1130473, "wall_clock_seconds": 142.8, "raw": 0.040147, "normalized": 0.040147, "discount": 1.0, "effective": 0.040147, "running_max": 0.040147, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 142.18}, {"index": 2, "tokens": 10283416, "wall_clock_seconds": 140.9, "raw": 0.187199, "normalized": 0.187199, "discount": 1.0, "effective": 0.187199, "running_max": 0.187199, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 140.18}, {"index": 3, "tokens": 27619082, "wall_clock_seconds": 294.9, "raw": 0.201616, "normalized": 0.201616, "discount": 1.0, "effective": 0.201616, "running_max": 0.201616, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 294.13}, {"index": 4, "tokens": 28213687, "wall_clock_seconds": 291.3, "raw": 0.232582, "normalized": 0.232582, "discount": 1.0, "effective": 0.232582, "running_max": 0.232582, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 290.64}, {"index": 5, "tokens": 28818267, "wall_clock_seconds": 414.9, "raw": 0.238894, "normalized": 0.238894, "discount": 1.0, "effective": 0.238894, "running_max": 0.238894, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 414.11}, {"index": 6, "tokens": 30083106, "wall_clock_seconds": 579.9, "raw": 0.239462, "normalized": 0.239462, "discount": 1.0, "effective": 0.239462, "running_max": 0.239462, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 579.15}, {"index": 7, "tokens": 32072863, "wall_clock_seconds": 575.4, "raw": 0.245418, "normalized": 0.245418, "discount": 1.0, "effective": 0.245418, "running_max": 0.245418, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 574.7}, {"index": 8, "tokens": 32526191, "wall_clock_seconds": 576.1, "raw": 0.247763, "normalized": 0.247763, "discount": 1.0, "effective": 0.247763, "running_max": 0.247763, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 575.19}, {"index": 9, "tokens": 33697505, "wall_clock_seconds": 589.5, "raw": 0.250136, "normalized": 0.250136, "discount": 1.0, "effective": 0.250136, "running_max": 0.250136, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 588.7} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:splash_generalize", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.591439, "area_term": 0.576521, "final_best": 0.651111, "best_raw": 0.686, "best_effective": 0.651111, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6343108, "usage": {"input": 4039, "output": 125343, "cache_read": 6028573, "cache_write": 185153}, "cost_usd": 11.5177, "cost_source": "list_price", "wall_clock_seconds": 4035.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 786521, "wall_clock_seconds": 51.9, "raw": 0.585, "normalized": 0.538889, "discount": 1.0, "effective": 0.538889, "running_max": 0.538889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 51.23}, {"index": 2, "tokens": 1045853, "wall_clock_seconds": 68.4, "raw": 0.66, "normalized": 0.622222, "discount": 1.0, "effective": 0.622222, "running_max": 0.622222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 67.82}, {"index": 3, "tokens": 1444040, "wall_clock_seconds": 80.0, "raw": 0.675, "normalized": 0.638889, "discount": 1.0, "effective": 0.638889, "running_max": 0.638889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 79.47}, {"index": 4, "tokens": 2044706, "wall_clock_seconds": 81.6, "raw": 0.681, "normalized": 0.645556, "discount": 1.0, "effective": 0.645556, "running_max": 0.645556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 80.84}, {"index": 5, "tokens": 2876438, "wall_clock_seconds": 73.5, "raw": 0.681, "normalized": 0.645556, "discount": 1.0, "effective": 0.645556, "running_max": 0.645556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 72.94}, {"index": 6, "tokens": 3502626, "wall_clock_seconds": 87.9, "raw": 0.686, "normalized": 0.651111, "discount": 1.0, "effective": 0.651111, "running_max": 0.651111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.17}, {"index": 7, "tokens": 4185722, "wall_clock_seconds": 90.1, "raw": 0.652, "normalized": 0.613333, "discount": 1.0, "effective": 0.613333, "running_max": 0.651111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 89.52}, {"index": 8, "tokens": 5329292, "wall_clock_seconds": 87.6, "raw": 0.635, "normalized": 0.594444, "discount": 1.0, "effective": 0.594444, "running_max": 0.651111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.93}, {"index": 9, "tokens": 5931792, "wall_clock_seconds": 83.8, "raw": 0.67, "normalized": 0.633333, "discount": 1.0, "effective": 0.633333, "running_max": 0.651111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 83.22}, {"index": 10, "tokens": 6343108, "wall_clock_seconds": 87.8, "raw": 0.679, "normalized": 0.643333, "discount": 1.0, "effective": 0.643333, "running_max": 0.651111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.27} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:tod_pipeline--nohints", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.129433, "area_term": 0.117977, "final_best": 0.17526, "best_raw": 0.17526, "best_effective": 0.17526, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13155832, "usage": {"input": 14019, "output": 208097, "cache_read": 12612310, "cache_write": 321406}, "cost_usd": 20.1262, "cost_source": "list_price", "wall_clock_seconds": 5360.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2252530, "wall_clock_seconds": 16.5, "raw": 0.17526, "normalized": 0.17526, "discount": 1.0, "effective": 0.17526, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 16.24}, {"index": 2, "tokens": 3855821, "wall_clock_seconds": 13.7, "raw": 0.081559, "normalized": 0.081559, "discount": 1.0, "effective": 0.081559, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.4}, {"index": 3, "tokens": 5100991, "wall_clock_seconds": 33.8, "raw": 0.159597, "normalized": 0.159597, "discount": 1.0, "effective": 0.159597, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 32.37}, {"index": 4, "tokens": 6264838, "wall_clock_seconds": 33.7, "raw": 0.079039, "normalized": 0.079039, "discount": 1.0, "effective": 0.079039, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 33.34}, {"index": 5, "tokens": 7807407, "wall_clock_seconds": 18.5, "raw": 0.167067, "normalized": 0.167067, "discount": 1.0, "effective": 0.167067, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.15}, {"index": 6, "tokens": 8627746, "wall_clock_seconds": 53.4, "raw": 0.169795, "normalized": 0.169795, "discount": 1.0, "effective": 0.169795, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 53.11}, {"index": 7, "tokens": 9474599, "wall_clock_seconds": 55.3, "raw": 0.152044, "normalized": 0.152044, "discount": 1.0, "effective": 0.152044, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 55.03}, {"index": 8, "tokens": 10351138, "wall_clock_seconds": 26.1, "raw": 0.152433, "normalized": 0.152433, "discount": 1.0, "effective": 0.152433, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.77}, {"index": 9, "tokens": 11568500, "wall_clock_seconds": 29.4, "raw": 0.164196, "normalized": 0.164196, "discount": 1.0, "effective": 0.164196, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 29.11}, {"index": 10, "tokens": 13155832, "wall_clock_seconds": 28.9, "raw": 0.152733, "normalized": 0.152733, "discount": 1.0, "effective": 0.152733, "running_max": 0.17526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.65} ]}, {"sample_key": "NK5PyThnqnuCtnxKzNUot6:weirdml_bonanza", "eval_id": "NK5PyThnqnuCtnxKzNUot6", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "de63a61cfe75dc6a71b8dc472a91942937561f8dfb48551a1e6c7f9093e811d5", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.696216, "area_term": 0.654249, "final_best": 0.864084, "best_raw": 0.864084, "best_effective": 0.864084, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17790059, "usage": {"input": 13457, "output": 194265, "cache_read": 17285376, "cache_write": 296961}, "cost_usd": 20.1084, "cost_source": "list_price", "wall_clock_seconds": 8231.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1060318, "wall_clock_seconds": 84.8, "raw": 0.61706, "normalized": 0.61706, "discount": 1.0, "effective": 0.61706, "running_max": 0.61706, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 84.24}, {"index": 2, "tokens": 2666633, "wall_clock_seconds": 103.9, "raw": 0.728916, "normalized": 0.728916, "discount": 1.0, "effective": 0.728916, "running_max": 0.728916, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 102.29}, {"index": 3, "tokens": 3997727, "wall_clock_seconds": 99.2, "raw": 0.802449, "normalized": 0.802449, "discount": 1.0, "effective": 0.802449, "running_max": 0.802449, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.51}, {"index": 4, "tokens": 6120141, "wall_clock_seconds": 106.2, "raw": 0.85201, "normalized": 0.85201, "discount": 1.0, "effective": 0.85201, "running_max": 0.85201, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.7}, {"index": 5, "tokens": 8475099, "wall_clock_seconds": 96.9, "raw": 0.852339, "normalized": 0.852339, "discount": 1.0, "effective": 0.852339, "running_max": 0.852339, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.27}, {"index": 6, "tokens": 11011386, "wall_clock_seconds": 98.6, "raw": 0.856379, "normalized": 0.856379, "discount": 1.0, "effective": 0.856379, "running_max": 0.856379, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.02}, {"index": 7, "tokens": 12483488, "wall_clock_seconds": 101.9, "raw": 0.863983, "normalized": 0.863983, "discount": 1.0, "effective": 0.863983, "running_max": 0.863983, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.24}, {"index": 8, "tokens": 14321744, "wall_clock_seconds": 102.7, "raw": 0.864084, "normalized": 0.864084, "discount": 1.0, "effective": 0.864084, "running_max": 0.864084, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.29}, {"index": 9, "tokens": 15727569, "wall_clock_seconds": 109.8, "raw": 0.85945, "normalized": 0.85945, "discount": 1.0, "effective": 0.85945, "running_max": 0.864084, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 109.14}, {"index": 10, "tokens": 17790059, "wall_clock_seconds": 100.7, "raw": 0.858235, "normalized": 0.858235, "discount": 1.0, "effective": 0.858235, "running_max": 0.864084, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.93} ]}, {"sample_key": "T9k4VXZUj86KFYadfWiX4i:mystery_box", "eval_id": "T9k4VXZUj86KFYadfWiX4i", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "51568f6633420c85aa4147fa81c8c28aa804543335f120a6120024c92a4637b2", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.003267, "area_term": 0.002807, "final_best": 0.005106, "best_raw": 0.057566, "best_effective": 0.005106, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 16112040, "usage": {"input": 2703649, "output": 278855, "cache_read": 13129536, "cache_write": 0}, "cost_usd": 0.4694, "cost_source": "list_price", "wall_clock_seconds": 14120.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 141253, "wall_clock_seconds": 7.9, "raw": 0.04671, "normalized": 0.007802, "discount": 0.25, "effective": 0.001951, "running_max": 0.001951, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 8468022, "wall_clock_seconds": 4.2, "raw": 0.046486, "normalized": 0.007542, "discount": 0.25, "effective": 0.001885, "running_max": 0.001951, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 10733430, "wall_clock_seconds": 5.3, "raw": 0.046041, "normalized": 0.007025, "discount": 0.25, "effective": 0.001756, "running_max": 0.001951, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 12040600, "wall_clock_seconds": 4.8, "raw": 0.04482, "normalized": 0.005605, "discount": 0.25, "effective": 0.001401, "running_max": 0.001951, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 12479308, "wall_clock_seconds": 4.9, "raw": 0.043043, "normalized": 0.003539, "discount": 0.25, "effective": 0.000885, "running_max": 0.001951, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 14276870, "wall_clock_seconds": 3.4, "raw": 0.056186, "normalized": 0.018821, "discount": 0.25, "effective": 0.004705, "running_max": 0.004705, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 14733365, "wall_clock_seconds": 7.9, "raw": 0.057566, "normalized": 0.020426, "discount": 0.25, "effective": 0.005106, "running_max": 0.005106, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 15190969, "wall_clock_seconds": 3.6, "raw": 0.053947, "normalized": 0.016217, "discount": 0.25, "effective": 0.004054, "running_max": 0.005106, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 15650635, "wall_clock_seconds": 4.8, "raw": 0.054033, "normalized": 0.016317, "discount": 0.25, "effective": 0.004079, "running_max": 0.005106, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 16112040, "wall_clock_seconds": 6.0, "raw": 0.057546, "normalized": 0.020402, "discount": 0.25, "effective": 0.005101, "running_max": 0.005106, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "T9k4VXZUj86KFYadfWiX4i:mystery_box--nohints", "eval_id": "T9k4VXZUj86KFYadfWiX4i", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "51568f6633420c85aa4147fa81c8c28aa804543335f120a6120024c92a4637b2", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.026969, "area_term": 0.02372, "final_best": 0.039963, "best_raw": 0.074369, "best_effective": 0.039963, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 14634040, "usage": {"input": 3737737, "output": 473071, "cache_read": 10423232, "cache_write": 0}, "cost_usd": 0.5549, "cost_source": "list_price", "wall_clock_seconds": 14903.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 311170, "wall_clock_seconds": 4.0, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 377283, "wall_clock_seconds": 2.1, "raw": 0.05207, "normalized": 0.014035, "discount": 1.0, "effective": 0.014035, "running_max": 0.014035, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 453800, "wall_clock_seconds": 2.1, "raw": 0.039607, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.014035, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 6691970, "wall_clock_seconds": 3.7, "raw": 0.052133, "normalized": 0.014108, "discount": 1.0, "effective": 0.014108, "running_max": 0.014108, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 7876904, "wall_clock_seconds": 2.4, "raw": 0.061792, "normalized": 0.02534, "discount": 1.0, "effective": 0.02534, "running_max": 0.02534, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8498685, "wall_clock_seconds": 3.8, "raw": 0.046092, "normalized": 0.007083, "discount": 1.0, "effective": 0.007083, "running_max": 0.02534, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 9176789, "wall_clock_seconds": 3.7, "raw": 0.052133, "normalized": 0.014108, "discount": 1.0, "effective": 0.014108, "running_max": 0.02534, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 9889600, "wall_clock_seconds": 2.3, "raw": 0.074369, "normalized": 0.039963, "discount": 1.0, "effective": 0.039963, "running_max": 0.039963, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 11939139, "wall_clock_seconds": 2.6, "raw": 0.073537, "normalized": 0.038996, "discount": 1.0, "effective": 0.038996, "running_max": 0.039963, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 14634040, "wall_clock_seconds": 4.0, "raw": 0.048836, "normalized": 0.010275, "discount": 1.0, "effective": 0.010275, "running_max": 0.039963, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "T9k4VXZUj86KFYadfWiX4i:reaction_rates", "eval_id": "T9k4VXZUj86KFYadfWiX4i", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "51568f6633420c85aa4147fa81c8c28aa804543335f120a6120024c92a4637b2", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.001981, "area_term": 0.000868, "final_best": 0.006434, "best_raw": 0.021883, "best_effective": 0.006434, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 46281330, "usage": {"input": 7093151, "output": 353427, "cache_read": 38834752, "cache_write": 0}, "cost_usd": 1.2029, "cost_source": "list_price", "wall_clock_seconds": 20094.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 1}, {"id": 2, "multiplier": 0.75, "before_submission": 1}, {"id": 3, "multiplier": 0.7, "before_submission": 1}, {"id": 4, "multiplier": 0.8, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 12037095, "wall_clock_seconds": 56.0, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 55.51}, {"index": 2, "tokens": 24907216, "wall_clock_seconds": 246.0, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 245.78}, {"index": 3, "tokens": 26867291, "wall_clock_seconds": 561.6, "raw": 0.021883, "normalized": 0.021883, "discount": 0.294, "effective": 0.006434, "running_max": 0.006434, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 561.38}, {"index": 4, "tokens": 33223417, "wall_clock_seconds": 600.2, "raw": null, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.006434, "hints_active": 4, "valid": false, "invalid_kind": "rejected_by_metric", "runtime_seconds": 600.0}, {"index": 5, "tokens": 36342288, "wall_clock_seconds": 89.9, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.006434, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 89.58}, {"index": 6, "tokens": 37696556, "wall_clock_seconds": 123.5, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.006434, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 123.17}, {"index": 7, "tokens": 39081273, "wall_clock_seconds": 145.3, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.006434, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 145.0}, {"index": 8, "tokens": 40493339, "wall_clock_seconds": 522.9, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.006434, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 522.53}, {"index": 9, "tokens": 45300182, "wall_clock_seconds": 382.5, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.006434, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 382.14}, {"index": 10, "tokens": 46281330, "wall_clock_seconds": 311.4, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.006434, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 311.11} ]}, {"sample_key": "T9k4VXZUj86KFYadfWiX4i:reaction_rates--nohints", "eval_id": "T9k4VXZUj86KFYadfWiX4i", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "51568f6633420c85aa4147fa81c8c28aa804543335f120a6120024c92a4637b2", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.005886, "area_term": 0.003174, "final_best": 0.016733, "best_raw": 0.016733, "best_effective": 0.016733, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 24710467, "usage": {"input": 615240, "output": 449915, "cache_read": 23645312, "cache_write": 0}, "cost_usd": 0.5133, "cost_source": "list_price", "wall_clock_seconds": 13267.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2497378, "wall_clock_seconds": 12.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 11.82}, {"index": 2, "tokens": 2785767, "wall_clock_seconds": 1.9, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 1.59}, {"index": 3, "tokens": 3521310, "wall_clock_seconds": 12.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 11.95}, {"index": 4, "tokens": 12788925, "wall_clock_seconds": 13.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.09}, {"index": 5, "tokens": 15853677, "wall_clock_seconds": 212.1, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 211.56}, {"index": 6, "tokens": 20871808, "wall_clock_seconds": 13.5, "raw": 0.016733, "normalized": 0.016733, "discount": 1.0, "effective": 0.016733, "running_max": 0.016733, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.92}, {"index": 7, "tokens": 21991885, "wall_clock_seconds": 13.1, "raw": 0.016733, "normalized": 0.016733, "discount": 1.0, "effective": 0.016733, "running_max": 0.016733, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.52}, {"index": 8, "tokens": 22876996, "wall_clock_seconds": 13.7, "raw": 0.016552, "normalized": 0.016552, "discount": 1.0, "effective": 0.016552, "running_max": 0.016733, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.1}, {"index": 9, "tokens": 24091945, "wall_clock_seconds": 4.7, "raw": 0.016733, "normalized": 0.016733, "discount": 1.0, "effective": 0.016733, "running_max": 0.016733, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.09}, {"index": 10, "tokens": 24710467, "wall_clock_seconds": 12.6, "raw": 0.016733, "normalized": 0.016733, "discount": 1.0, "effective": 0.016733, "running_max": 0.016733, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.04} ]}, {"sample_key": "T9k4VXZUj86KFYadfWiX4i:scan_stitch", "eval_id": "T9k4VXZUj86KFYadfWiX4i", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "51568f6633420c85aa4147fa81c8c28aa804543335f120a6120024c92a4637b2", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.046269, "area_term": 0.022571, "final_best": 0.14106, "best_raw": 0.422422, "best_effective": 0.14106, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 31449097, "usage": {"input": 3225488, "output": 255225, "cache_read": 27968384, "cache_write": 0}, "cost_usd": 0.7252, "cost_source": "list_price", "wall_clock_seconds": 21002.6, "stop_reason": "agent_finished", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 223953, "wall_clock_seconds": 0.4, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 11047827, "wall_clock_seconds": 0.4, "raw": 0.087232, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 20426519, "wall_clock_seconds": 0.3, "raw": 0.261785, "normalized": 0.202231, "discount": 0.35, "effective": 0.070781, "running_max": 0.070781, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 24844845, "wall_clock_seconds": 0.3, "raw": 0.09376, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.070781, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 25353471, "wall_clock_seconds": 0.3, "raw": 0.290613, "normalized": 0.238266, "discount": 0.35, "effective": 0.083393, "running_max": 0.083393, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 28698743, "wall_clock_seconds": 0.3, "raw": 0.422422, "normalized": 0.403028, "discount": 0.35, "effective": 0.14106, "running_max": 0.14106, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 30332565, "wall_clock_seconds": 0.3, "raw": 0.398736, "normalized": 0.37342, "discount": 0.35, "effective": 0.130697, "running_max": 0.14106, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 31169230, "wall_clock_seconds": 0.3, "raw": 0.399329, "normalized": 0.374161, "discount": 0.35, "effective": 0.130956, "running_max": 0.14106, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "T9k4VXZUj86KFYadfWiX4i:scan_stitch--nohints", "eval_id": "T9k4VXZUj86KFYadfWiX4i", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "51568f6633420c85aa4147fa81c8c28aa804543335f120a6120024c92a4637b2", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.004316, "area_term": 0.004085, "final_best": 0.005239, "best_raw": 0.104191, "best_effective": 0.005239, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5323326, "usage": {"input": 835984, "output": 328110, "cache_read": 4159232, "cache_write": 0}, "cost_usd": 0.2071, "cost_source": "list_price", "wall_clock_seconds": 8287.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 283223, "wall_clock_seconds": 0.2, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 628784, "wall_clock_seconds": 0.6, "raw": 0.054566, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 724282, "wall_clock_seconds": 0.5, "raw": 0.054494, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1173098, "wall_clock_seconds": 0.3, "raw": 0.102151, "normalized": 0.002689, "discount": 1.0, "effective": 0.002689, "running_max": 0.002689, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1633477, "wall_clock_seconds": 0.3, "raw": 0.104191, "normalized": 0.005239, "discount": 1.0, "effective": 0.005239, "running_max": 0.005239, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 2406909, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.005239, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 2617595, "wall_clock_seconds": 0.3, "raw": 0.102151, "normalized": 0.002689, "discount": 1.0, "effective": 0.002689, "running_max": 0.005239, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2833798, "wall_clock_seconds": 0.4, "raw": 0.102151, "normalized": 0.002689, "discount": 1.0, "effective": 0.002689, "running_max": 0.005239, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 4341135, "wall_clock_seconds": 0.3, "raw": 0.064692, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.005239, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5323326, "wall_clock_seconds": 0.3, "raw": 0.085478, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.005239, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "T9k4VXZUj86KFYadfWiX4i:shapes_generalize", "eval_id": "T9k4VXZUj86KFYadfWiX4i", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "31c67b7", "source_log_sha256": "51568f6633420c85aa4147fa81c8c28aa804543335f120a6120024c92a4637b2", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.059875, "area_term": 0.04586, "final_best": 0.115937, "best_raw": 0.465, "best_effective": 0.115937, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11988436, "usage": {"input": 606381, "output": 165671, "cache_read": 11216384, "cache_write": 0}, "cost_usd": 0.2551, "cost_source": "list_price", "wall_clock_seconds": 14082.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.5, "before_submission": 5}, {"id": 2, "multiplier": 0.35, "before_submission": 5}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4563382, "wall_clock_seconds": 120.6, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 120.0}, {"index": 2, "tokens": 5593344, "wall_clock_seconds": 22.7, "raw": 0.105, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.07}, {"index": 3, "tokens": 5927964, "wall_clock_seconds": 21.1, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 20.67}, {"index": 4, "tokens": 6157689, "wall_clock_seconds": 0.7, "raw": 0.173, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 5, "tokens": 7407338, "wall_clock_seconds": 0.8, "raw": 0.188, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.04125, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 6, "tokens": 7669695, "wall_clock_seconds": 0.8, "raw": 0.2, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.04125, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 0.23}, {"index": 7, "tokens": 9331881, "wall_clock_seconds": 26.2, "raw": 0.436, "normalized": 0.295, "discount": 0.35, "effective": 0.10325, "running_max": 0.10325, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 25.63}, {"index": 8, "tokens": 10228881, "wall_clock_seconds": 32.1, "raw": 0.451, "normalized": 0.31375, "discount": 0.35, "effective": 0.109812, "running_max": 0.109812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 31.37}, {"index": 9, "tokens": 11494183, "wall_clock_seconds": 37.3, "raw": 0.423, "normalized": 0.27875, "discount": 0.35, "effective": 0.097562, "running_max": 0.109812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 36.75}, {"index": 10, "tokens": 11988436, "wall_clock_seconds": 55.5, "raw": 0.465, "normalized": 0.33125, "discount": 0.35, "effective": 0.115937, "running_max": 0.115937, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 54.9} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:mystery_box", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.002042, "area_term": 0.001088, "final_best": 0.005856, "best_raw": 0.060145, "best_effective": 0.005856, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 44383682, "usage": {"input": 368214, "output": 213980, "cache_read": 43801488, "cache_write": 42209063}, "cost_usd": 6.1238, "cost_source": "list_price", "wall_clock_seconds": 5433.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 14242385, "wall_clock_seconds": 18.8, "raw": 0.05268, "normalized": 0.014745, "discount": 0.25, "effective": 0.003686, "running_max": 0.003686, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 22615589, "wall_clock_seconds": 3.9, "raw": 0.049101, "normalized": 0.010582, "discount": 0.25, "effective": 0.002646, "running_max": 0.003686, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 36776277, "wall_clock_seconds": 4.4, "raw": 0.047483, "normalized": 0.008702, "discount": 0.25, "effective": 0.002175, "running_max": 0.003686, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 38477239, "wall_clock_seconds": 6.3, "raw": 0.050138, "normalized": 0.011788, "discount": 0.25, "effective": 0.002947, "running_max": 0.003686, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 40221911, "wall_clock_seconds": 3.5, "raw": 0.052046, "normalized": 0.014007, "discount": 0.25, "effective": 0.003502, "running_max": 0.003686, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 40806344, "wall_clock_seconds": 3.5, "raw": 0.053344, "normalized": 0.015517, "discount": 0.25, "effective": 0.003879, "running_max": 0.003879, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 41392711, "wall_clock_seconds": 3.4, "raw": 0.057311, "normalized": 0.020129, "discount": 0.25, "effective": 0.005032, "running_max": 0.005032, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 42585013, "wall_clock_seconds": 3.6, "raw": 0.05956, "normalized": 0.022745, "discount": 0.25, "effective": 0.005686, "running_max": 0.005686, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 43180896, "wall_clock_seconds": 3.7, "raw": 0.058899, "normalized": 0.021976, "discount": 0.25, "effective": 0.005494, "running_max": 0.005686, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 44383682, "wall_clock_seconds": 3.3, "raw": 0.060145, "normalized": 0.023425, "discount": 0.25, "effective": 0.005856, "running_max": 0.005856, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:mystery_box--nohints", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.019678, "area_term": 0.016533, "final_best": 0.03226, "best_raw": 0.067743, "best_effective": 0.03226, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8052307, "usage": {"input": 132484, "output": 71972, "cache_read": 7847851, "cache_write": 7539303}, "cost_usd": 1.2722, "cost_source": "list_price", "wall_clock_seconds": 1205.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4259890, "wall_clock_seconds": 5.2, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4596757, "wall_clock_seconds": 2.6, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.026049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4763595, "wall_clock_seconds": 2.6, "raw": 0.062378, "normalized": 0.026021, "discount": 1.0, "effective": 0.026021, "running_max": 0.026049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 4847563, "wall_clock_seconds": 2.6, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 5102159, "wall_clock_seconds": 2.5, "raw": 0.038724, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 5452890, "wall_clock_seconds": 2.5, "raw": 0.041161, "normalized": 0.00135, "discount": 1.0, "effective": 0.00135, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5805222, "wall_clock_seconds": 2.5, "raw": 0.064623, "normalized": 0.028631, "discount": 1.0, "effective": 0.028631, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6450723, "wall_clock_seconds": 3.2, "raw": 0.067743, "normalized": 0.03226, "discount": 1.0, "effective": 0.03226, "running_max": 0.03226, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 7222848, "wall_clock_seconds": 2.4, "raw": 0.067743, "normalized": 0.03226, "discount": 1.0, "effective": 0.03226, "running_max": 0.03226, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 8052307, "wall_clock_seconds": 2.4, "raw": 0.063593, "normalized": 0.027434, "discount": 1.0, "effective": 0.027434, "running_max": 0.03226, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:night_school", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.115125, "area_term": 0.097068, "final_best": 0.187353, "best_raw": 0.395, "best_effective": 0.187353, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 18468750, "usage": {"input": 256150, "output": 143250, "cache_read": 18069350, "cache_write": 18024643}, "cost_usd": 2.8361, "cost_source": "list_price", "wall_clock_seconds": 3082.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 419781, "wall_clock_seconds": 2.5, "raw": 0.197, "normalized": 0.055294, "discount": 1.0, "effective": 0.055294, "running_max": 0.055294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2503735, "wall_clock_seconds": 11.1, "raw": 0.1905, "normalized": 0.047647, "discount": 0.65, "effective": 0.030971, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 10584321, "wall_clock_seconds": 10.6, "raw": 0.326, "normalized": 0.207059, "discount": 0.65, "effective": 0.134588, "running_max": 0.134588, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 11758129, "wall_clock_seconds": 9.6, "raw": 0.347, "normalized": 0.231765, "discount": 0.65, "effective": 0.150647, "running_max": 0.150647, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 14143057, "wall_clock_seconds": 11.3, "raw": 0.3885, "normalized": 0.280588, "discount": 0.65, "effective": 0.182382, "running_max": 0.182382, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 15128672, "wall_clock_seconds": 11.3, "raw": 0.395, "normalized": 0.288235, "discount": 0.65, "effective": 0.187353, "running_max": 0.187353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 15526009, "wall_clock_seconds": 11.2, "raw": 0.365, "normalized": 0.252941, "discount": 0.65, "effective": 0.164412, "running_max": 0.187353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 16132399, "wall_clock_seconds": 10.8, "raw": 0.372, "normalized": 0.261176, "discount": 0.65, "effective": 0.169765, "running_max": 0.187353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 17393418, "wall_clock_seconds": 11.1, "raw": 0.365, "normalized": 0.252941, "discount": 0.65, "effective": 0.164412, "running_max": 0.187353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 18468750, "wall_clock_seconds": 11.1, "raw": 0.369, "normalized": 0.257647, "discount": 0.65, "effective": 0.167471, "running_max": 0.187353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:reaction_rates", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.005172, "area_term": 0.000124, "final_best": 0.025361, "best_raw": 0.101487, "best_effective": 0.025361, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 49312388, "usage": {"input": 318206, "output": 204221, "cache_read": 48789961, "cache_write": 48048890}, "cost_usd": 6.6674, "cost_source": "list_price", "wall_clock_seconds": 3704.7, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 6}, {"id": 4, "multiplier": 0.8, "before_submission": 6}, {"id": 3, "multiplier": 0.7, "before_submission": 8}, {"id": 5, "multiplier": 0.85, "before_submission": 8}, {"id": 2, "multiplier": 0.75, "before_submission": 9}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1442635, "wall_clock_seconds": 2.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.57}, {"index": 2, "tokens": 2339537, "wall_clock_seconds": 3.2, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.67}, {"index": 3, "tokens": 14509682, "wall_clock_seconds": 3.9, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.49}, {"index": 4, "tokens": 16345175, "wall_clock_seconds": 69.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 69.24}, {"index": 5, "tokens": 20004709, "wall_clock_seconds": 55.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 55.1}, {"index": 6, "tokens": 26130892, "wall_clock_seconds": 36.0, "raw": 0.0, "normalized": 0.0, "discount": 0.56, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 35.53}, {"index": 7, "tokens": 35239139, "wall_clock_seconds": 9.7, "raw": 0.0, "normalized": 0.0, "discount": 0.56, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 9.26}, {"index": 8, "tokens": 42025073, "wall_clock_seconds": 42.2, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 41.75}, {"index": 9, "tokens": 46719579, "wall_clock_seconds": 43.5, "raw": 0.016411, "normalized": 0.016411, "discount": 0.2499, "effective": 0.004101, "running_max": 0.004101, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 43.11}, {"index": 10, "tokens": 49312388, "wall_clock_seconds": 6.4, "raw": 0.101487, "normalized": 0.101487, "discount": 0.2499, "effective": 0.025361, "running_max": 0.025361, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 6.01} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:reaction_rates--nohints", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.013164, "area_term": 0.004062, "final_best": 0.049571, "best_raw": 0.049571, "best_effective": 0.049571, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 49020787, "usage": {"input": 406796, "output": 188142, "cache_read": 48425849, "cache_write": 47017855}, "cost_usd": 6.6032, "cost_source": "list_price", "wall_clock_seconds": 3737.8, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5795307, "wall_clock_seconds": 5.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.0}, {"index": 2, "tokens": 25280113, "wall_clock_seconds": 4.5, "raw": 0.004171, "normalized": 0.004171, "discount": 1.0, "effective": 0.004171, "running_max": 0.004171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.01}, {"index": 3, "tokens": 35255280, "wall_clock_seconds": 4.4, "raw": 0.049571, "normalized": 0.049571, "discount": 1.0, "effective": 0.049571, "running_max": 0.049571, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.0}, {"index": 4, "tokens": 38950424, "wall_clock_seconds": 5.2, "raw": 0.049571, "normalized": 0.049571, "discount": 1.0, "effective": 0.049571, "running_max": 0.049571, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.8}, {"index": 5, "tokens": 44979231, "wall_clock_seconds": 4.9, "raw": 0.000994, "normalized": 0.000994, "discount": 1.0, "effective": 0.000994, "running_max": 0.049571, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.55}, {"index": 6, "tokens": 47210918, "wall_clock_seconds": 14.8, "raw": 0.045899, "normalized": 0.045899, "discount": 1.0, "effective": 0.045899, "running_max": 0.049571, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.38} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:scan_stitch", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.034211, "area_term": 0.019703, "final_best": 0.092243, "best_raw": 0.310842, "best_effective": 0.092243, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 27211157, "usage": {"input": 293043, "output": 158472, "cache_read": 26759642, "cache_write": 24297333}, "cost_usd": 3.8342, "cost_source": "list_price", "wall_clock_seconds": 3368.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4425106, "wall_clock_seconds": 0.3, "raw": 0.085441, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 10986836, "wall_clock_seconds": 0.2, "raw": 0.087317, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 18023530, "wall_clock_seconds": 0.2, "raw": 0.289685, "normalized": 0.237106, "discount": 0.35, "effective": 0.082987, "running_max": 0.082987, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 19203516, "wall_clock_seconds": 0.3, "raw": 0.251355, "normalized": 0.189194, "discount": 0.35, "effective": 0.066218, "running_max": 0.082987, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 20435896, "wall_clock_seconds": 0.2, "raw": 0.272129, "normalized": 0.215161, "discount": 0.35, "effective": 0.075306, "running_max": 0.082987, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 22772334, "wall_clock_seconds": 0.3, "raw": 0.280564, "normalized": 0.225705, "discount": 0.35, "effective": 0.078997, "running_max": 0.082987, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 24523788, "wall_clock_seconds": 0.2, "raw": 0.29259, "normalized": 0.240737, "discount": 0.35, "effective": 0.084258, "running_max": 0.084258, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 25408306, "wall_clock_seconds": 0.3, "raw": 0.299273, "normalized": 0.249091, "discount": 0.35, "effective": 0.087182, "running_max": 0.087182, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 26308386, "wall_clock_seconds": 0.3, "raw": 0.305648, "normalized": 0.25706, "discount": 0.35, "effective": 0.089971, "running_max": 0.089971, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 27211157, "wall_clock_seconds": 0.2, "raw": 0.310842, "normalized": 0.263552, "discount": 0.35, "effective": 0.092243, "running_max": 0.092243, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:scan_stitch--nohints", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.099305, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17493272, "usage": {"input": 204491, "output": 78669, "cache_read": 17210112, "cache_write": 15932109}, "cost_usd": 2.4035, "cost_source": "list_price", "wall_clock_seconds": 1632.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4565446, "wall_clock_seconds": 0.2, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7556147, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 7812065, "wall_clock_seconds": 0.3, "raw": 0.098552, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 10802031, "wall_clock_seconds": 0.2, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 14221216, "wall_clock_seconds": 0.3, "raw": 0.099305, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 15006185, "wall_clock_seconds": 0.2, "raw": 0.070403, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 15335882, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 15984666, "wall_clock_seconds": 0.3, "raw": 0.082371, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 16644476, "wall_clock_seconds": 0.3, "raw": 0.037252, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 17493272, "wall_clock_seconds": 0.2, "raw": 0.092567, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:shapes_generalize", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.07823, "area_term": 0.062787, "final_best": 0.14, "best_raw": 0.312, "best_effective": 0.14, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 14876122, "usage": {"input": 227147, "output": 177108, "cache_read": 14471867, "cache_write": 13248286}, "cost_usd": 2.4724, "cost_source": "list_price", "wall_clock_seconds": 1836.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3014735, "wall_clock_seconds": 2.9, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.27}, {"index": 2, "tokens": 3585848, "wall_clock_seconds": 2.8, "raw": 0.173, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.24}, {"index": 3, "tokens": 3857405, "wall_clock_seconds": 2.8, "raw": 0.188, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.26}, {"index": 4, "tokens": 4130399, "wall_clock_seconds": 3.1, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.61}, {"index": 5, "tokens": 6279523, "wall_clock_seconds": 6.9, "raw": 0.229, "normalized": 0.03625, "discount": 1.0, "effective": 0.03625, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.35}, {"index": 6, "tokens": 7575084, "wall_clock_seconds": 6.8, "raw": 0.052, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.28}, {"index": 7, "tokens": 8586425, "wall_clock_seconds": 6.8, "raw": 0.311, "normalized": 0.13875, "discount": 1.0, "effective": 0.13875, "running_max": 0.13875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.3}, {"index": 8, "tokens": 11613518, "wall_clock_seconds": 6.7, "raw": 0.224, "normalized": 0.03, "discount": 1.0, "effective": 0.03, "running_max": 0.13875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.19}, {"index": 9, "tokens": 13272333, "wall_clock_seconds": 6.9, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.13875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.4}, {"index": 10, "tokens": 14876122, "wall_clock_seconds": 8.2, "raw": 0.312, "normalized": 0.14, "discount": 1.0, "effective": 0.14, "running_max": 0.14, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.75} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:shattered_prior", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.022623, "area_term": 0.016552, "final_best": 0.046907, "best_raw": 0.085285, "best_effective": 0.046907, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 18834171, "usage": {"input": 382787, "output": 117031, "cache_read": 18334353, "cache_write": 18122991}, "cost_usd": 2.8568, "cost_source": "list_price", "wall_clock_seconds": 2580.7, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.65, "before_submission": 1}, {"id": 4, "multiplier": 0.55, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7527021, "wall_clock_seconds": 1.6, "raw": 0.04873, "normalized": 0.04873, "discount": 0.55, "effective": 0.026802, "running_max": 0.026802, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 10909406, "wall_clock_seconds": 1.7, "raw": 0.001989, "normalized": 0.001989, "discount": 0.55, "effective": 0.001094, "running_max": 0.026802, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 12055068, "wall_clock_seconds": 1.6, "raw": 0.059889, "normalized": 0.059889, "discount": 0.55, "effective": 0.032939, "running_max": 0.032939, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 14823042, "wall_clock_seconds": 1.7, "raw": 0.078208, "normalized": 0.078208, "discount": 0.55, "effective": 0.043014, "running_max": 0.043014, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 15627434, "wall_clock_seconds": 1.6, "raw": 0.083269, "normalized": 0.083269, "discount": 0.55, "effective": 0.045798, "running_max": 0.045798, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 16113059, "wall_clock_seconds": 1.8, "raw": 0.085285, "normalized": 0.085285, "discount": 0.55, "effective": 0.046907, "running_max": 0.046907, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 16611913, "wall_clock_seconds": 1.6, "raw": 0.084664, "normalized": 0.084664, "discount": 0.55, "effective": 0.046565, "running_max": 0.046907, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 17109393, "wall_clock_seconds": 1.6, "raw": 0.085285, "normalized": 0.085285, "discount": 0.55, "effective": 0.046907, "running_max": 0.046907, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:shattered_prior--nohints", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.04873, "area_term": 0.04873, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 27542977, "usage": {"input": 417621, "output": 114560, "cache_read": 27010796, "cache_write": 27010796}, "cost_usd": 3.895, "cost_source": "list_price", "wall_clock_seconds": 1844.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 79842, "wall_clock_seconds": 1.7, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 11071117, "wall_clock_seconds": 1.6, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 22603064, "wall_clock_seconds": 1.7, "raw": 0.047588, "normalized": 0.047588, "discount": 1.0, "effective": 0.047588, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 22982203, "wall_clock_seconds": 1.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:ship_tune", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.032444, "area_term": 0.021945, "final_best": 0.074441, "best_raw": 0.074441, "best_effective": 0.074441, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 36465538, "usage": {"input": 858932, "output": 273009, "cache_read": 35333597, "cache_write": 29202008}, "cost_usd": 5.5357, "cost_source": "list_price", "wall_clock_seconds": 5540.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 11876634, "wall_clock_seconds": 111.7, "raw": 0.066104, "normalized": 0.066104, "discount": 1.0, "effective": 0.066104, "running_max": 0.066104, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 110.36}, {"index": 2, "tokens": 20896769, "wall_clock_seconds": 119.8, "raw": 0.063994, "normalized": 0.063994, "discount": 1.0, "effective": 0.063994, "running_max": 0.066104, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 119.3}, {"index": 3, "tokens": 22631731, "wall_clock_seconds": 117.1, "raw": 0.071854, "normalized": 0.071854, "discount": 1.0, "effective": 0.071854, "running_max": 0.071854, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 116.41}, {"index": 4, "tokens": 25340708, "wall_clock_seconds": 120.0, "raw": 0.069384, "normalized": 0.069384, "discount": 1.0, "effective": 0.069384, "running_max": 0.071854, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 119.47}, {"index": 5, "tokens": 28034806, "wall_clock_seconds": 121.3, "raw": 0.074147, "normalized": 0.074147, "discount": 1.0, "effective": 0.074147, "running_max": 0.074147, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.71}, {"index": 6, "tokens": 29677167, "wall_clock_seconds": 118.1, "raw": 0.074441, "normalized": 0.074441, "discount": 1.0, "effective": 0.074441, "running_max": 0.074441, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 117.59}, {"index": 7, "tokens": 31396180, "wall_clock_seconds": 118.0, "raw": 0.068471, "normalized": 0.068471, "discount": 1.0, "effective": 0.068471, "running_max": 0.074441, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 117.02}, {"index": 8, "tokens": 33195040, "wall_clock_seconds": 120.4, "raw": 0.074255, "normalized": 0.074255, "discount": 1.0, "effective": 0.074255, "running_max": 0.074441, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 119.83} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:splash_generalize", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.30452, "area_term": 0.228428, "final_best": 0.608889, "best_raw": 0.648, "best_effective": 0.608889, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 7, "max_submissions": 10, "tokens_total": 25723356, "usage": {"input": 443446, "output": 333166, "cache_read": 24946744, "cache_write": 21861513}, "cost_usd": 4.3646, "cost_source": "list_price", "wall_clock_seconds": 4247.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4923746, "wall_clock_seconds": 0.9, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.35}, {"index": 2, "tokens": 8763668, "wall_clock_seconds": 44.6, "raw": 0.629, "normalized": 0.587778, "discount": 1.0, "effective": 0.587778, "running_max": 0.587778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.02}, {"index": 3, "tokens": 13030067, "wall_clock_seconds": 93.4, "raw": 0.648, "normalized": 0.608889, "discount": 1.0, "effective": 0.608889, "running_max": 0.608889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 92.8}, {"index": 4, "tokens": 17438560, "wall_clock_seconds": 28.9, "raw": 0.621, "normalized": 0.578889, "discount": 1.0, "effective": 0.578889, "running_max": 0.608889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.26}, {"index": 5, "tokens": 19396608, "wall_clock_seconds": 83.0, "raw": 0.636, "normalized": 0.595556, "discount": 1.0, "effective": 0.595556, "running_max": 0.608889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 82.49}, {"index": 6, "tokens": 22108095, "wall_clock_seconds": 87.5, "raw": 0.644, "normalized": 0.604444, "discount": 1.0, "effective": 0.604444, "running_max": 0.608889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.91}, {"index": 7, "tokens": 23857275, "wall_clock_seconds": 102.0, "raw": 0.645, "normalized": 0.605556, "discount": 1.0, "effective": 0.605556, "running_max": 0.608889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.38} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:tod_pipeline--nohints", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.023325, "area_term": 0.01185, "final_best": 0.069221, "best_raw": 0.069221, "best_effective": 0.069221, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 2, "max_submissions": 10, "tokens_total": 28799042, "usage": {"input": 248154, "output": 128033, "cache_read": 28422855, "cache_write": 28191626}, "cost_usd": 3.9735, "cost_source": "list_price", "wall_clock_seconds": 2244.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 22687415, "wall_clock_seconds": 19.2, "raw": 0.068418, "normalized": 0.068418, "discount": 1.0, "effective": 0.068418, "running_max": 0.068418, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 19.0}, {"index": 2, "tokens": 26553280, "wall_clock_seconds": 21.3, "raw": 0.069221, "normalized": 0.069221, "discount": 1.0, "effective": 0.069221, "running_max": 0.069221, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.0} ]}, {"sample_key": "ZNgyw3Cwva3t8TwNEbPj7y:weirdml_bonanza", "eval_id": "ZNgyw3Cwva3t8TwNEbPj7y", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "26af5f6412245f0b560f5e1cd5333e845561c85a12488df52110659959979a13", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.289641, "area_term": 0.222706, "final_best": 0.557379, "best_raw": 0.557379, "best_effective": 0.557379, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 38104526, "usage": {"input": 413209, "output": 210981, "cache_read": 37480336, "cache_write": 37431822}, "cost_usd": 5.473, "cost_source": "list_price", "wall_clock_seconds": 3341.6, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 6469200, "wall_clock_seconds": 29.6, "raw": 0.405844, "normalized": 0.405844, "discount": 1.0, "effective": 0.405844, "running_max": 0.405844, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.85}, {"index": 2, "tokens": 9832780, "wall_clock_seconds": 54.4, "raw": 0.465617, "normalized": 0.465617, "discount": 1.0, "effective": 0.465617, "running_max": 0.465617, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 53.76}, {"index": 3, "tokens": 14849055, "wall_clock_seconds": 59.5, "raw": 0.479744, "normalized": 0.479744, "discount": 1.0, "effective": 0.479744, "running_max": 0.479744, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.93}, {"index": 4, "tokens": 17384770, "wall_clock_seconds": 67.5, "raw": 0.554803, "normalized": 0.554803, "discount": 1.0, "effective": 0.554803, "running_max": 0.554803, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 66.87}, {"index": 5, "tokens": 21295464, "wall_clock_seconds": 81.1, "raw": 0.556879, "normalized": 0.556879, "discount": 1.0, "effective": 0.556879, "running_max": 0.556879, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 80.46}, {"index": 6, "tokens": 26183660, "wall_clock_seconds": 97.1, "raw": 0.552291, "normalized": 0.552291, "discount": 1.0, "effective": 0.552291, "running_max": 0.556879, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.5}, {"index": 7, "tokens": 29919099, "wall_clock_seconds": 85.5, "raw": 0.557268, "normalized": 0.557268, "discount": 1.0, "effective": 0.557268, "running_max": 0.557268, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 84.92}, {"index": 8, "tokens": 33400560, "wall_clock_seconds": 102.0, "raw": 0.554325, "normalized": 0.554325, "discount": 1.0, "effective": 0.554325, "running_max": 0.557268, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.3}, {"index": 9, "tokens": 36775665, "wall_clock_seconds": 86.6, "raw": 0.557379, "normalized": 0.557379, "discount": 1.0, "effective": 0.557379, "running_max": 0.557379, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.02} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:mystery_box", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.357416, "area_term": 0.339906, "final_best": 0.427457, "best_raw": 0.652688, "best_effective": 0.427457, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5152420, "usage": {"input": 276, "output": 77642, "cache_read": 4886922, "cache_write": 187580}, "cost_usd": 11.1165, "cost_source": "billed", "wall_clock_seconds": 3683.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1145108, "wall_clock_seconds": 11.1, "raw": 0.447383, "normalized": 0.473701, "discount": 0.6, "effective": 0.284221, "running_max": 0.284221, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1286306, "wall_clock_seconds": 5.0, "raw": 0.486732, "normalized": 0.519455, "discount": 0.6, "effective": 0.311673, "running_max": 0.311673, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1689465, "wall_clock_seconds": 3.5, "raw": 0.650912, "normalized": 0.710363, "discount": 0.6, "effective": 0.426218, "running_max": 0.426218, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2245291, "wall_clock_seconds": 3.3, "raw": 0.649476, "normalized": 0.708693, "discount": 0.6, "effective": 0.425216, "running_max": 0.426218, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2872593, "wall_clock_seconds": 4.4, "raw": 0.650912, "normalized": 0.710363, "discount": 0.6, "effective": 0.426218, "running_max": 0.426218, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3094397, "wall_clock_seconds": 3.4, "raw": 0.652561, "normalized": 0.71228, "discount": 0.6, "effective": 0.427368, "running_max": 0.427368, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 3205778, "wall_clock_seconds": 4.0, "raw": 0.652688, "normalized": 0.712428, "discount": 0.6, "effective": 0.427457, "running_max": 0.427457, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 3433717, "wall_clock_seconds": 3.8, "raw": 0.65088, "normalized": 0.710326, "discount": 0.6, "effective": 0.426196, "running_max": 0.427457, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 3549066, "wall_clock_seconds": 3.2, "raw": 0.651628, "normalized": 0.711195, "discount": 0.6, "effective": 0.426717, "running_max": 0.427457, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5152420, "wall_clock_seconds": 3.3, "raw": 0.651265, "normalized": 0.710773, "discount": 0.6, "effective": 0.426464, "running_max": 0.427457, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:mystery_box--nohints", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.029363, "area_term": 0.029171, "final_best": 0.030128, "best_raw": 0.06591, "best_effective": 0.030128, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3239505, "usage": {"input": 177, "output": 61701, "cache_read": 3053527, "cache_write": 124100}, "cost_usd": 7.6916, "cost_source": "billed", "wall_clock_seconds": 1945.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 179826, "wall_clock_seconds": 2.4, "raw": 0.057095, "normalized": 0.019878, "discount": 1.0, "effective": 0.019878, "running_max": 0.019878, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 496176, "wall_clock_seconds": 2.7, "raw": 0.052928, "normalized": 0.015033, "discount": 1.0, "effective": 0.015033, "running_max": 0.019878, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 768287, "wall_clock_seconds": 2.8, "raw": 0.06591, "normalized": 0.030128, "discount": 1.0, "effective": 0.030128, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 872747, "wall_clock_seconds": 2.6, "raw": 0.063445, "normalized": 0.027262, "discount": 1.0, "effective": 0.027262, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1101713, "wall_clock_seconds": 2.6, "raw": 0.061062, "normalized": 0.024491, "discount": 1.0, "effective": 0.024491, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1489400, "wall_clock_seconds": 2.4, "raw": 0.039013, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1851278, "wall_clock_seconds": 2.7, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2085142, "wall_clock_seconds": 3.0, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2820426, "wall_clock_seconds": 2.6, "raw": 0.065651, "normalized": 0.029826, "discount": 1.0, "effective": 0.029826, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 3239505, "wall_clock_seconds": 2.5, "raw": 0.061131, "normalized": 0.024572, "discount": 1.0, "effective": 0.024572, "running_max": 0.030128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:night_school", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.533345, "area_term": 0.526549, "final_best": 0.560529, "best_raw": 0.883, "best_effective": 0.560529, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 16471462, "usage": {"input": 651, "output": 242353, "cache_read": 14650887, "cache_write": 1577571}, "cost_usd": 46.4947, "cost_source": "billed", "wall_clock_seconds": 31778.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 475693, "wall_clock_seconds": 11.1, "raw": 0.463, "normalized": 0.368235, "discount": 0.65, "effective": 0.239353, "running_max": 0.239353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 737338, "wall_clock_seconds": 12.3, "raw": 0.858, "normalized": 0.832941, "discount": 0.65, "effective": 0.541412, "running_max": 0.541412, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1790874, "wall_clock_seconds": 5.0, "raw": 0.871, "normalized": 0.848235, "discount": 0.65, "effective": 0.551353, "running_max": 0.551353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3918680, "wall_clock_seconds": 5.2, "raw": 0.867, "normalized": 0.843529, "discount": 0.65, "effective": 0.548294, "running_max": 0.551353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4038840, "wall_clock_seconds": 5.4, "raw": 0.865, "normalized": 0.841176, "discount": 0.65, "effective": 0.546765, "running_max": 0.551353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 6440806, "wall_clock_seconds": 5.1, "raw": 0.8775, "normalized": 0.855882, "discount": 0.65, "effective": 0.556324, "running_max": 0.556324, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 8619104, "wall_clock_seconds": 5.8, "raw": 0.879, "normalized": 0.857647, "discount": 0.65, "effective": 0.557471, "running_max": 0.557471, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 10088377, "wall_clock_seconds": 4.9, "raw": 0.8795, "normalized": 0.858235, "discount": 0.65, "effective": 0.557853, "running_max": 0.557853, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 11929924, "wall_clock_seconds": 4.9, "raw": 0.84, "normalized": 0.811765, "discount": 0.65, "effective": 0.527647, "running_max": 0.557853, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 16471462, "wall_clock_seconds": 5.6, "raw": 0.883, "normalized": 0.862353, "discount": 0.65, "effective": 0.560529, "running_max": 0.560529, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:reaction_rates", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.498386, "area_term": 0.427099, "final_best": 0.783537, "best_raw": 0.979421, "best_effective": 0.783537, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 19498956, "usage": {"input": 642, "output": 163586, "cache_read": 18610612, "cache_write": 724116}, "cost_usd": 35.8478, "cost_source": "billed", "wall_clock_seconds": 9948.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.8, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1500601, "wall_clock_seconds": 20.3, "raw": 0.217646, "normalized": 0.217646, "discount": 1.0, "effective": 0.217646, "running_max": 0.217646, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 19.82}, {"index": 2, "tokens": 3379352, "wall_clock_seconds": 19.2, "raw": 0.40521, "normalized": 0.40521, "discount": 0.8, "effective": 0.324168, "running_max": 0.324168, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 18.85}, {"index": 3, "tokens": 6160251, "wall_clock_seconds": 176.4, "raw": 0.892928, "normalized": 0.892928, "discount": 0.8, "effective": 0.714343, "running_max": 0.714343, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 176.1}, {"index": 4, "tokens": 8838849, "wall_clock_seconds": 206.4, "raw": 0.926082, "normalized": 0.926082, "discount": 0.8, "effective": 0.740866, "running_max": 0.740866, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 206.06}, {"index": 5, "tokens": 12501476, "wall_clock_seconds": 213.3, "raw": 0.935212, "normalized": 0.935212, "discount": 0.8, "effective": 0.74817, "running_max": 0.74817, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 213.04}, {"index": 6, "tokens": 13375058, "wall_clock_seconds": 210.0, "raw": 0.935385, "normalized": 0.935385, "discount": 0.8, "effective": 0.748308, "running_max": 0.748308, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 209.66}, {"index": 7, "tokens": 14497418, "wall_clock_seconds": 208.5, "raw": 0.978913, "normalized": 0.978913, "discount": 0.8, "effective": 0.78313, "running_max": 0.78313, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 208.17}, {"index": 8, "tokens": 18214193, "wall_clock_seconds": 234.1, "raw": 0.978992, "normalized": 0.978992, "discount": 0.8, "effective": 0.783193, "running_max": 0.783193, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 233.78}, {"index": 9, "tokens": 19186199, "wall_clock_seconds": 266.9, "raw": 0.978836, "normalized": 0.978836, "discount": 0.8, "effective": 0.783069, "running_max": 0.783193, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 266.61}, {"index": 10, "tokens": 19498956, "wall_clock_seconds": 468.8, "raw": 0.979421, "normalized": 0.979421, "discount": 0.8, "effective": 0.783537, "running_max": 0.783537, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 468.52} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:reaction_rates--nohints", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.37728, "area_term": 0.335903, "final_best": 0.542789, "best_raw": 0.542789, "best_effective": 0.542789, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12586238, "usage": {"input": 327, "output": 155392, "cache_read": 11847138, "cache_write": 583381}, "cost_usd": 26.9123, "cost_source": "billed", "wall_clock_seconds": 6212.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1220002, "wall_clock_seconds": 14.7, "raw": 0.209543, "normalized": 0.209543, "discount": 1.0, "effective": 0.209543, "running_max": 0.209543, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.41}, {"index": 2, "tokens": 1628343, "wall_clock_seconds": 12.3, "raw": 0.287332, "normalized": 0.287332, "discount": 1.0, "effective": 0.287332, "running_max": 0.287332, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.03}, {"index": 3, "tokens": 4239705, "wall_clock_seconds": 25.6, "raw": 0.331359, "normalized": 0.331359, "discount": 1.0, "effective": 0.331359, "running_max": 0.331359, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.21}, {"index": 4, "tokens": 4897865, "wall_clock_seconds": 25.5, "raw": 0.418321, "normalized": 0.418321, "discount": 1.0, "effective": 0.418321, "running_max": 0.418321, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.21}, {"index": 5, "tokens": 5241272, "wall_clock_seconds": 25.4, "raw": 0.315203, "normalized": 0.315203, "discount": 1.0, "effective": 0.315203, "running_max": 0.418321, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.01}, {"index": 6, "tokens": 6570937, "wall_clock_seconds": 25.6, "raw": 0.394161, "normalized": 0.394161, "discount": 1.0, "effective": 0.394161, "running_max": 0.418321, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.14}, {"index": 7, "tokens": 9919040, "wall_clock_seconds": 31.0, "raw": 0.48036, "normalized": 0.48036, "discount": 1.0, "effective": 0.48036, "running_max": 0.48036, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.52}, {"index": 8, "tokens": 11270428, "wall_clock_seconds": 73.5, "raw": 0.511886, "normalized": 0.511886, "discount": 1.0, "effective": 0.511886, "running_max": 0.511886, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 73.09}, {"index": 9, "tokens": 11868470, "wall_clock_seconds": 149.5, "raw": 0.542789, "normalized": 0.542789, "discount": 1.0, "effective": 0.542789, "running_max": 0.542789, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 149.28}, {"index": 10, "tokens": 12586238, "wall_clock_seconds": 237.8, "raw": 0.528401, "normalized": 0.528401, "discount": 1.0, "effective": 0.528401, "running_max": 0.542789, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 237.5} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:scan_stitch", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.371975, "area_term": 0.348769, "final_best": 0.464799, "best_raw": 0.829097, "best_effective": 0.464799, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6349258, "usage": {"input": 297, "output": 68530, "cache_read": 6077525, "cache_write": 202906}, "cost_usd": 12.0433, "cost_source": "billed", "wall_clock_seconds": 2586.6, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1410167, "wall_clock_seconds": 0.3, "raw": 0.554608, "normalized": 0.56826, "discount": 0.51, "effective": 0.289813, "running_max": 0.289813, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1647984, "wall_clock_seconds": 0.3, "raw": 0.781073, "normalized": 0.851341, "discount": 0.51, "effective": 0.434184, "running_max": 0.434184, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3057548, "wall_clock_seconds": 0.3, "raw": 0.79658, "normalized": 0.870725, "discount": 0.51, "effective": 0.44407, "running_max": 0.44407, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3513897, "wall_clock_seconds": 0.3, "raw": 0.79428, "normalized": 0.86785, "discount": 0.51, "effective": 0.442604, "running_max": 0.44407, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4000030, "wall_clock_seconds": 0.3, "raw": 0.825232, "normalized": 0.90654, "discount": 0.51, "effective": 0.462335, "running_max": 0.462335, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4654141, "wall_clock_seconds": 0.4, "raw": 0.825622, "normalized": 0.907028, "discount": 0.51, "effective": 0.462584, "running_max": 0.462584, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4788070, "wall_clock_seconds": 0.3, "raw": 0.826639, "normalized": 0.908299, "discount": 0.51, "effective": 0.463233, "running_max": 0.463233, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5478801, "wall_clock_seconds": 0.3, "raw": 0.829097, "normalized": 0.911371, "discount": 0.51, "effective": 0.464799, "running_max": 0.464799, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6055251, "wall_clock_seconds": 0.4, "raw": 0.828238, "normalized": 0.910298, "discount": 0.51, "effective": 0.464252, "running_max": 0.464799, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 6349258, "wall_clock_seconds": 0.3, "raw": 0.827333, "normalized": 0.909166, "discount": 0.51, "effective": 0.463675, "running_max": 0.464799, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:shapes_generalize", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.772753, "area_term": 0.760628, "final_best": 0.82125, "best_raw": 0.857, "best_effective": 0.82125, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2094607, "usage": {"input": 141, "output": 41340, "cache_read": 1971892, "cache_write": 81234}, "cost_usd": 5.0557, "cost_source": "billed", "wall_clock_seconds": 3750.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 173332, "wall_clock_seconds": 3.3, "raw": 0.221, "normalized": 0.02625, "discount": 1.0, "effective": 0.02625, "running_max": 0.02625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.24}, {"index": 2, "tokens": 287773, "wall_clock_seconds": 99.1, "raw": 0.324, "normalized": 0.155, "discount": 1.0, "effective": 0.155, "running_max": 0.155, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.38}, {"index": 3, "tokens": 495834, "wall_clock_seconds": 100.9, "raw": 0.238, "normalized": 0.0475, "discount": 1.0, "effective": 0.0475, "running_max": 0.155, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.25}, {"index": 4, "tokens": 658888, "wall_clock_seconds": 103.9, "raw": 0.024, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.155, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.27}, {"index": 5, "tokens": 746367, "wall_clock_seconds": 0.8, "raw": 0.842, "normalized": 0.8025, "discount": 1.0, "effective": 0.8025, "running_max": 0.8025, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 6, "tokens": 889523, "wall_clock_seconds": 107.3, "raw": 0.846, "normalized": 0.8075, "discount": 1.0, "effective": 0.8075, "running_max": 0.8075, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.68}, {"index": 7, "tokens": 1164125, "wall_clock_seconds": 7.7, "raw": 0.848, "normalized": 0.81, "discount": 1.0, "effective": 0.81, "running_max": 0.81, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.03}, {"index": 8, "tokens": 1537726, "wall_clock_seconds": 10.0, "raw": 0.849, "normalized": 0.81125, "discount": 1.0, "effective": 0.81125, "running_max": 0.81125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.36}, {"index": 9, "tokens": 1801037, "wall_clock_seconds": 7.8, "raw": 0.854, "normalized": 0.8175, "discount": 1.0, "effective": 0.8175, "running_max": 0.8175, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.22}, {"index": 10, "tokens": 2094607, "wall_clock_seconds": 8.7, "raw": 0.857, "normalized": 0.82125, "discount": 1.0, "effective": 0.82125, "running_max": 0.82125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.09} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:shattered_prior", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.404504, "area_term": 0.33063, "final_best": 0.7, "best_raw": 1.0, "best_effective": 0.7, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 11363327, "usage": {"input": 498, "output": 126388, "cache_read": 10866775, "cache_write": 369666}, "cost_usd": 21.812, "cost_source": "billed", "wall_clock_seconds": 4289.6, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.7, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2689600, "wall_clock_seconds": 2.0, "raw": 0.140409, "normalized": 0.140409, "discount": 0.7, "effective": 0.098286, "running_max": 0.098286, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5439704, "wall_clock_seconds": 1.7, "raw": 0.651398, "normalized": 0.651398, "discount": 0.7, "effective": 0.455979, "running_max": 0.455979, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 7060399, "wall_clock_seconds": 1.8, "raw": 0.876369, "normalized": 0.876369, "discount": 0.7, "effective": 0.613458, "running_max": 0.613458, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 10676811, "wall_clock_seconds": 1.8, "raw": 1.0, "normalized": 1.0, "discount": 0.7, "effective": 0.7, "running_max": 0.7, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:shattered_prior--nohints", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.588751, "area_term": 0.485938, "final_best": 1.0, "best_raw": 1.0, "best_effective": 1.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 11649041, "usage": {"input": 276, "output": 119657, "cache_read": 11299005, "cache_write": 230103}, "cost_usd": 20.1609, "cost_source": "billed", "wall_clock_seconds": 3954.0, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1996790, "wall_clock_seconds": 1.6, "raw": 0.153375, "normalized": 0.153375, "discount": 1.0, "effective": 0.153375, "running_max": 0.153375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5042650, "wall_clock_seconds": 1.5, "raw": 0.493693, "normalized": 0.493693, "discount": 1.0, "effective": 0.493693, "running_max": 0.493693, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 6749860, "wall_clock_seconds": 1.7, "raw": 0.876369, "normalized": 0.876369, "discount": 1.0, "effective": 0.876369, "running_max": 0.876369, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 9664575, "wall_clock_seconds": 1.5, "raw": 0.950164, "normalized": 0.950164, "discount": 1.0, "effective": 0.950164, "running_max": 0.950164, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 10974990, "wall_clock_seconds": 1.7, "raw": 1.0, "normalized": 1.0, "discount": 1.0, "effective": 1.0, "running_max": 1.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:ship_tune", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.203321, "area_term": 0.179998, "final_best": 0.296612, "best_raw": 0.296612, "best_effective": 0.296612, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13846148, "usage": {"input": 369, "output": 173798, "cache_read": 11815029, "cache_write": 1856952}, "cost_usd": 43.7205, "cost_source": "billed", "wall_clock_seconds": 29368.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2220489, "wall_clock_seconds": 88.2, "raw": 0.189767, "normalized": 0.189767, "discount": 1.0, "effective": 0.189767, "running_max": 0.189767, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.47}, {"index": 2, "tokens": 3356341, "wall_clock_seconds": 181.2, "raw": 0.249486, "normalized": 0.249486, "discount": 1.0, "effective": 0.249486, "running_max": 0.249486, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 180.51}, {"index": 3, "tokens": 5822266, "wall_clock_seconds": 302.2, "raw": 0.241994, "normalized": 0.241994, "discount": 1.0, "effective": 0.241994, "running_max": 0.249486, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 301.42}, {"index": 4, "tokens": 6112552, "wall_clock_seconds": 125.2, "raw": 0.227148, "normalized": 0.227148, "discount": 1.0, "effective": 0.227148, "running_max": 0.249486, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 124.59}, {"index": 5, "tokens": 7914603, "wall_clock_seconds": 370.3, "raw": 0.251874, "normalized": 0.251874, "discount": 1.0, "effective": 0.251874, "running_max": 0.251874, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 369.49}, {"index": 6, "tokens": 8280010, "wall_clock_seconds": 382.0, "raw": 0.27228, "normalized": 0.27228, "discount": 1.0, "effective": 0.27228, "running_max": 0.27228, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 380.87}, {"index": 7, "tokens": 8648357, "wall_clock_seconds": 355.1, "raw": 0.281009, "normalized": 0.281009, "discount": 1.0, "effective": 0.281009, "running_max": 0.281009, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 354.3}, {"index": 8, "tokens": 12261918, "wall_clock_seconds": 344.6, "raw": 0.28172, "normalized": 0.28172, "discount": 1.0, "effective": 0.28172, "running_max": 0.28172, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 343.88}, {"index": 9, "tokens": 13451548, "wall_clock_seconds": 559.2, "raw": 0.284722, "normalized": 0.284722, "discount": 1.0, "effective": 0.284722, "running_max": 0.284722, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 558.36}, {"index": 10, "tokens": 13846148, "wall_clock_seconds": 551.4, "raw": 0.296612, "normalized": 0.296612, "discount": 1.0, "effective": 0.296612, "running_max": 0.296612, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 550.61} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:splash_generalize", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.368624, "area_term": 0.355919, "final_best": 0.419444, "best_raw": 0.855, "best_effective": 0.419444, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5095000, "usage": {"input": 360, "output": 97981, "cache_read": 4770719, "cache_write": 225940}, "cost_usd": 12.4976, "cost_source": "billed", "wall_clock_seconds": 3728.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.5, "before_submission": 6}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 97390, "wall_clock_seconds": 1.9, "raw": 0.107, "normalized": 0.007778, "discount": 1.0, "effective": 0.007778, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.39}, {"index": 2, "tokens": 143964, "wall_clock_seconds": 6.3, "raw": 0.09, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.8}, {"index": 3, "tokens": 196503, "wall_clock_seconds": 2.0, "raw": 0.103, "normalized": 0.003333, "discount": 1.0, "effective": 0.003333, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.34}, {"index": 4, "tokens": 375623, "wall_clock_seconds": 4.8, "raw": 0.192, "normalized": 0.102222, "discount": 1.0, "effective": 0.102222, "running_max": 0.102222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.21}, {"index": 5, "tokens": 563288, "wall_clock_seconds": 5.5, "raw": 0.312, "normalized": 0.235556, "discount": 1.0, "effective": 0.235556, "running_max": 0.235556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.03}, {"index": 6, "tokens": 1710930, "wall_clock_seconds": 29.0, "raw": 0.667, "normalized": 0.63, "discount": 0.5, "effective": 0.315, "running_max": 0.315, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 28.38}, {"index": 7, "tokens": 2186816, "wall_clock_seconds": 34.4, "raw": 0.708, "normalized": 0.675556, "discount": 0.5, "effective": 0.337778, "running_max": 0.337778, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 33.77}, {"index": 8, "tokens": 2754018, "wall_clock_seconds": 33.3, "raw": 0.829, "normalized": 0.81, "discount": 0.5, "effective": 0.405, "running_max": 0.405, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 32.7}, {"index": 9, "tokens": 3290611, "wall_clock_seconds": 50.6, "raw": 0.841, "normalized": 0.823333, "discount": 0.5, "effective": 0.411667, "running_max": 0.411667, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 50.13}, {"index": 10, "tokens": 5095000, "wall_clock_seconds": 87.3, "raw": 0.855, "normalized": 0.838889, "discount": 0.5, "effective": 0.419444, "running_max": 0.419444, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 86.7} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:tod_pipeline--nohints", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.204331, "area_term": 0.193586, "final_best": 0.24731, "best_raw": 0.24731, "best_effective": 0.24731, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5803480, "usage": {"input": 225, "output": 89103, "cache_read": 5550766, "cache_write": 163386}, "cost_usd": 12.0505, "cost_source": "billed", "wall_clock_seconds": 4733.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 609703, "wall_clock_seconds": 27.9, "raw": 0.083438, "normalized": 0.083438, "discount": 1.0, "effective": 0.083438, "running_max": 0.083438, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.69}, {"index": 2, "tokens": 1060936, "wall_clock_seconds": 32.7, "raw": 0.097425, "normalized": 0.097425, "discount": 1.0, "effective": 0.097425, "running_max": 0.097425, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 32.37}, {"index": 3, "tokens": 1536276, "wall_clock_seconds": 31.1, "raw": 0.117709, "normalized": 0.117709, "discount": 1.0, "effective": 0.117709, "running_max": 0.117709, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.86}, {"index": 4, "tokens": 2268336, "wall_clock_seconds": 40.5, "raw": 0.243286, "normalized": 0.243286, "discount": 1.0, "effective": 0.243286, "running_max": 0.243286, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 40.23}, {"index": 5, "tokens": 3361904, "wall_clock_seconds": 54.4, "raw": 0.24731, "normalized": 0.24731, "discount": 1.0, "effective": 0.24731, "running_max": 0.24731, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 54.15}, {"index": 6, "tokens": 4052037, "wall_clock_seconds": 182.6, "raw": 0.240863, "normalized": 0.240863, "discount": 1.0, "effective": 0.240863, "running_max": 0.24731, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 182.36}, {"index": 7, "tokens": 4420829, "wall_clock_seconds": 242.1, "raw": 0.241339, "normalized": 0.241339, "discount": 1.0, "effective": 0.241339, "running_max": 0.24731, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 241.82}, {"index": 8, "tokens": 4940206, "wall_clock_seconds": 257.6, "raw": 0.216827, "normalized": 0.216827, "discount": 1.0, "effective": 0.216827, "running_max": 0.24731, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 256.99}, {"index": 9, "tokens": 5361968, "wall_clock_seconds": 100.1, "raw": 0.22517, "normalized": 0.22517, "discount": 1.0, "effective": 0.22517, "running_max": 0.24731, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.87}, {"index": 10, "tokens": 5803480, "wall_clock_seconds": 150.8, "raw": 0.221041, "normalized": 0.221041, "discount": 1.0, "effective": 0.221041, "running_max": 0.24731, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 150.53} ]}, {"sample_key": "aQThvAhZwd4xUVD9Cfwzy3:weirdml_bonanza", "eval_id": "aQThvAhZwd4xUVD9Cfwzy3", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "3c75c6b66453f59710b88a583ae25ac2c46f3d6fcf494d78a6624f64211a50d5", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.80542, "area_term": 0.789293, "final_best": 0.869931, "best_raw": 0.869931, "best_effective": 0.869931, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13294448, "usage": {"input": 372, "output": 199312, "cache_read": 12529749, "cache_write": 565015}, "cost_usd": 29.5618, "cost_source": "billed", "wall_clock_seconds": 9058.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 443875, "wall_clock_seconds": 103.9, "raw": 0.631086, "normalized": 0.631086, "discount": 1.0, "effective": 0.631086, "running_max": 0.631086, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.39}, {"index": 2, "tokens": 1886953, "wall_clock_seconds": 106.0, "raw": 0.820127, "normalized": 0.820127, "discount": 1.0, "effective": 0.820127, "running_max": 0.820127, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.44}, {"index": 3, "tokens": 2970858, "wall_clock_seconds": 105.9, "raw": 0.842582, "normalized": 0.842582, "discount": 1.0, "effective": 0.842582, "running_max": 0.842582, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.36}, {"index": 4, "tokens": 3548844, "wall_clock_seconds": 108.1, "raw": 0.835353, "normalized": 0.835353, "discount": 1.0, "effective": 0.835353, "running_max": 0.842582, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.41}, {"index": 5, "tokens": 5493045, "wall_clock_seconds": 115.8, "raw": 0.837941, "normalized": 0.837941, "discount": 1.0, "effective": 0.837941, "running_max": 0.842582, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 115.19}, {"index": 6, "tokens": 7080190, "wall_clock_seconds": 116.8, "raw": 0.842812, "normalized": 0.842812, "discount": 1.0, "effective": 0.842812, "running_max": 0.842812, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 116.23}, {"index": 7, "tokens": 8372413, "wall_clock_seconds": 115.1, "raw": 0.821582, "normalized": 0.821582, "discount": 1.0, "effective": 0.821582, "running_max": 0.842812, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 114.57}, {"index": 8, "tokens": 9177764, "wall_clock_seconds": 114.6, "raw": 0.867724, "normalized": 0.867724, "discount": 1.0, "effective": 0.867724, "running_max": 0.867724, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 114.09}, {"index": 9, "tokens": 11227431, "wall_clock_seconds": 117.9, "raw": 0.868137, "normalized": 0.868137, "discount": 1.0, "effective": 0.868137, "running_max": 0.868137, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 117.32}, {"index": 10, "tokens": 13294448, "wall_clock_seconds": 118.5, "raw": 0.869931, "normalized": 0.869931, "discount": 1.0, "effective": 0.869931, "running_max": 0.869931, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 117.94} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:mystery_box", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.158172, "area_term": 0.093383, "final_best": 0.417329, "best_raw": 0.638172, "best_effective": 0.417329, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 27511633, "usage": {"input": 24816, "output": 296497, "cache_read": 26822038, "cache_write": 368282}, "cost_usd": 29.1442, "cost_source": "list_price", "wall_clock_seconds": 8443.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 17421909, "wall_clock_seconds": 3.8, "raw": 0.561088, "normalized": 0.605916, "discount": 0.6, "effective": 0.36355, "running_max": 0.36355, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 18381316, "wall_clock_seconds": 3.6, "raw": 0.561228, "normalized": 0.606079, "discount": 0.6, "effective": 0.363647, "running_max": 0.363647, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 20028452, "wall_clock_seconds": 3.9, "raw": 0.578553, "normalized": 0.626224, "discount": 0.6, "effective": 0.375734, "running_max": 0.375734, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 21037716, "wall_clock_seconds": 3.7, "raw": 0.62582, "normalized": 0.681187, "discount": 0.6, "effective": 0.408712, "running_max": 0.408712, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 22061796, "wall_clock_seconds": 3.6, "raw": 0.638172, "normalized": 0.695549, "discount": 0.6, "effective": 0.417329, "running_max": 0.417329, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 23103415, "wall_clock_seconds": 3.7, "raw": 0.339997, "normalized": 0.348834, "discount": 0.6, "effective": 0.2093, "running_max": 0.417329, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 24898799, "wall_clock_seconds": 3.1, "raw": 0.578835, "normalized": 0.626552, "discount": 0.6, "effective": 0.375931, "running_max": 0.417329, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 25638149, "wall_clock_seconds": 3.4, "raw": 0.635457, "normalized": 0.692392, "discount": 0.6, "effective": 0.415435, "running_max": 0.417329, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 26381824, "wall_clock_seconds": 3.3, "raw": 0.636786, "normalized": 0.693937, "discount": 0.6, "effective": 0.416362, "running_max": 0.417329, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 27511633, "wall_clock_seconds": 4.9, "raw": 0.635457, "normalized": 0.692392, "discount": 0.6, "effective": 0.415435, "running_max": 0.417329, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:mystery_box--nohints", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.023855, "area_term": 0.023855, "final_best": 0.023855, "best_raw": 0.060516, "best_effective": 0.023855, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9836592, "usage": {"input": 4170, "output": 305346, "cache_read": 9201403, "cache_write": 325673}, "cost_usd": 24.1228, "cost_source": "list_price", "wall_clock_seconds": 4927.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 188910, "wall_clock_seconds": 4.0, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 445546, "wall_clock_seconds": 4.1, "raw": 0.060516, "normalized": 0.023855, "discount": 1.0, "effective": 0.023855, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 576507, "wall_clock_seconds": 3.1, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 986090, "wall_clock_seconds": 4.2, "raw": 0.041594, "normalized": 0.001854, "discount": 1.0, "effective": 0.001854, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1780664, "wall_clock_seconds": 5.2, "raw": 0.038781, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 2564502, "wall_clock_seconds": 2.1, "raw": 0.05207, "normalized": 0.014035, "discount": 1.0, "effective": 0.014035, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 2752576, "wall_clock_seconds": 2.2, "raw": 0.037374, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4758087, "wall_clock_seconds": 3.8, "raw": 0.041602, "normalized": 0.001863, "discount": 1.0, "effective": 0.001863, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6083468, "wall_clock_seconds": 2.1, "raw": 0.042564, "normalized": 0.002981, "discount": 1.0, "effective": 0.002981, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9836592, "wall_clock_seconds": 5.5, "raw": 0.051318, "normalized": 0.01316, "discount": 1.0, "effective": 0.01316, "running_max": 0.023855, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:reaction_rates", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.166172, "area_term": 0.113448, "final_best": 0.377069, "best_raw": 0.673337, "best_effective": 0.377069, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 31883531, "usage": {"input": 16007, "output": 261847, "cache_read": 31158314, "cache_write": 447363}, "cost_usd": 29.9893, "cost_source": "list_price", "wall_clock_seconds": 22769.7, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.8, "before_submission": 1}, {"id": 1, "multiplier": 0.7, "before_submission": 5}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 11658650, "wall_clock_seconds": 75.6, "raw": 0.312541, "normalized": 0.312541, "discount": 0.8, "effective": 0.250033, "running_max": 0.250033, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 74.98}, {"index": 2, "tokens": 12597531, "wall_clock_seconds": 157.7, "raw": 0.3949, "normalized": 0.3949, "discount": 0.8, "effective": 0.31592, "running_max": 0.31592, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 155.82}, {"index": 3, "tokens": 13578085, "wall_clock_seconds": 146.6, "raw": 0.452247, "normalized": 0.452247, "discount": 0.8, "effective": 0.361798, "running_max": 0.361798, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 145.73}, {"index": 4, "tokens": 15281979, "wall_clock_seconds": 151.6, "raw": 0.435266, "normalized": 0.435266, "discount": 0.8, "effective": 0.348213, "running_max": 0.361798, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 151.03}, {"index": 5, "tokens": 21172811, "wall_clock_seconds": 206.5, "raw": 0.541704, "normalized": 0.541704, "discount": 0.56, "effective": 0.303354, "running_max": 0.361798, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 206.03}, {"index": 6, "tokens": 24407992, "wall_clock_seconds": 216.4, "raw": 0.557176, "normalized": 0.557176, "discount": 0.56, "effective": 0.312019, "running_max": 0.361798, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 215.91}, {"index": 7, "tokens": 26527737, "wall_clock_seconds": 256.3, "raw": 0.590301, "normalized": 0.590301, "discount": 0.56, "effective": 0.330569, "running_max": 0.361798, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 255.75}, {"index": 8, "tokens": 27824039, "wall_clock_seconds": 264.0, "raw": 0.597311, "normalized": 0.597311, "discount": 0.56, "effective": 0.334494, "running_max": 0.361798, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 262.63}, {"index": 9, "tokens": 30055013, "wall_clock_seconds": 281.3, "raw": 0.673337, "normalized": 0.673337, "discount": 0.56, "effective": 0.377069, "running_max": 0.377069, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 280.82}, {"index": 10, "tokens": 31883531, "wall_clock_seconds": 300.0, "raw": 0.655738, "normalized": 0.655738, "discount": 0.56, "effective": 0.367213, "running_max": 0.377069, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 299.51} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:reaction_rates--nohints", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.152574, "area_term": 0.126653, "final_best": 0.256258, "best_raw": 0.256258, "best_effective": 0.256258, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 16163597, "usage": {"input": 11011, "output": 204912, "cache_read": 15616078, "cache_write": 331596}, "cost_usd": 20.8916, "cost_source": "list_price", "wall_clock_seconds": 9898.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2319592, "wall_clock_seconds": 7.5, "raw": 0.121734, "normalized": 0.121734, "discount": 1.0, "effective": 0.121734, "running_max": 0.121734, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.86}, {"index": 2, "tokens": 6144840, "wall_clock_seconds": 29.5, "raw": 0.140627, "normalized": 0.140627, "discount": 1.0, "effective": 0.140627, "running_max": 0.140627, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.97}, {"index": 3, "tokens": 7624239, "wall_clock_seconds": 16.8, "raw": 0.163246, "normalized": 0.163246, "discount": 1.0, "effective": 0.163246, "running_max": 0.163246, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 16.18}, {"index": 4, "tokens": 9575448, "wall_clock_seconds": 14.2, "raw": 0.182935, "normalized": 0.182935, "discount": 1.0, "effective": 0.182935, "running_max": 0.182935, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.68}, {"index": 5, "tokens": 11413718, "wall_clock_seconds": 45.7, "raw": 0.211708, "normalized": 0.211708, "discount": 1.0, "effective": 0.211708, "running_max": 0.211708, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 45.18}, {"index": 6, "tokens": 12054913, "wall_clock_seconds": 40.2, "raw": 0.201111, "normalized": 0.201111, "discount": 1.0, "effective": 0.201111, "running_max": 0.211708, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 39.66}, {"index": 7, "tokens": 13716769, "wall_clock_seconds": 66.6, "raw": 0.20615, "normalized": 0.20615, "discount": 1.0, "effective": 0.20615, "running_max": 0.211708, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 66.08}, {"index": 8, "tokens": 14758879, "wall_clock_seconds": 44.0, "raw": 0.224451, "normalized": 0.224451, "discount": 1.0, "effective": 0.224451, "running_max": 0.224451, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 43.5}, {"index": 9, "tokens": 15459277, "wall_clock_seconds": 43.7, "raw": 0.242323, "normalized": 0.242323, "discount": 1.0, "effective": 0.242323, "running_max": 0.242323, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 43.17}, {"index": 10, "tokens": 16163597, "wall_clock_seconds": 39.8, "raw": 0.256258, "normalized": 0.256258, "discount": 1.0, "effective": 0.256258, "running_max": 0.256258, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 39.33} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:scan_stitch", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.197676, "area_term": 0.176761, "final_best": 0.281336, "best_raw": 0.743054, "best_effective": 0.281336, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 10056474, "usage": {"input": 4432, "output": 170099, "cache_read": 9655474, "cache_write": 226469}, "cost_usd": 15.4925, "cost_source": "list_price", "wall_clock_seconds": 6029.6, "stop_reason": "agent_finished", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2480441, "wall_clock_seconds": 0.3, "raw": 0.555445, "normalized": 0.569306, "discount": 0.35, "effective": 0.199257, "running_max": 0.199257, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3063971, "wall_clock_seconds": 0.3, "raw": 0.671991, "normalized": 0.714988, "discount": 0.35, "effective": 0.250246, "running_max": 0.250246, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4189289, "wall_clock_seconds": 0.3, "raw": 0.703612, "normalized": 0.754515, "discount": 0.35, "effective": 0.26408, "running_max": 0.26408, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 5262098, "wall_clock_seconds": 0.3, "raw": 0.743054, "normalized": 0.803818, "discount": 0.35, "effective": 0.281336, "running_max": 0.281336, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 6026343, "wall_clock_seconds": 0.3, "raw": 0.743054, "normalized": 0.803818, "discount": 0.35, "effective": 0.281336, "running_max": 0.281336, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 7917342, "wall_clock_seconds": 0.3, "raw": 0.741773, "normalized": 0.802216, "discount": 0.35, "effective": 0.280776, "running_max": 0.281336, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 9084786, "wall_clock_seconds": 0.4, "raw": 0.743054, "normalized": 0.803818, "discount": 0.35, "effective": 0.281336, "running_max": 0.281336, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 9568504, "wall_clock_seconds": 0.3, "raw": 0.742576, "normalized": 0.803219, "discount": 0.35, "effective": 0.281127, "running_max": 0.281336, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 9812095, "wall_clock_seconds": 0.3, "raw": 0.742238, "normalized": 0.802797, "discount": 0.35, "effective": 0.280979, "running_max": 0.281336, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:scan_stitch--nohints", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.00056, "area_term": 0.000547, "final_best": 0.000616, "best_raw": 0.100493, "best_effective": 0.000616, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5895453, "usage": {"input": 4039, "output": 156066, "cache_read": 5544718, "cache_write": 190630}, "cost_usd": 13.0425, "cost_source": "list_price", "wall_clock_seconds": 2629.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 656012, "wall_clock_seconds": 0.3, "raw": 0.100437, "normalized": 0.000546, "discount": 1.0, "effective": 0.000546, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 816008, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 991587, "wall_clock_seconds": 0.8, "raw": 0.06467, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2826429, "wall_clock_seconds": 0.5, "raw": 0.080305, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3110876, "wall_clock_seconds": 0.3, "raw": 0.069874, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3861665, "wall_clock_seconds": 0.3, "raw": 0.093842, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4362865, "wall_clock_seconds": 0.3, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4708408, "wall_clock_seconds": 0.3, "raw": 0.100437, "normalized": 0.000546, "discount": 1.0, "effective": 0.000546, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5064447, "wall_clock_seconds": 0.2, "raw": 0.070963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000546, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5895453, "wall_clock_seconds": 0.3, "raw": 0.100493, "normalized": 0.000616, "discount": 1.0, "effective": 0.000616, "running_max": 0.000616, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:shapes_generalize", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.567939, "area_term": 0.492423, "final_best": 0.87, "best_raw": 0.896, "best_effective": 0.87, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9939259, "usage": {"input": 14158, "output": 136919, "cache_read": 9581516, "cache_write": 206666}, "cost_usd": 13.5162, "cost_source": "list_price", "wall_clock_seconds": 15544.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1070400, "wall_clock_seconds": 7.9, "raw": 0.211, "normalized": 0.01375, "discount": 1.0, "effective": 0.01375, "running_max": 0.01375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.44}, {"index": 2, "tokens": 1295516, "wall_clock_seconds": 19.9, "raw": 0.24, "normalized": 0.05, "discount": 1.0, "effective": 0.05, "running_max": 0.05, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 19.34}, {"index": 3, "tokens": 3244137, "wall_clock_seconds": 8.4, "raw": 0.336, "normalized": 0.17, "discount": 1.0, "effective": 0.17, "running_max": 0.17, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.68}, {"index": 4, "tokens": 3619403, "wall_clock_seconds": 11.1, "raw": 0.789, "normalized": 0.73625, "discount": 1.0, "effective": 0.73625, "running_max": 0.73625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 10.35}, {"index": 5, "tokens": 4150434, "wall_clock_seconds": 8.8, "raw": 0.836, "normalized": 0.795, "discount": 1.0, "effective": 0.795, "running_max": 0.795, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.94}, {"index": 6, "tokens": 5005479, "wall_clock_seconds": 11.4, "raw": 0.834, "normalized": 0.7925, "discount": 1.0, "effective": 0.7925, "running_max": 0.795, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 10.71}, {"index": 7, "tokens": 5301688, "wall_clock_seconds": 27.6, "raw": 0.822, "normalized": 0.7775, "discount": 1.0, "effective": 0.7775, "running_max": 0.795, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.98}, {"index": 8, "tokens": 7550928, "wall_clock_seconds": 19.6, "raw": 0.828, "normalized": 0.785, "discount": 1.0, "effective": 0.785, "running_max": 0.795, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.8}, {"index": 9, "tokens": 8707692, "wall_clock_seconds": 14.5, "raw": 0.834, "normalized": 0.7925, "discount": 1.0, "effective": 0.7925, "running_max": 0.795, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.89}, {"index": 10, "tokens": 9939259, "wall_clock_seconds": 14.6, "raw": 0.896, "normalized": 0.87, "discount": 1.0, "effective": 0.87, "running_max": 0.87, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.93} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:shattered_prior", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.184228, "area_term": 0.036711, "final_best": 0.774298, "best_raw": 0.774298, "best_effective": 0.774298, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 48142549, "usage": {"input": 6967, "output": 358133, "cache_read": 47236858, "cache_write": 540591}, "cost_usd": 40.5974, "cost_source": "list_price", "wall_clock_seconds": 8255.8, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 38629453, "wall_clock_seconds": 1.6, "raw": 0.494333, "normalized": 0.494333, "discount": 1.0, "effective": 0.494333, "running_max": 0.494333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 40638012, "wall_clock_seconds": 1.6, "raw": 0.651398, "normalized": 0.651398, "discount": 1.0, "effective": 0.651398, "running_max": 0.651398, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 46486421, "wall_clock_seconds": 1.6, "raw": 0.774298, "normalized": 0.774298, "discount": 1.0, "effective": 0.774298, "running_max": 0.774298, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:shattered_prior--nohints", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.020789, "area_term": 0.008385, "final_best": 0.070408, "best_raw": 0.070408, "best_effective": 0.070408, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 47316498, "usage": {"input": 7081, "output": 405486, "cache_read": 46364354, "cache_write": 539577}, "cost_usd": 42.7277, "cost_source": "list_price", "wall_clock_seconds": 18780.7, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 24595238, "wall_clock_seconds": 1.7, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 35770949, "wall_clock_seconds": 2.0, "raw": 0.034143, "normalized": 0.034143, "discount": 1.0, "effective": 0.034143, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 38435047, "wall_clock_seconds": 1.6, "raw": 0.032105, "normalized": 0.032105, "discount": 1.0, "effective": 0.032105, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 41177108, "wall_clock_seconds": 1.7, "raw": 0.068049, "normalized": 0.068049, "discount": 1.0, "effective": 0.068049, "running_max": 0.068049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 42841465, "wall_clock_seconds": 1.6, "raw": 0.069221, "normalized": 0.069221, "discount": 1.0, "effective": 0.069221, "running_max": 0.069221, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 43954791, "wall_clock_seconds": 1.7, "raw": 0.051478, "normalized": 0.051478, "discount": 1.0, "effective": 0.051478, "running_max": 0.069221, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 45633453, "wall_clock_seconds": 1.9, "raw": 0.070408, "normalized": 0.070408, "discount": 1.0, "effective": 0.070408, "running_max": 0.070408, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 46754910, "wall_clock_seconds": 1.8, "raw": 0.069948, "normalized": 0.069948, "discount": 1.0, "effective": 0.069948, "running_max": 0.070408, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:ship_detect", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.061869, "area_term": 0.04898, "final_best": 0.113423, "best_raw": 0.113423, "best_effective": 0.113423, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": null, "tokens_total": 32818355, "usage": {"input": 31937, "output": 177975, "cache_read": 31318759, "cache_write": 1289684}, "cost_usd": 42.8415, "cost_source": "list_price", "wall_clock_seconds": 40562.9, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 12, "tokens": 437163, "cost_weighted": 28835.6, "wall_clock_seconds": 914.2, "raw": 0.015698, "normalized": 0.015698, "effective": 0.015698, "running_max": 0.015698, "scored": true, "final": false}, {"index": 2, "model_call": 42, "tokens": 2993377, "cost_weighted": 105922.1, "wall_clock_seconds": 7526.6, "raw": 0.031786, "normalized": 0.031786, "effective": 0.031786, "running_max": 0.031786, "scored": true, "final": false}, {"index": 3, "model_call": 51, "tokens": 4130536, "cost_weighted": 124026.3, "wall_clock_seconds": 10010.3, "raw": 0.039182, "normalized": 0.039182, "effective": 0.039182, "running_max": 0.039182, "scored": true, "final": false}, {"index": 4, "model_call": 59, "tokens": 5255496, "cost_weighted": 144334.3, "wall_clock_seconds": 13367.9, "raw": 0.038778, "normalized": 0.038778, "effective": 0.038778, "running_max": 0.039182, "scored": true, "final": false}, {"index": 5, "model_call": 103, "tokens": 13659712, "cost_weighted": 255689.2, "wall_clock_seconds": 18573.5, "raw": 0.059269, "normalized": 0.059269, "effective": 0.059269, "running_max": 0.059269, "scored": true, "final": false}, {"index": 6, "model_call": 164, "tokens": 26222922, "cost_weighted": 784312.1, "wall_clock_seconds": 27316.6, "raw": 0.091797, "normalized": 0.091797, "effective": 0.091797, "running_max": 0.091797, "scored": true, "final": false}, {"index": 7, "model_call": 177, "tokens": 29145254, "cost_weighted": 813293.7, "wall_clock_seconds": 33758.0, "raw": 0.111565, "normalized": 0.111565, "effective": 0.111565, "running_max": 0.111565, "scored": true, "final": false}, {"index": 8, "model_call": 185, "tokens": 31067329, "cost_weighted": 833898.0, "wall_clock_seconds": 39110.8, "raw": 0.113423, "normalized": 0.113423, "effective": 0.113423, "running_max": 0.113423, "scored": true, "final": false} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:ship_tune", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.076578, "area_term": 0.052895, "final_best": 0.17131, "best_raw": 0.17131, "best_effective": 0.17131, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 37730061, "usage": {"input": 46759, "output": 328894, "cache_read": 36749439, "cache_write": 604969}, "cost_usd": 38.199, "cost_source": "list_price", "wall_clock_seconds": 31983.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7085187, "wall_clock_seconds": 404.5, "raw": 0.069317, "normalized": 0.069317, "discount": 1.0, "effective": 0.069317, "running_max": 0.069317, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 403.97}, {"index": 2, "tokens": 10149765, "wall_clock_seconds": 450.7, "raw": 0.084532, "normalized": 0.084532, "discount": 1.0, "effective": 0.084532, "running_max": 0.084532, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 449.93}, {"index": 3, "tokens": 13517746, "wall_clock_seconds": 334.6, "raw": 0.109625, "normalized": 0.109625, "discount": 1.0, "effective": 0.109625, "running_max": 0.109625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 334.1}, {"index": 4, "tokens": 16764338, "wall_clock_seconds": 411.2, "raw": 0.116752, "normalized": 0.116752, "discount": 1.0, "effective": 0.116752, "running_max": 0.116752, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 410.62}, {"index": 5, "tokens": 19317955, "wall_clock_seconds": 482.0, "raw": 0.150208, "normalized": 0.150208, "discount": 1.0, "effective": 0.150208, "running_max": 0.150208, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 481.25}, {"index": 6, "tokens": 22319537, "wall_clock_seconds": 617.1, "raw": 0.160715, "normalized": 0.160715, "discount": 1.0, "effective": 0.160715, "running_max": 0.160715, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 616.45}, {"index": 7, "tokens": 28543792, "wall_clock_seconds": 551.9, "raw": 0.157153, "normalized": 0.157153, "discount": 1.0, "effective": 0.157153, "running_max": 0.160715, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 551.02}, {"index": 8, "tokens": 30132199, "wall_clock_seconds": 853.1, "raw": 0.14245, "normalized": 0.14245, "discount": 1.0, "effective": 0.14245, "running_max": 0.160715, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 852.36}, {"index": 9, "tokens": 31432204, "wall_clock_seconds": 583.1, "raw": 0.159413, "normalized": 0.159413, "discount": 1.0, "effective": 0.159413, "running_max": 0.160715, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 582.41}, {"index": 10, "tokens": 37730061, "wall_clock_seconds": 569.2, "raw": 0.17131, "normalized": 0.17131, "discount": 1.0, "effective": 0.17131, "running_max": 0.17131, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 568.52} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:splash_generalize", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.566407, "area_term": 0.539953, "final_best": 0.672222, "best_raw": 0.705, "best_effective": 0.672222, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5725321, "usage": {"input": 4948, "output": 110681, "cache_read": 5453765, "cache_write": 155927}, "cost_usd": 10.0655, "cost_source": "list_price", "wall_clock_seconds": 4054.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1185020, "wall_clock_seconds": 61.6, "raw": 0.657, "normalized": 0.618889, "discount": 1.0, "effective": 0.618889, "running_max": 0.618889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 61.02}, {"index": 2, "tokens": 1527475, "wall_clock_seconds": 63.5, "raw": 0.679, "normalized": 0.643333, "discount": 1.0, "effective": 0.643333, "running_max": 0.643333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 62.7}, {"index": 3, "tokens": 2031980, "wall_clock_seconds": 83.8, "raw": 0.69, "normalized": 0.655556, "discount": 1.0, "effective": 0.655556, "running_max": 0.655556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 83.22}, {"index": 4, "tokens": 2375420, "wall_clock_seconds": 74.3, "raw": 0.696, "normalized": 0.662222, "discount": 1.0, "effective": 0.662222, "running_max": 0.662222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 73.66}, {"index": 5, "tokens": 2879518, "wall_clock_seconds": 79.6, "raw": 0.699, "normalized": 0.665556, "discount": 1.0, "effective": 0.665556, "running_max": 0.665556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 78.94}, {"index": 6, "tokens": 3288776, "wall_clock_seconds": 77.9, "raw": 0.689, "normalized": 0.654444, "discount": 1.0, "effective": 0.654444, "running_max": 0.665556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 77.27}, {"index": 7, "tokens": 3722388, "wall_clock_seconds": 93.7, "raw": 0.692, "normalized": 0.657778, "discount": 1.0, "effective": 0.657778, "running_max": 0.665556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 92.88}, {"index": 8, "tokens": 4348070, "wall_clock_seconds": 74.8, "raw": 0.705, "normalized": 0.672222, "discount": 1.0, "effective": 0.672222, "running_max": 0.672222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 74.23}, {"index": 9, "tokens": 5020837, "wall_clock_seconds": 90.3, "raw": 0.686, "normalized": 0.651111, "discount": 1.0, "effective": 0.651111, "running_max": 0.672222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 89.75}, {"index": 10, "tokens": 5725321, "wall_clock_seconds": 86.1, "raw": 0.699, "normalized": 0.665556, "discount": 1.0, "effective": 0.665556, "running_max": 0.672222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 84.85} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:tod_pipeline--nohints", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.114063, "area_term": 0.074884, "final_best": 0.270776, "best_raw": 0.270776, "best_effective": 0.270776, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 28772160, "usage": {"input": 38249, "output": 303315, "cache_read": 27987240, "cache_write": 443356}, "cost_usd": 31.4122, "cost_source": "list_price", "wall_clock_seconds": 5907.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 8814706, "wall_clock_seconds": 19.9, "raw": 0.131387, "normalized": 0.131387, "discount": 1.0, "effective": 0.131387, "running_max": 0.131387, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 19.56}, {"index": 2, "tokens": 14437020, "wall_clock_seconds": 24.9, "raw": 0.055518, "normalized": 0.055518, "discount": 1.0, "effective": 0.055518, "running_max": 0.131387, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 24.46}, {"index": 3, "tokens": 17613700, "wall_clock_seconds": 21.9, "raw": 0.170659, "normalized": 0.170659, "discount": 1.0, "effective": 0.170659, "running_max": 0.170659, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.59}, {"index": 4, "tokens": 21085568, "wall_clock_seconds": 22.4, "raw": 0.127157, "normalized": 0.127157, "discount": 1.0, "effective": 0.127157, "running_max": 0.170659, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.06}, {"index": 5, "tokens": 22288265, "wall_clock_seconds": 30.6, "raw": 0.235979, "normalized": 0.235979, "discount": 1.0, "effective": 0.235979, "running_max": 0.235979, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.29}, {"index": 6, "tokens": 23513646, "wall_clock_seconds": 31.6, "raw": 0.250186, "normalized": 0.250186, "discount": 1.0, "effective": 0.250186, "running_max": 0.250186, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 31.26}, {"index": 7, "tokens": 24770771, "wall_clock_seconds": 47.7, "raw": 0.250187, "normalized": 0.250187, "discount": 1.0, "effective": 0.250187, "running_max": 0.250187, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 47.35}, {"index": 8, "tokens": 25631464, "wall_clock_seconds": 47.1, "raw": 0.253902, "normalized": 0.253902, "discount": 1.0, "effective": 0.253902, "running_max": 0.253902, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 46.78}, {"index": 9, "tokens": 27861161, "wall_clock_seconds": 46.4, "raw": 0.270776, "normalized": 0.270776, "discount": 1.0, "effective": 0.270776, "running_max": 0.270776, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 45.31}, {"index": 10, "tokens": 28772160, "wall_clock_seconds": 47.6, "raw": 0.266583, "normalized": 0.266583, "discount": 1.0, "effective": 0.266583, "running_max": 0.270776, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 47.23} ]}, {"sample_key": "eUbKFJ5Cso73wQSDnEMLqb:weirdml_bonanza", "eval_id": "eUbKFJ5Cso73wQSDnEMLqb", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "361c4df", "source_log_sha256": "43da104ed6e9784a9fd487c64d5df87f22bb588c214c733a1a041bf7aaaaf11f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.468885, "area_term": 0.380333, "final_best": 0.823092, "best_raw": 0.823092, "best_effective": 0.823092, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 15795183, "usage": {"input": 34318, "output": 190612, "cache_read": 15270111, "cache_write": 300142}, "cost_usd": 19.6941, "cost_source": "list_price", "wall_clock_seconds": 6889.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5764969, "wall_clock_seconds": 82.8, "raw": 0.744731, "normalized": 0.744731, "discount": 1.0, "effective": 0.744731, "running_max": 0.744731, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 81.49}, {"index": 2, "tokens": 7026840, "wall_clock_seconds": 95.8, "raw": 0.797261, "normalized": 0.797261, "discount": 1.0, "effective": 0.797261, "running_max": 0.797261, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.1}, {"index": 3, "tokens": 9180440, "wall_clock_seconds": 103.3, "raw": 0.80918, "normalized": 0.80918, "discount": 1.0, "effective": 0.80918, "running_max": 0.80918, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.22}, {"index": 4, "tokens": 9957192, "wall_clock_seconds": 89.4, "raw": 0.805363, "normalized": 0.805363, "discount": 1.0, "effective": 0.805363, "running_max": 0.80918, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 88.41}, {"index": 5, "tokens": 10492712, "wall_clock_seconds": 75.9, "raw": 0.806327, "normalized": 0.806327, "discount": 1.0, "effective": 0.806327, "running_max": 0.80918, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.29}, {"index": 6, "tokens": 11599126, "wall_clock_seconds": 78.4, "raw": 0.816622, "normalized": 0.816622, "discount": 1.0, "effective": 0.816622, "running_max": 0.816622, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 77.64}, {"index": 7, "tokens": 12452051, "wall_clock_seconds": 95.4, "raw": 0.816816, "normalized": 0.816816, "discount": 1.0, "effective": 0.816816, "running_max": 0.816816, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.72}, {"index": 8, "tokens": 13033565, "wall_clock_seconds": 87.3, "raw": 0.822322, "normalized": 0.822322, "discount": 1.0, "effective": 0.822322, "running_max": 0.822322, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.52}, {"index": 9, "tokens": 13920837, "wall_clock_seconds": 88.4, "raw": 0.821492, "normalized": 0.821492, "discount": 1.0, "effective": 0.821492, "running_max": 0.822322, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.78}, {"index": 10, "tokens": 15795183, "wall_clock_seconds": 92.6, "raw": 0.823092, "normalized": 0.823092, "discount": 1.0, "effective": 0.823092, "running_max": 0.823092, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 91.52} ]}, {"sample_key": "fYQMoLFuN7HSzESdvYi46v:mystery_box", "eval_id": "fYQMoLFuN7HSzESdvYi46v", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "fable-5", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "a3a336191a3b9e143911a7d11f0007f2d32978cb36596a9eaebdbb582744f09d", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.100716, "area_term": 0.08788, "final_best": 0.152058, "best_raw": 0.563081, "best_effective": 0.152058, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12858213, "usage": {"input": 5024, "output": 187042, "cache_read": 12417458, "cache_write": 248689}, "cost_usd": 26.7936, "cost_source": "list_price", "wall_clock_seconds": 3551.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 552387, "wall_clock_seconds": 15.0, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3191877, "wall_clock_seconds": 3.7, "raw": 0.421157, "normalized": 0.443205, "discount": 0.25, "effective": 0.110801, "running_max": 0.110801, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4803194, "wall_clock_seconds": 3.9, "raw": 0.520041, "normalized": 0.558187, "discount": 0.25, "effective": 0.139547, "running_max": 0.139547, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 5783366, "wall_clock_seconds": 3.4, "raw": 0.523773, "normalized": 0.562526, "discount": 0.25, "effective": 0.140632, "running_max": 0.140632, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 6664410, "wall_clock_seconds": 3.2, "raw": 0.517739, "normalized": 0.555511, "discount": 0.25, "effective": 0.138878, "running_max": 0.140632, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 7209764, "wall_clock_seconds": 3.3, "raw": 0.528999, "normalized": 0.568603, "discount": 0.25, "effective": 0.142151, "running_max": 0.142151, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 8364098, "wall_clock_seconds": 3.4, "raw": 0.538579, "normalized": 0.579743, "discount": 0.25, "effective": 0.144936, "running_max": 0.144936, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 10935431, "wall_clock_seconds": 3.3, "raw": 0.559314, "normalized": 0.603854, "discount": 0.25, "effective": 0.150964, "running_max": 0.150964, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 12127152, "wall_clock_seconds": 3.5, "raw": 0.563081, "normalized": 0.608234, "discount": 0.25, "effective": 0.152058, "running_max": 0.152058, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 12858213, "wall_clock_seconds": 3.3, "raw": 0.559246, "normalized": 0.603774, "discount": 0.25, "effective": 0.150944, "running_max": 0.152058, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "fYQMoLFuN7HSzESdvYi46v:mystery_box--nohints", "eval_id": "fYQMoLFuN7HSzESdvYi46v", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "fable-5", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "31c67b7", "source_log_sha256": "a3a336191a3b9e143911a7d11f0007f2d32978cb36596a9eaebdbb582744f09d", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.02005, "area_term": 0.018019, "final_best": 0.028171, "best_raw": 0.064227, "best_effective": 0.028171, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9584764, "usage": {"input": 4530, "output": 190168, "cache_read": 9130403, "cache_write": 259663}, "cost_usd": 23.8774, "cost_source": "list_price", "wall_clock_seconds": 3481.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2563033, "wall_clock_seconds": 3.9, "raw": 0.052697, "normalized": 0.014764, "discount": 1.0, "effective": 0.014764, "running_max": 0.014764, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2702413, "wall_clock_seconds": 2.5, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2988775, "wall_clock_seconds": 2.6, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3288517, "wall_clock_seconds": 3.5, "raw": 0.04343, "normalized": 0.003989, "discount": 1.0, "effective": 0.003989, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3594084, "wall_clock_seconds": 5.5, "raw": 0.061977, "normalized": 0.025554, "discount": 1.0, "effective": 0.025554, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4965051, "wall_clock_seconds": 2.6, "raw": 0.056098, "normalized": 0.018719, "discount": 1.0, "effective": 0.018719, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 7192632, "wall_clock_seconds": 2.8, "raw": 0.055276, "normalized": 0.017763, "discount": 1.0, "effective": 0.017763, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 8100975, "wall_clock_seconds": 4.6, "raw": 0.049317, "normalized": 0.010834, "discount": 1.0, "effective": 0.010834, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8575480, "wall_clock_seconds": 3.5, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9584764, "wall_clock_seconds": 2.7, "raw": 0.049145, "normalized": 0.010634, "discount": 1.0, "effective": 0.010634, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "fYQMoLFuN7HSzESdvYi46v:night_school", "eval_id": "fYQMoLFuN7HSzESdvYi46v", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "fable-5", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "31c67b7", "source_log_sha256": "a3a336191a3b9e143911a7d11f0007f2d32978cb36596a9eaebdbb582744f09d", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.256275, "area_term": 0.220549, "final_best": 0.399176, "best_raw": 0.672, "best_effective": 0.399176, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 14794622, "usage": {"input": 5680, "output": 229812, "cache_read": 14262441, "cache_write": 296689}, "cost_usd": 31.7436, "cost_source": "list_price", "wall_clock_seconds": 15772.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1194219, "wall_clock_seconds": 8.4, "raw": 0.222, "normalized": 0.084706, "discount": 1.0, "effective": 0.084706, "running_max": 0.084706, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4791588, "wall_clock_seconds": 12.2, "raw": 0.364, "normalized": 0.251765, "discount": 0.65, "effective": 0.163647, "running_max": 0.163647, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 5384298, "wall_clock_seconds": 10.5, "raw": 0.565, "normalized": 0.488235, "discount": 0.65, "effective": 0.317353, "running_max": 0.317353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 6004557, "wall_clock_seconds": 11.1, "raw": 0.6685, "normalized": 0.61, "discount": 0.65, "effective": 0.3965, "running_max": 0.3965, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 6654076, "wall_clock_seconds": 10.4, "raw": 0.6655, "normalized": 0.606471, "discount": 0.65, "effective": 0.394206, "running_max": 0.3965, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 9765289, "wall_clock_seconds": 9.9, "raw": 0.653, "normalized": 0.591765, "discount": 0.65, "effective": 0.384647, "running_max": 0.3965, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 11370523, "wall_clock_seconds": 10.6, "raw": 0.669, "normalized": 0.610588, "discount": 0.65, "effective": 0.396882, "running_max": 0.396882, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 11646387, "wall_clock_seconds": 10.7, "raw": 0.672, "normalized": 0.614118, "discount": 0.65, "effective": 0.399176, "running_max": 0.399176, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 14501387, "wall_clock_seconds": 10.8, "raw": 0.6715, "normalized": 0.613529, "discount": 0.65, "effective": 0.398794, "running_max": 0.399176, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 14794622, "wall_clock_seconds": 10.1, "raw": 0.6705, "normalized": 0.612353, "discount": 0.65, "effective": 0.398029, "running_max": 0.399176, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "fYQMoLFuN7HSzESdvYi46v:reaction_rates--nohints", "eval_id": "fYQMoLFuN7HSzESdvYi46v", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "fable-5", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "31c67b7", "source_log_sha256": "a3a336191a3b9e143911a7d11f0007f2d32978cb36596a9eaebdbb582744f09d", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.016918, "area_term": 0.010709, "final_best": 0.041757, "best_raw": 0.041757, "best_effective": 0.041757, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 38972842, "usage": {"input": 25964, "output": 328255, "cache_read": 38158083, "cache_write": 460540}, "cost_usd": 64.0413, "cost_source": "list_price", "wall_clock_seconds": 27235.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5747504, "wall_clock_seconds": 28.6, "raw": 0.01887, "normalized": 0.01887, "discount": 1.0, "effective": 0.01887, "running_max": 0.01887, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.05}, {"index": 2, "tokens": 12037041, "wall_clock_seconds": 29.3, "raw": 0.000903, "normalized": 0.000903, "discount": 1.0, "effective": 0.000903, "running_max": 0.01887, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.73}, {"index": 3, "tokens": 19030699, "wall_clock_seconds": 51.2, "raw": 0.022189, "normalized": 0.022189, "discount": 1.0, "effective": 0.022189, "running_max": 0.022189, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 50.65}, {"index": 4, "tokens": 27031091, "wall_clock_seconds": 23.6, "raw": 0.003121, "normalized": 0.003121, "discount": 1.0, "effective": 0.003121, "running_max": 0.022189, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.58}, {"index": 5, "tokens": 28920724, "wall_clock_seconds": 23.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.022189, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.35}, {"index": 6, "tokens": 32074065, "wall_clock_seconds": 35.6, "raw": 0.022189, "normalized": 0.022189, "discount": 1.0, "effective": 0.022189, "running_max": 0.022189, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 35.07}, {"index": 7, "tokens": 33307617, "wall_clock_seconds": 9.8, "raw": 0.022189, "normalized": 0.022189, "discount": 1.0, "effective": 0.022189, "running_max": 0.022189, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.29}, {"index": 8, "tokens": 35851595, "wall_clock_seconds": 39.1, "raw": 0.027133, "normalized": 0.027133, "discount": 1.0, "effective": 0.027133, "running_max": 0.027133, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 38.36}, {"index": 9, "tokens": 37165798, "wall_clock_seconds": 14.1, "raw": 0.012581, "normalized": 0.012581, "discount": 1.0, "effective": 0.012581, "running_max": 0.027133, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.61}, {"index": 10, "tokens": 38972842, "wall_clock_seconds": 39.5, "raw": 0.041757, "normalized": 0.041757, "discount": 1.0, "effective": 0.041757, "running_max": 0.041757, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 38.66} ]}, {"sample_key": "fYQMoLFuN7HSzESdvYi46v:scan_stitch", "eval_id": "fYQMoLFuN7HSzESdvYi46v", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "fable-5", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "a3a336191a3b9e143911a7d11f0007f2d32978cb36596a9eaebdbb582744f09d", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.12781, "area_term": 0.107679, "final_best": 0.208335, "best_raw": 0.576194, "best_effective": 0.208335, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9694681, "usage": {"input": 4819, "output": 152685, "cache_read": 9323239, "cache_write": 213938}, "cost_usd": 21.2844, "cost_source": "list_price", "wall_clock_seconds": 5846.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 675046, "wall_clock_seconds": 0.3, "raw": 0.0984, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1951863, "wall_clock_seconds": 0.3, "raw": 0.095083, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4383430, "wall_clock_seconds": 0.3, "raw": 0.467294, "normalized": 0.459118, "discount": 0.35, "effective": 0.160691, "running_max": 0.160691, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 4968104, "wall_clock_seconds": 0.3, "raw": 0.536004, "normalized": 0.545005, "discount": 0.35, "effective": 0.190752, "running_max": 0.190752, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 5744685, "wall_clock_seconds": 0.3, "raw": 0.548336, "normalized": 0.56042, "discount": 0.35, "effective": 0.196147, "running_max": 0.196147, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 6403825, "wall_clock_seconds": 0.3, "raw": 0.568442, "normalized": 0.585553, "discount": 0.35, "effective": 0.204943, "running_max": 0.204943, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 8060969, "wall_clock_seconds": 0.3, "raw": 0.537361, "normalized": 0.546701, "discount": 0.35, "effective": 0.191345, "running_max": 0.204943, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 8858540, "wall_clock_seconds": 0.3, "raw": 0.547245, "normalized": 0.559057, "discount": 0.35, "effective": 0.19567, "running_max": 0.204943, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 9272659, "wall_clock_seconds": 0.3, "raw": 0.567346, "normalized": 0.584183, "discount": 0.35, "effective": 0.204464, "running_max": 0.204943, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9694681, "wall_clock_seconds": 0.3, "raw": 0.576194, "normalized": 0.595243, "discount": 0.35, "effective": 0.208335, "running_max": 0.208335, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "fYQMoLFuN7HSzESdvYi46v:scan_stitch--nohints", "eval_id": "fYQMoLFuN7HSzESdvYi46v", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "fable-5", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "31c67b7", "source_log_sha256": "a3a336191a3b9e143911a7d11f0007f2d32978cb36596a9eaebdbb582744f09d", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.097462, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11727270, "usage": {"input": 5050, "output": 182179, "cache_read": 11284574, "cache_write": 255467}, "cost_usd": 25.5534, "cost_source": "list_price", "wall_clock_seconds": 4222.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3490260, "wall_clock_seconds": 0.3, "raw": 0.097462, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3756545, "wall_clock_seconds": 0.3, "raw": 0.087653, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4165953, "wall_clock_seconds": 0.6, "raw": 0.07675, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 6423103, "wall_clock_seconds": 0.3, "raw": 0.096282, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 6986824, "wall_clock_seconds": 0.3, "raw": 0.06819, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 7783405, "wall_clock_seconds": 0.6, "raw": 0.064685, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 8198253, "wall_clock_seconds": 0.3, "raw": 0.08318, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 9771654, "wall_clock_seconds": 0.2, "raw": 0.069908, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 10967026, "wall_clock_seconds": 0.4, "raw": 0.076844, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 11727270, "wall_clock_seconds": 0.6, "raw": 0.054041, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "fYQMoLFuN7HSzESdvYi46v:shapes_generalize", "eval_id": "fYQMoLFuN7HSzESdvYi46v", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "fable-5", "run_date": "2026-09-13", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "31c67b7", "source_log_sha256": "a3a336191a3b9e143911a7d11f0007f2d32978cb36596a9eaebdbb582744f09d", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.160087, "area_term": 0.124484, "final_best": 0.3025, "best_raw": 0.442, "best_effective": 0.3025, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10230764, "usage": {"input": 10192, "output": 196475, "cache_read": 9752843, "cache_write": 271254}, "cost_usd": 25.1036, "cost_source": "list_price", "wall_clock_seconds": 8705.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2711661, "wall_clock_seconds": 6.3, "raw": 0.241, "normalized": 0.05125, "discount": 1.0, "effective": 0.05125, "running_max": 0.05125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.7}, {"index": 2, "tokens": 3841672, "wall_clock_seconds": 7.1, "raw": 0.108, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.05125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.55}, {"index": 3, "tokens": 4589340, "wall_clock_seconds": 9.8, "raw": 0.167, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.05125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.17}, {"index": 4, "tokens": 6311410, "wall_clock_seconds": 7.3, "raw": 0.209, "normalized": 0.01125, "discount": 1.0, "effective": 0.01125, "running_max": 0.05125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.64}, {"index": 5, "tokens": 6890135, "wall_clock_seconds": 0.8, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.05125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 6, "tokens": 7506218, "wall_clock_seconds": 7.4, "raw": 0.157, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.05125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.89}, {"index": 7, "tokens": 7941812, "wall_clock_seconds": 9.9, "raw": 0.198, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.05125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.95}, {"index": 8, "tokens": 8392304, "wall_clock_seconds": 6.6, "raw": 0.264, "normalized": 0.08, "discount": 1.0, "effective": 0.08, "running_max": 0.08, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.05}, {"index": 9, "tokens": 9155667, "wall_clock_seconds": 8.9, "raw": 0.405, "normalized": 0.25625, "discount": 1.0, "effective": 0.25625, "running_max": 0.25625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.24}, {"index": 10, "tokens": 10230764, "wall_clock_seconds": 7.4, "raw": 0.442, "normalized": 0.3025, "discount": 1.0, "effective": 0.3025, "running_max": 0.3025, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.51} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:mystery_box", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.064205, "area_term": 0.047026, "final_best": 0.132922, "best_raw": 0.497253, "best_effective": 0.132922, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 27260901, "usage": {"input": 25656, "output": 291637, "cache_read": 26539364, "cache_write": 404244}, "cost_usd": 29.5581, "cost_source": "list_price", "wall_clock_seconds": 8357.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 6623786, "wall_clock_seconds": 6.2, "raw": 0.274289, "normalized": 0.272429, "discount": 0.25, "effective": 0.068107, "running_max": 0.068107, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 10095644, "wall_clock_seconds": 5.1, "raw": 0.238191, "normalized": 0.230455, "discount": 0.25, "effective": 0.057614, "running_max": 0.068107, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 11406658, "wall_clock_seconds": 5.6, "raw": 0.386151, "normalized": 0.402502, "discount": 0.25, "effective": 0.100625, "running_max": 0.100625, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 14789606, "wall_clock_seconds": 6.1, "raw": 0.39428, "normalized": 0.411953, "discount": 0.25, "effective": 0.102988, "running_max": 0.102988, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 18535030, "wall_clock_seconds": 3.6, "raw": 0.430318, "normalized": 0.453858, "discount": 0.25, "effective": 0.113464, "running_max": 0.113464, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 19533033, "wall_clock_seconds": 3.7, "raw": 0.466311, "normalized": 0.495711, "discount": 0.25, "effective": 0.123928, "running_max": 0.123928, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 20912796, "wall_clock_seconds": 3.6, "raw": 0.495066, "normalized": 0.529147, "discount": 0.25, "effective": 0.132287, "running_max": 0.132287, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 24928927, "wall_clock_seconds": 3.4, "raw": 0.497253, "normalized": 0.53169, "discount": 0.25, "effective": 0.132922, "running_max": 0.132922, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 25310835, "wall_clock_seconds": 3.5, "raw": 0.497253, "normalized": 0.53169, "discount": 0.25, "effective": 0.132922, "running_max": 0.132922, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 27260901, "wall_clock_seconds": 4.2, "raw": 0.496678, "normalized": 0.531021, "discount": 0.25, "effective": 0.132755, "running_max": 0.132922, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:mystery_box--nohints", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.022213, "area_term": 0.021341, "final_best": 0.025703, "best_raw": 0.062104, "best_effective": 0.025703, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9595096, "usage": {"input": 4169, "output": 291103, "cache_read": 8951361, "cache_write": 348463}, "cost_usd": 23.8039, "cost_source": "list_price", "wall_clock_seconds": 5095.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 663494, "wall_clock_seconds": 4.8, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1296436, "wall_clock_seconds": 4.0, "raw": 0.062104, "normalized": 0.025703, "discount": 1.0, "effective": 0.025703, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1912609, "wall_clock_seconds": 7.2, "raw": 0.044181, "normalized": 0.004862, "discount": 1.0, "effective": 0.004862, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2721118, "wall_clock_seconds": 5.4, "raw": 0.056355, "normalized": 0.019018, "discount": 1.0, "effective": 0.019018, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3098373, "wall_clock_seconds": 7.2, "raw": 0.054221, "normalized": 0.016536, "discount": 1.0, "effective": 0.016536, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4559786, "wall_clock_seconds": 5.1, "raw": 0.050838, "normalized": 0.012603, "discount": 1.0, "effective": 0.012603, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5543607, "wall_clock_seconds": 5.0, "raw": 0.049329, "normalized": 0.010848, "discount": 1.0, "effective": 0.010848, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 7906336, "wall_clock_seconds": 9.0, "raw": 0.061097, "normalized": 0.024531, "discount": 1.0, "effective": 0.024531, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8559132, "wall_clock_seconds": 5.3, "raw": 0.059241, "normalized": 0.022374, "discount": 1.0, "effective": 0.022374, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9595096, "wall_clock_seconds": 7.2, "raw": 0.041322, "normalized": 0.001537, "discount": 1.0, "effective": 0.001537, "running_max": 0.025703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:night_school", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.45066, "area_term": 0.365348, "final_best": 0.791912, "best_raw": 0.823125, "best_effective": 0.791912, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 30101252, "usage": {"input": 6058, "output": 247315, "cache_read": 29510027, "cache_write": 337852}, "cost_usd": 26.5609, "cost_source": "list_price", "wall_clock_seconds": 41543.6, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5247450, "wall_clock_seconds": 14.4, "raw": 0.667031, "normalized": 0.608272, "discount": 1.0, "effective": 0.608272, "running_max": 0.608272, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6726806, "wall_clock_seconds": 13.7, "raw": 0.768563, "normalized": 0.727721, "discount": 1.0, "effective": 0.727721, "running_max": 0.727721, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 9408538, "wall_clock_seconds": 15.9, "raw": 0.790063, "normalized": 0.753015, "discount": 1.0, "effective": 0.753015, "running_max": 0.753015, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 13775375, "wall_clock_seconds": 12.2, "raw": 0.7905, "normalized": 0.753529, "discount": 1.0, "effective": 0.753529, "running_max": 0.753529, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 22694676, "wall_clock_seconds": 14.3, "raw": 0.815531, "normalized": 0.782978, "discount": 1.0, "effective": 0.782978, "running_max": 0.782978, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 25134241, "wall_clock_seconds": 15.2, "raw": 0.811906, "normalized": 0.778713, "discount": 1.0, "effective": 0.778713, "running_max": 0.782978, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 26538502, "wall_clock_seconds": 12.2, "raw": 0.821875, "normalized": 0.790441, "discount": 1.0, "effective": 0.790441, "running_max": 0.790441, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 29741907, "wall_clock_seconds": 13.5, "raw": 0.823125, "normalized": 0.791912, "discount": 1.0, "effective": 0.791912, "running_max": 0.791912, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:reaction_rates", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.23708, "area_term": 0.146068, "final_best": 0.601126, "best_raw": 0.751408, "best_effective": 0.601126, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 31149881, "usage": {"input": 15049, "output": 296627, "cache_read": 30443420, "cache_write": 394785}, "cost_usd": 30.4884, "cost_source": "list_price", "wall_clock_seconds": 32436.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.8, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 6400654, "wall_clock_seconds": 17.4, "raw": 0.200064, "normalized": 0.200064, "discount": 0.8, "effective": 0.160051, "running_max": 0.160051, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 16.19}, {"index": 2, "tokens": 16189256, "wall_clock_seconds": 114.0, "raw": 0.260754, "normalized": 0.260754, "discount": 0.8, "effective": 0.208603, "running_max": 0.208603, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 113.45}, {"index": 3, "tokens": 20214158, "wall_clock_seconds": 256.8, "raw": 0.505019, "normalized": 0.505019, "discount": 0.8, "effective": 0.404016, "running_max": 0.404016, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 256.17}, {"index": 4, "tokens": 21974326, "wall_clock_seconds": 124.9, "raw": 0.532298, "normalized": 0.532298, "discount": 0.8, "effective": 0.425838, "running_max": 0.425838, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 124.34}, {"index": 5, "tokens": 24910699, "wall_clock_seconds": 127.2, "raw": 0.534291, "normalized": 0.534291, "discount": 0.8, "effective": 0.427433, "running_max": 0.427433, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 126.56}, {"index": 6, "tokens": 26428489, "wall_clock_seconds": 130.5, "raw": 0.513867, "normalized": 0.513867, "discount": 0.8, "effective": 0.411094, "running_max": 0.427433, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 129.89}, {"index": 7, "tokens": 26807014, "wall_clock_seconds": 144.3, "raw": 0.407416, "normalized": 0.407416, "discount": 0.8, "effective": 0.325933, "running_max": 0.427433, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 143.81}, {"index": 8, "tokens": 27962451, "wall_clock_seconds": 115.5, "raw": 0.656281, "normalized": 0.656281, "discount": 0.8, "effective": 0.525025, "running_max": 0.525025, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 114.94}, {"index": 9, "tokens": 29933859, "wall_clock_seconds": 121.9, "raw": 0.66127, "normalized": 0.66127, "discount": 0.8, "effective": 0.529016, "running_max": 0.529016, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 121.36}, {"index": 10, "tokens": 31149881, "wall_clock_seconds": 118.3, "raw": 0.751408, "normalized": 0.751408, "discount": 0.8, "effective": 0.601126, "running_max": 0.601126, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 117.71} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:reaction_rates--nohints", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.182272, "area_term": 0.143245, "final_best": 0.33838, "best_raw": 0.33838, "best_effective": 0.33838, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 28486491, "usage": {"input": 16950, "output": 266576, "cache_read": 27795108, "cache_write": 407857}, "cost_usd": 28.6042, "cost_source": "list_price", "wall_clock_seconds": 25114.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5598455, "wall_clock_seconds": 7.4, "raw": 0.2541, "normalized": 0.2541, "discount": 1.0, "effective": 0.2541, "running_max": 0.2541, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.68}, {"index": 2, "tokens": 10634673, "wall_clock_seconds": 31.8, "raw": 0.297919, "normalized": 0.297919, "discount": 1.0, "effective": 0.297919, "running_max": 0.297919, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 31.25}, {"index": 3, "tokens": 14829399, "wall_clock_seconds": 162.3, "raw": 0.130024, "normalized": 0.130024, "discount": 1.0, "effective": 0.130024, "running_max": 0.297919, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 161.81}, {"index": 4, "tokens": 16141845, "wall_clock_seconds": 32.9, "raw": 0.320265, "normalized": 0.320265, "discount": 1.0, "effective": 0.320265, "running_max": 0.320265, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 32.44}, {"index": 5, "tokens": 17167392, "wall_clock_seconds": 32.1, "raw": 0.310243, "normalized": 0.310243, "discount": 1.0, "effective": 0.310243, "running_max": 0.320265, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 31.54}, {"index": 6, "tokens": 20074861, "wall_clock_seconds": 21.5, "raw": 0.267061, "normalized": 0.267061, "discount": 1.0, "effective": 0.267061, "running_max": 0.320265, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 20.95}, {"index": 7, "tokens": 23539488, "wall_clock_seconds": 37.9, "raw": 0.301669, "normalized": 0.301669, "discount": 1.0, "effective": 0.301669, "running_max": 0.320265, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 37.27}, {"index": 8, "tokens": 25143252, "wall_clock_seconds": 31.0, "raw": 0.320271, "normalized": 0.320271, "discount": 1.0, "effective": 0.320271, "running_max": 0.320271, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.42}, {"index": 9, "tokens": 27212072, "wall_clock_seconds": 38.3, "raw": 0.320798, "normalized": 0.320798, "discount": 1.0, "effective": 0.320798, "running_max": 0.320798, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 37.49}, {"index": 10, "tokens": 28486491, "wall_clock_seconds": 52.1, "raw": 0.33838, "normalized": 0.33838, "discount": 1.0, "effective": 0.33838, "running_max": 0.33838, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 51.56} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:scan_stitch", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.074002, "area_term": 0.05777, "final_best": 0.138929, "best_raw": 0.417553, "best_effective": 0.138929, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 18640842, "usage": {"input": 15440, "output": 231950, "cache_read": 18056468, "cache_write": 336984}, "cost_usd": 23.0057, "cost_source": "list_price", "wall_clock_seconds": 6473.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7006112, "wall_clock_seconds": 0.3, "raw": 0.393285, "normalized": 0.366606, "discount": 0.35, "effective": 0.128312, "running_max": 0.128312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7864446, "wall_clock_seconds": 0.4, "raw": 0.376747, "normalized": 0.345934, "discount": 0.35, "effective": 0.121077, "running_max": 0.128312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8305391, "wall_clock_seconds": 0.3, "raw": 0.092742, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.128312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 10410012, "wall_clock_seconds": 0.3, "raw": 0.355985, "normalized": 0.319981, "discount": 0.35, "effective": 0.111993, "running_max": 0.128312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 11688842, "wall_clock_seconds": 0.3, "raw": 0.392392, "normalized": 0.36549, "discount": 0.35, "effective": 0.127922, "running_max": 0.128312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 13347835, "wall_clock_seconds": 0.3, "raw": 0.416548, "normalized": 0.395685, "discount": 0.35, "effective": 0.13849, "running_max": 0.13849, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 15720075, "wall_clock_seconds": 0.3, "raw": 0.381397, "normalized": 0.351747, "discount": 0.35, "effective": 0.123111, "running_max": 0.13849, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 16668436, "wall_clock_seconds": 0.3, "raw": 0.409169, "normalized": 0.386461, "discount": 0.35, "effective": 0.135261, "running_max": 0.13849, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 17972497, "wall_clock_seconds": 0.3, "raw": 0.397315, "normalized": 0.371644, "discount": 0.35, "effective": 0.130075, "running_max": 0.13849, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 18640842, "wall_clock_seconds": 0.3, "raw": 0.417553, "normalized": 0.396941, "discount": 0.35, "effective": 0.138929, "running_max": 0.138929, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:scan_stitch--nohints", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.000413, "area_term": 0.00034, "final_best": 0.000703, "best_raw": 0.100563, "best_effective": 0.000703, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10891103, "usage": {"input": 4377, "output": 295489, "cache_read": 10231470, "cache_write": 359767}, "cost_usd": 24.5714, "cost_source": "list_price", "wall_clock_seconds": 4725.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 755620, "wall_clock_seconds": 0.3, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1006433, "wall_clock_seconds": 0.2, "raw": 0.097422, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2992858, "wall_clock_seconds": 0.2, "raw": 0.086907, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3318800, "wall_clock_seconds": 0.3, "raw": 0.069708, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3853274, "wall_clock_seconds": 0.4, "raw": 0.055325, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4890889, "wall_clock_seconds": 0.3, "raw": 0.068703, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5385486, "wall_clock_seconds": 0.4, "raw": 0.100563, "normalized": 0.000703, "discount": 1.0, "effective": 0.000703, "running_max": 0.000703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6455491, "wall_clock_seconds": 0.3, "raw": 0.100563, "normalized": 0.000703, "discount": 1.0, "effective": 0.000703, "running_max": 0.000703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 9816123, "wall_clock_seconds": 0.2, "raw": 0.06123, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 10891103, "wall_clock_seconds": 0.3, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:shapes_generalize", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.605635, "area_term": 0.557356, "final_best": 0.79875, "best_raw": 0.839, "best_effective": 0.79875, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 9981827, "usage": {"input": 20709, "output": 122927, "cache_read": 9656694, "cache_write": 181497}, "cost_usd": 12.3976, "cost_source": "list_price", "wall_clock_seconds": 18419.9, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1732276, "wall_clock_seconds": 6.3, "raw": 0.355, "normalized": 0.19375, "discount": 1.0, "effective": 0.19375, "running_max": 0.19375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.78}, {"index": 2, "tokens": 2017169, "wall_clock_seconds": 8.4, "raw": 0.81, "normalized": 0.7625, "discount": 1.0, "effective": 0.7625, "running_max": 0.7625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.59}, {"index": 3, "tokens": 2320214, "wall_clock_seconds": 6.5, "raw": 0.816, "normalized": 0.77, "discount": 1.0, "effective": 0.77, "running_max": 0.77, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.01}, {"index": 4, "tokens": 2881323, "wall_clock_seconds": 7.8, "raw": 0.82, "normalized": 0.775, "discount": 1.0, "effective": 0.775, "running_max": 0.775, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.17}, {"index": 5, "tokens": 3123907, "wall_clock_seconds": 10.5, "raw": 0.826, "normalized": 0.7825, "discount": 1.0, "effective": 0.7825, "running_max": 0.7825, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.69}, {"index": 6, "tokens": 4738797, "wall_clock_seconds": 9.0, "raw": 0.831, "normalized": 0.78875, "discount": 1.0, "effective": 0.78875, "running_max": 0.78875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.15}, {"index": 7, "tokens": 8037761, "wall_clock_seconds": 7.4, "raw": 0.822, "normalized": 0.7775, "discount": 1.0, "effective": 0.7775, "running_max": 0.78875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.9}, {"index": 8, "tokens": 8220633, "wall_clock_seconds": 7.9, "raw": 0.809, "normalized": 0.76125, "discount": 1.0, "effective": 0.76125, "running_max": 0.78875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.34}, {"index": 9, "tokens": 9578878, "wall_clock_seconds": 7.9, "raw": 0.839, "normalized": 0.79875, "discount": 1.0, "effective": 0.79875, "running_max": 0.79875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.24} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:shattered_prior", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.243901, "area_term": 0.112664, "final_best": 0.768848, "best_raw": 0.768848, "best_effective": 0.768848, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 45769817, "usage": {"input": 9108, "output": 368308, "cache_read": 44850589, "cache_write": 541812}, "cost_usd": 40.5554, "cost_source": "list_price", "wall_clock_seconds": 15748.8, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 23794735, "wall_clock_seconds": 2.1, "raw": 0.651055, "normalized": 0.651055, "discount": 1.0, "effective": 0.651055, "running_max": 0.651055, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 36647639, "wall_clock_seconds": 1.8, "raw": 0.753776, "normalized": 0.753776, "discount": 1.0, "effective": 0.753776, "running_max": 0.753776, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 37682083, "wall_clock_seconds": 2.4, "raw": 0.664182, "normalized": 0.664182, "discount": 1.0, "effective": 0.664182, "running_max": 0.753776, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 39250254, "wall_clock_seconds": 2.1, "raw": 0.762395, "normalized": 0.762395, "discount": 1.0, "effective": 0.762395, "running_max": 0.762395, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 40306581, "wall_clock_seconds": 1.8, "raw": 0.768848, "normalized": 0.768848, "discount": 1.0, "effective": 0.768848, "running_max": 0.768848, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:shattered_prior--nohints", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.084626, "area_term": 0.018601, "final_best": 0.348724, "best_raw": 0.348724, "best_effective": 0.348724, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 48267352, "usage": {"input": 6383, "output": 369291, "cache_read": 47329221, "cache_write": 562457}, "cost_usd": 41.6098, "cost_source": "list_price", "wall_clock_seconds": 6452.0, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 26055196, "wall_clock_seconds": 2.0, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 38782561, "wall_clock_seconds": 2.3, "raw": 0.180726, "normalized": 0.180726, "discount": 1.0, "effective": 0.180726, "running_max": 0.180726, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 40962763, "wall_clock_seconds": 2.0, "raw": 0.157521, "normalized": 0.157521, "discount": 1.0, "effective": 0.157521, "running_max": 0.180726, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 41511421, "wall_clock_seconds": 1.8, "raw": 0.1477, "normalized": 0.1477, "discount": 1.0, "effective": 0.1477, "running_max": 0.180726, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 44291659, "wall_clock_seconds": 2.6, "raw": 0.348724, "normalized": 0.348724, "discount": 1.0, "effective": 0.348724, "running_max": 0.348724, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 47695158, "wall_clock_seconds": 1.8, "raw": 0.326128, "normalized": 0.326128, "discount": 1.0, "effective": 0.326128, "running_max": 0.348724, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:ship_detect", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.096764, "area_term": 0.088288, "final_best": 0.130666, "best_raw": 0.130666, "best_effective": 0.130666, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 18, "max_submissions": null, "tokens_total": 39647077, "usage": {"input": 26130, "output": 200143, "cache_read": 39040134, "cache_write": 380670}, "cost_usd": 27.6419, "cost_source": "list_price", "wall_clock_seconds": 42141.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 14, "tokens": 545874, "cost_weighted": 32802.7, "wall_clock_seconds": 1018.2, "raw": 0.049969, "normalized": 0.049969, "effective": 0.049969, "running_max": 0.049969, "scored": true, "final": false}, {"index": 2, "model_call": 34, "tokens": 2184719, "cost_weighted": 94607.9, "wall_clock_seconds": 4034.7, "raw": 0.079892, "normalized": 0.079892, "effective": 0.079892, "running_max": 0.079892, "scored": true, "final": false}, {"index": 3, "model_call": 51, "tokens": 4344177, "cost_weighted": 136724.5, "wall_clock_seconds": 5922.5, "raw": 0.088796, "normalized": 0.088796, "effective": 0.088796, "running_max": 0.088796, "scored": true, "final": false}, {"index": 4, "model_call": 57, "tokens": 5230315, "cost_weighted": 149683.4, "wall_clock_seconds": 6472.5, "raw": 0.09272, "normalized": 0.09272, "effective": 0.09272, "running_max": 0.09272, "scored": true, "final": false}, {"index": 5, "model_call": 67, "tokens": 6855223, "cost_weighted": 176905.6, "wall_clock_seconds": 8791.5, "raw": 0.099915, "normalized": 0.099915, "effective": 0.099915, "running_max": 0.099915, "scored": true, "final": false}, {"index": 6, "model_call": 79, "tokens": 9126587, "cost_weighted": 215118.7, "wall_clock_seconds": 10751.7, "raw": 0.110042, "normalized": 0.110042, "effective": 0.110042, "running_max": 0.110042, "scored": true, "final": false}, {"index": 7, "model_call": 88, "tokens": 11084400, "cost_weighted": 250787.8, "wall_clock_seconds": 12830.4, "raw": 0.109934, "normalized": 0.109934, "effective": 0.109934, "running_max": 0.110042, "scored": true, "final": false}, {"index": 8, "model_call": 94, "tokens": 12544251, "cost_weighted": 276612.3, "wall_clock_seconds": 15051.7, "raw": 0.120538, "normalized": 0.120538, "effective": 0.120538, "running_max": 0.120538, "scored": true, "final": false}, {"index": 9, "model_call": 109, "tokens": 16627691, "cost_weighted": 334148.2, "wall_clock_seconds": 18899.0, "raw": 0.122002, "normalized": 0.122002, "effective": 0.122002, "running_max": 0.122002, "scored": true, "final": false}, {"index": 10, "model_call": 135, "tokens": 24721987, "cost_weighted": 413484.3, "wall_clock_seconds": 23705.0, "raw": 0.12391, "normalized": 0.12391, "effective": 0.12391, "running_max": 0.12391, "scored": true, "final": false}, {"index": 11, "model_call": 145, "tokens": 28109673, "cost_weighted": 447072.9, "wall_clock_seconds": 27301.7, "raw": 0.123977, "normalized": 0.123977, "effective": 0.123977, "running_max": 0.123977, "scored": true, "final": false}, {"index": 12, "model_call": 149, "tokens": 29511766, "cost_weighted": 461072.1, "wall_clock_seconds": 29355.2, "raw": 0.125155, "normalized": 0.125155, "effective": 0.125155, "running_max": 0.125155, "scored": true, "final": false}, {"index": 13, "model_call": 156, "tokens": 32037554, "cost_weighted": 484332.1, "wall_clock_seconds": 31934.6, "raw": 0.127079, "normalized": 0.127079, "effective": 0.127079, "running_max": 0.127079, "scored": true, "final": false}, {"index": 14, "model_call": 163, "tokens": 34638511, "cost_weighted": 506822.9, "wall_clock_seconds": 34478.1, "raw": 0.129259, "normalized": 0.129259, "effective": 0.129259, "running_max": 0.129259, "scored": true, "final": false}, {"index": 15, "model_call": 167, "tokens": 36152673, "cost_weighted": 519162.4, "wall_clock_seconds": 37040.4, "raw": 0.129354, "normalized": 0.129354, "effective": 0.129354, "running_max": 0.129354, "scored": true, "final": false}, {"index": 16, "model_call": 171, "tokens": 37688176, "cost_weighted": 531995.5, "wall_clock_seconds": 39341.8, "raw": 0.130238, "normalized": 0.130238, "effective": 0.130238, "running_max": 0.130238, "scored": true, "final": false}, {"index": 17, "model_call": 173, "tokens": 38460573, "cost_weighted": 537440.8, "wall_clock_seconds": 41097.4, "raw": 0.130442, "normalized": 0.130442, "effective": 0.130442, "running_max": 0.130442, "scored": true, "final": false}, {"index": 18, "model_call": 175, "tokens": 39247261, "cost_weighted": 545892.9, "wall_clock_seconds": 42098.2, "raw": 0.130666, "normalized": 0.130666, "effective": 0.130666, "running_max": 0.130666, "scored": true, "final": false} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:ship_tune", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.093076, "area_term": 0.055828, "final_best": 0.242065, "best_raw": 0.242065, "best_effective": 0.242065, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 34321870, "usage": {"input": 18169, "output": 200929, "cache_read": 32243976, "cache_write": 1858796}, "cost_usd": 55.4651, "cost_source": "list_price", "wall_clock_seconds": 31590.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 13637238, "wall_clock_seconds": 133.7, "raw": 0.078471, "normalized": 0.078471, "discount": 1.0, "effective": 0.078471, "running_max": 0.078471, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 132.98}, {"index": 2, "tokens": 18453087, "wall_clock_seconds": 424.2, "raw": 0.209453, "normalized": 0.209453, "discount": 1.0, "effective": 0.209453, "running_max": 0.209453, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 423.49}, {"index": 3, "tokens": 20783848, "wall_clock_seconds": 407.4, "raw": 0.23002, "normalized": 0.23002, "discount": 1.0, "effective": 0.23002, "running_max": 0.23002, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 406.71}, {"index": 4, "tokens": 24505807, "wall_clock_seconds": 403.6, "raw": 0.22344, "normalized": 0.22344, "discount": 1.0, "effective": 0.22344, "running_max": 0.23002, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 402.89}, {"index": 5, "tokens": 25929500, "wall_clock_seconds": 367.1, "raw": 0.235755, "normalized": 0.235755, "discount": 1.0, "effective": 0.235755, "running_max": 0.235755, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 366.27}, {"index": 6, "tokens": 26411588, "wall_clock_seconds": 263.4, "raw": 0.222488, "normalized": 0.222488, "discount": 1.0, "effective": 0.222488, "running_max": 0.235755, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 262.64}, {"index": 7, "tokens": 30323451, "wall_clock_seconds": 364.2, "raw": 0.22156, "normalized": 0.22156, "discount": 1.0, "effective": 0.22156, "running_max": 0.235755, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 363.38}, {"index": 8, "tokens": 30835086, "wall_clock_seconds": 375.5, "raw": 0.222204, "normalized": 0.222204, "discount": 1.0, "effective": 0.222204, "running_max": 0.235755, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 374.52}, {"index": 9, "tokens": 32144607, "wall_clock_seconds": 379.4, "raw": 0.241741, "normalized": 0.241741, "discount": 1.0, "effective": 0.241741, "running_max": 0.241741, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 378.65}, {"index": 10, "tokens": 34321870, "wall_clock_seconds": 413.8, "raw": 0.242065, "normalized": 0.242065, "discount": 1.0, "effective": 0.242065, "running_max": 0.242065, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 412.86} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:splash_generalize", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.50562, "area_term": 0.46508, "final_best": 0.667778, "best_raw": 0.701, "best_effective": 0.667778, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8425455, "usage": {"input": 4819, "output": 132443, "cache_read": 8087260, "cache_write": 200933}, "cost_usd": 12.7108, "cost_source": "list_price", "wall_clock_seconds": 4616.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1933817, "wall_clock_seconds": 75.8, "raw": 0.643, "normalized": 0.603333, "discount": 1.0, "effective": 0.603333, "running_max": 0.603333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.08}, {"index": 2, "tokens": 2263174, "wall_clock_seconds": 43.8, "raw": 0.64, "normalized": 0.6, "discount": 1.0, "effective": 0.6, "running_max": 0.603333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 43.1}, {"index": 3, "tokens": 2537211, "wall_clock_seconds": 45.0, "raw": 0.639, "normalized": 0.598889, "discount": 1.0, "effective": 0.598889, "running_max": 0.603333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.26}, {"index": 4, "tokens": 3089458, "wall_clock_seconds": 70.1, "raw": 0.701, "normalized": 0.667778, "discount": 1.0, "effective": 0.667778, "running_max": 0.667778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 69.48}, {"index": 5, "tokens": 3576925, "wall_clock_seconds": 67.3, "raw": 0.691, "normalized": 0.656667, "discount": 1.0, "effective": 0.656667, "running_max": 0.667778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 66.79}, {"index": 6, "tokens": 4389592, "wall_clock_seconds": 71.3, "raw": 0.69, "normalized": 0.655556, "discount": 1.0, "effective": 0.655556, "running_max": 0.667778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 70.28}, {"index": 7, "tokens": 5141905, "wall_clock_seconds": 76.9, "raw": 0.7, "normalized": 0.666667, "discount": 1.0, "effective": 0.666667, "running_max": 0.667778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 76.3}, {"index": 8, "tokens": 5964568, "wall_clock_seconds": 75.8, "raw": 0.698, "normalized": 0.664444, "discount": 1.0, "effective": 0.664444, "running_max": 0.667778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.26}, {"index": 9, "tokens": 6671174, "wall_clock_seconds": 74.8, "raw": 0.69, "normalized": 0.655556, "discount": 1.0, "effective": 0.655556, "running_max": 0.667778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 74.28}, {"index": 10, "tokens": 8425455, "wall_clock_seconds": 86.2, "raw": 0.693, "normalized": 0.658889, "discount": 1.0, "effective": 0.658889, "running_max": 0.667778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 85.57} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:tod_pipeline--nohints", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.074875, "area_term": 0.05619, "final_best": 0.149614, "best_raw": 0.149614, "best_effective": 0.149614, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 40893369, "usage": {"input": 32918, "output": 319957, "cache_read": 40052007, "cache_write": 488487}, "cost_usd": 36.1098, "cost_source": "list_price", "wall_clock_seconds": 10290.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7651812, "wall_clock_seconds": 93.1, "raw": 0.127793, "normalized": 0.127793, "discount": 1.0, "effective": 0.127793, "running_max": 0.127793, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 92.59}, {"index": 2, "tokens": 11191584, "wall_clock_seconds": 85.8, "raw": 0.127901, "normalized": 0.127901, "discount": 1.0, "effective": 0.127901, "running_max": 0.127901, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 85.3}, {"index": 3, "tokens": 16632226, "wall_clock_seconds": 114.6, "raw": 0.08641, "normalized": 0.08641, "discount": 1.0, "effective": 0.08641, "running_max": 0.127901, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 114.37}, {"index": 4, "tokens": 21107989, "wall_clock_seconds": 112.6, "raw": 0.149614, "normalized": 0.149614, "discount": 1.0, "effective": 0.149614, "running_max": 0.149614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 112.37}, {"index": 5, "tokens": 24784744, "wall_clock_seconds": 112.3, "raw": 0.131345, "normalized": 0.131345, "discount": 1.0, "effective": 0.131345, "running_max": 0.149614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 112.07}, {"index": 6, "tokens": 29871365, "wall_clock_seconds": 123.7, "raw": 0.137542, "normalized": 0.137542, "discount": 1.0, "effective": 0.137542, "running_max": 0.149614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 123.38}, {"index": 7, "tokens": 32333590, "wall_clock_seconds": 131.3, "raw": 0.137542, "normalized": 0.137542, "discount": 1.0, "effective": 0.137542, "running_max": 0.149614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 131.04}, {"index": 8, "tokens": 36234312, "wall_clock_seconds": 121.4, "raw": 0.148338, "normalized": 0.148338, "discount": 1.0, "effective": 0.148338, "running_max": 0.149614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 121.17}, {"index": 9, "tokens": 37135933, "wall_clock_seconds": 121.2, "raw": 0.138601, "normalized": 0.138601, "discount": 1.0, "effective": 0.138601, "running_max": 0.149614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.92}, {"index": 10, "tokens": 40893369, "wall_clock_seconds": 123.3, "raw": 0.140615, "normalized": 0.140615, "discount": 1.0, "effective": 0.140615, "running_max": 0.149614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 123.06} ]}, {"sample_key": "3BWB2yRZxELHihLx6E9qZw:weirdml_bonanza", "eval_id": "3BWB2yRZxELHihLx6E9qZw", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "2e6a0ec1178d53b742385ccaeee21aa193242206f4652f4617c5f0f547bd2b28", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.459755, "area_term": 0.358144, "final_best": 0.866198, "best_raw": 0.866198, "best_effective": 0.866198, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 21655600, "usage": {"input": 24172, "output": 240219, "cache_read": 21025254, "cache_write": 365955}, "cost_usd": 24.8281, "cost_source": "list_price", "wall_clock_seconds": 7748.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7051771, "wall_clock_seconds": 96.8, "raw": 0.75279, "normalized": 0.75279, "discount": 1.0, "effective": 0.75279, "running_max": 0.75279, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.27}, {"index": 2, "tokens": 8147550, "wall_clock_seconds": 97.7, "raw": 0.773699, "normalized": 0.773699, "discount": 1.0, "effective": 0.773699, "running_max": 0.773699, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 97.19}, {"index": 3, "tokens": 8870205, "wall_clock_seconds": 104.4, "raw": 0.821186, "normalized": 0.821186, "discount": 1.0, "effective": 0.821186, "running_max": 0.821186, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.76}, {"index": 4, "tokens": 9881900, "wall_clock_seconds": 108.0, "raw": 0.826359, "normalized": 0.826359, "discount": 1.0, "effective": 0.826359, "running_max": 0.826359, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.4}, {"index": 5, "tokens": 12093822, "wall_clock_seconds": 103.7, "raw": 0.833304, "normalized": 0.833304, "discount": 1.0, "effective": 0.833304, "running_max": 0.833304, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.08}, {"index": 6, "tokens": 14187488, "wall_clock_seconds": 107.9, "raw": 0.852399, "normalized": 0.852399, "discount": 1.0, "effective": 0.852399, "running_max": 0.852399, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.21}, {"index": 7, "tokens": 15440445, "wall_clock_seconds": 106.9, "raw": 0.853406, "normalized": 0.853406, "discount": 1.0, "effective": 0.853406, "running_max": 0.853406, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.38}, {"index": 8, "tokens": 18079874, "wall_clock_seconds": 107.1, "raw": 0.849972, "normalized": 0.849972, "discount": 1.0, "effective": 0.849972, "running_max": 0.853406, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.56}, {"index": 9, "tokens": 20559949, "wall_clock_seconds": 109.6, "raw": 0.863992, "normalized": 0.863992, "discount": 1.0, "effective": 0.863992, "running_max": 0.863992, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 109.07}, {"index": 10, "tokens": 21655600, "wall_clock_seconds": 106.4, "raw": 0.866198, "normalized": 0.866198, "discount": 1.0, "effective": 0.866198, "running_max": 0.866198, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.61} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:mystery_box", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.120296, "area_term": 0.104258, "final_best": 0.184447, "best_raw": 0.674496, "best_effective": 0.184447, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 15947446, "usage": {"input": 657, "output": 202857, "cache_read": 15054608, "cache_write": 689324}, "cost_usd": 33.8206, "cost_source": "billed", "wall_clock_seconds": 8809.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 1, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3065822, "wall_clock_seconds": 3.8, "raw": 0.457899, "normalized": 0.485929, "discount": 0.25, "effective": 0.121482, "running_max": 0.121482, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4985318, "wall_clock_seconds": 3.6, "raw": 0.651903, "normalized": 0.711515, "discount": 0.25, "effective": 0.177879, "running_max": 0.177879, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 6501440, "wall_clock_seconds": 3.7, "raw": 0.648593, "normalized": 0.707667, "discount": 0.25, "effective": 0.176917, "running_max": 0.177879, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 6987777, "wall_clock_seconds": 4.2, "raw": 0.589204, "normalized": 0.638609, "discount": 0.25, "effective": 0.159652, "running_max": 0.177879, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 7999449, "wall_clock_seconds": 4.0, "raw": 0.663405, "normalized": 0.72489, "discount": 0.25, "effective": 0.181222, "running_max": 0.181222, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8519681, "wall_clock_seconds": 4.0, "raw": 0.66911, "normalized": 0.731523, "discount": 0.25, "effective": 0.182881, "running_max": 0.182881, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 9412526, "wall_clock_seconds": 3.8, "raw": 0.669804, "normalized": 0.73233, "discount": 0.25, "effective": 0.183083, "running_max": 0.183083, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 9775904, "wall_clock_seconds": 3.7, "raw": 0.669621, "normalized": 0.732118, "discount": 0.25, "effective": 0.183029, "running_max": 0.183083, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 9957959, "wall_clock_seconds": 3.8, "raw": 0.669672, "normalized": 0.732177, "discount": 0.25, "effective": 0.183044, "running_max": 0.183083, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 15947446, "wall_clock_seconds": 4.3, "raw": 0.674496, "normalized": 0.737786, "discount": 0.25, "effective": 0.184447, "running_max": 0.184447, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:mystery_box--nohints", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.03691, "area_term": 0.03691, "final_best": 0.03691, "best_raw": 0.071743, "best_effective": 0.03691, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2862586, "usage": {"input": 150, "output": 54047, "cache_read": 2666409, "cache_write": 141980}, "cost_usd": 7.145, "cost_source": "billed", "wall_clock_seconds": 1972.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 341905, "wall_clock_seconds": 2.9, "raw": 0.071743, "normalized": 0.03691, "discount": 1.0, "effective": 0.03691, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 379602, "wall_clock_seconds": 2.9, "raw": 0.044703, "normalized": 0.005469, "discount": 1.0, "effective": 0.005469, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 919203, "wall_clock_seconds": 4.2, "raw": 0.066075, "normalized": 0.03032, "discount": 1.0, "effective": 0.03032, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1042045, "wall_clock_seconds": 5.6, "raw": 0.069497, "normalized": 0.034299, "discount": 1.0, "effective": 0.034299, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1239868, "wall_clock_seconds": 5.5, "raw": 0.044703, "normalized": 0.005469, "discount": 1.0, "effective": 0.005469, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1376192, "wall_clock_seconds": 2.5, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1607231, "wall_clock_seconds": 2.5, "raw": 0.051389, "normalized": 0.013243, "discount": 1.0, "effective": 0.013243, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2134049, "wall_clock_seconds": 3.0, "raw": 0.056759, "normalized": 0.019487, "discount": 1.0, "effective": 0.019487, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2431119, "wall_clock_seconds": 2.5, "raw": 0.056063, "normalized": 0.018678, "discount": 1.0, "effective": 0.018678, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 2862586, "wall_clock_seconds": 2.9, "raw": 0.060326, "normalized": 0.023635, "discount": 1.0, "effective": 0.023635, "running_max": 0.03691, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:night_school", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.496981, "area_term": 0.480711, "final_best": 0.562059, "best_raw": 0.885, "best_effective": 0.562059, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7407063, "usage": {"input": 327, "output": 91183, "cache_read": 7137629, "cache_write": 177924}, "cost_usd": 13.9241, "cost_source": "billed", "wall_clock_seconds": 5473.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 547498, "wall_clock_seconds": 11.4, "raw": 0.4595, "normalized": 0.364118, "discount": 0.65, "effective": 0.236676, "running_max": 0.236676, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1250119, "wall_clock_seconds": 11.8, "raw": 0.752875, "normalized": 0.709265, "discount": 0.65, "effective": 0.461022, "running_max": 0.461022, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1953253, "wall_clock_seconds": 12.0, "raw": 0.869, "normalized": 0.845882, "discount": 0.65, "effective": 0.549824, "running_max": 0.549824, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2397964, "wall_clock_seconds": 11.2, "raw": 0.861, "normalized": 0.836471, "discount": 0.65, "effective": 0.543706, "running_max": 0.549824, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3461498, "wall_clock_seconds": 12.2, "raw": 0.8755, "normalized": 0.853529, "discount": 0.65, "effective": 0.554794, "running_max": 0.554794, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3575072, "wall_clock_seconds": 12.1, "raw": 0.8765, "normalized": 0.854706, "discount": 0.65, "effective": 0.555559, "running_max": 0.555559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4573719, "wall_clock_seconds": 13.1, "raw": 0.882, "normalized": 0.861176, "discount": 0.65, "effective": 0.559765, "running_max": 0.559765, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5648599, "wall_clock_seconds": 10.9, "raw": 0.864, "normalized": 0.84, "discount": 0.65, "effective": 0.546, "running_max": 0.559765, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6800961, "wall_clock_seconds": 12.2, "raw": 0.883, "normalized": 0.862353, "discount": 0.65, "effective": 0.560529, "running_max": 0.560529, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7407063, "wall_clock_seconds": 12.9, "raw": 0.885, "normalized": 0.864706, "discount": 0.65, "effective": 0.562059, "running_max": 0.562059, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:reaction_rates", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.401843, "area_term": 0.363674, "final_best": 0.554521, "best_raw": 0.990216, "best_effective": 0.554521, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 14138863, "usage": {"input": 667, "output": 135256, "cache_read": 13334692, "cache_write": 668248}, "cost_usd": 28.4573, "cost_source": "billed", "wall_clock_seconds": 9936.7, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 1}, {"id": 4, "multiplier": 0.8, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2301589, "wall_clock_seconds": 38.4, "raw": 0.891317, "normalized": 0.891317, "discount": 0.56, "effective": 0.499137, "running_max": 0.499137, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 38.06}, {"index": 2, "tokens": 3235681, "wall_clock_seconds": 115.2, "raw": 0.92255, "normalized": 0.92255, "discount": 0.56, "effective": 0.516628, "running_max": 0.516628, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 114.89}, {"index": 3, "tokens": 4567509, "wall_clock_seconds": 283.4, "raw": 0.986646, "normalized": 0.986646, "discount": 0.56, "effective": 0.552522, "running_max": 0.552522, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 283.06}, {"index": 4, "tokens": 5302722, "wall_clock_seconds": 306.8, "raw": 0.990054, "normalized": 0.990054, "discount": 0.56, "effective": 0.55443, "running_max": 0.55443, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 306.4}, {"index": 5, "tokens": 5774887, "wall_clock_seconds": 307.8, "raw": 0.990055, "normalized": 0.990055, "discount": 0.56, "effective": 0.554431, "running_max": 0.554431, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 307.52}, {"index": 6, "tokens": 7883681, "wall_clock_seconds": 302.0, "raw": 0.990189, "normalized": 0.990189, "discount": 0.56, "effective": 0.554506, "running_max": 0.554506, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 301.73}, {"index": 7, "tokens": 9044479, "wall_clock_seconds": 372.5, "raw": 0.990189, "normalized": 0.990189, "discount": 0.56, "effective": 0.554506, "running_max": 0.554506, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 372.16}, {"index": 8, "tokens": 12268183, "wall_clock_seconds": 459.7, "raw": 0.990166, "normalized": 0.990166, "discount": 0.56, "effective": 0.554493, "running_max": 0.554506, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 459.33}, {"index": 9, "tokens": 13210589, "wall_clock_seconds": 452.4, "raw": 0.990181, "normalized": 0.990181, "discount": 0.56, "effective": 0.554501, "running_max": 0.554506, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 452.08}, {"index": 10, "tokens": 14138863, "wall_clock_seconds": 467.1, "raw": 0.990216, "normalized": 0.990216, "discount": 0.56, "effective": 0.554521, "running_max": 0.554521, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 466.68} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:reaction_rates--nohints", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.483988, "area_term": 0.411236, "final_best": 0.774994, "best_raw": 0.774994, "best_effective": 0.774994, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 14598696, "usage": {"input": 348, "output": 160533, "cache_read": 13870841, "cache_write": 566974}, "cost_usd": 28.9881, "cost_source": "billed", "wall_clock_seconds": 7953.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1253005, "wall_clock_seconds": 15.9, "raw": 0.138149, "normalized": 0.138149, "discount": 1.0, "effective": 0.138149, "running_max": 0.138149, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.62}, {"index": 2, "tokens": 1787956, "wall_clock_seconds": 16.6, "raw": 0.287106, "normalized": 0.287106, "discount": 1.0, "effective": 0.287106, "running_max": 0.287106, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 16.29}, {"index": 3, "tokens": 3221477, "wall_clock_seconds": 68.3, "raw": 0.358795, "normalized": 0.358795, "discount": 1.0, "effective": 0.358795, "running_max": 0.358795, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 67.96}, {"index": 4, "tokens": 8203192, "wall_clock_seconds": 152.4, "raw": 0.628981, "normalized": 0.628981, "discount": 1.0, "effective": 0.628981, "running_max": 0.628981, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 152.12}, {"index": 5, "tokens": 10144274, "wall_clock_seconds": 79.2, "raw": 0.452003, "normalized": 0.452003, "discount": 1.0, "effective": 0.452003, "running_max": 0.628981, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 78.89}, {"index": 6, "tokens": 11546239, "wall_clock_seconds": 176.4, "raw": 0.641045, "normalized": 0.641045, "discount": 1.0, "effective": 0.641045, "running_max": 0.641045, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 176.08}, {"index": 7, "tokens": 12024089, "wall_clock_seconds": 272.7, "raw": 0.744963, "normalized": 0.744963, "discount": 1.0, "effective": 0.744963, "running_max": 0.744963, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 272.26}, {"index": 8, "tokens": 12509095, "wall_clock_seconds": 418.5, "raw": 0.602658, "normalized": 0.602658, "discount": 1.0, "effective": 0.602658, "running_max": 0.744963, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 418.24}, {"index": 9, "tokens": 13223019, "wall_clock_seconds": 230.9, "raw": 0.751695, "normalized": 0.751695, "discount": 1.0, "effective": 0.751695, "running_max": 0.751695, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 230.59}, {"index": 10, "tokens": 14598696, "wall_clock_seconds": 276.9, "raw": 0.774994, "normalized": 0.774994, "discount": 1.0, "effective": 0.774994, "running_max": 0.774994, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 276.62} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:scan_stitch", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.215542, "area_term": 0.187115, "final_best": 0.329251, "best_raw": 0.852574, "best_effective": 0.329251, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8407839, "usage": {"input": 486, "output": 76719, "cache_read": 7996120, "cache_write": 334514}, "cost_usd": 16.0184, "cost_source": "billed", "wall_clock_seconds": 3340.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 2}, {"id": 4, "multiplier": 0.35, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1006108, "wall_clock_seconds": 0.3, "raw": 0.08611, "normalized": 0.0, "discount": 0.85, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2199264, "wall_clock_seconds": 0.4, "raw": 0.111698, "normalized": 0.014623, "discount": 0.68, "effective": 0.009944, "running_max": 0.009944, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3320881, "wall_clock_seconds": 0.3, "raw": 0.514838, "normalized": 0.518547, "discount": 0.35, "effective": 0.181491, "running_max": 0.181491, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3897777, "wall_clock_seconds": 0.3, "raw": 0.741404, "normalized": 0.801755, "discount": 0.35, "effective": 0.280614, "running_max": 0.280614, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4646336, "wall_clock_seconds": 0.3, "raw": 0.821702, "normalized": 0.902128, "discount": 0.35, "effective": 0.315745, "running_max": 0.315745, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 5310798, "wall_clock_seconds": 0.3, "raw": 0.841074, "normalized": 0.926342, "discount": 0.35, "effective": 0.32422, "running_max": 0.32422, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 6007723, "wall_clock_seconds": 0.2, "raw": 0.847425, "normalized": 0.934281, "discount": 0.35, "effective": 0.326998, "running_max": 0.326998, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6733581, "wall_clock_seconds": 0.3, "raw": 0.83992, "normalized": 0.9249, "discount": 0.35, "effective": 0.323715, "running_max": 0.326998, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 7789459, "wall_clock_seconds": 0.3, "raw": 0.851433, "normalized": 0.939291, "discount": 0.35, "effective": 0.328752, "running_max": 0.328752, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 8407839, "wall_clock_seconds": 0.3, "raw": 0.852574, "normalized": 0.940717, "discount": 0.35, "effective": 0.329251, "running_max": 0.329251, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:shapes_generalize", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.792789, "area_term": 0.781924, "final_best": 0.83625, "best_raw": 0.869, "best_effective": 0.83625, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3494574, "usage": {"input": 168, "output": 63663, "cache_read": 3321538, "cache_write": 109205}, "cost_usd": 7.8714, "cost_source": "billed", "wall_clock_seconds": 3571.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 180928, "wall_clock_seconds": 2.8, "raw": 0.22, "normalized": 0.025, "discount": 1.0, "effective": 0.025, "running_max": 0.025, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.21}, {"index": 2, "tokens": 243842, "wall_clock_seconds": 5.7, "raw": 0.209, "normalized": 0.01125, "discount": 1.0, "effective": 0.01125, "running_max": 0.025, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.11}, {"index": 3, "tokens": 430878, "wall_clock_seconds": 106.5, "raw": 0.341, "normalized": 0.17625, "discount": 1.0, "effective": 0.17625, "running_max": 0.17625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.01}, {"index": 4, "tokens": 521268, "wall_clock_seconds": 106.9, "raw": 0.038, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.17625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.34}, {"index": 5, "tokens": 664811, "wall_clock_seconds": 108.6, "raw": 0.777, "normalized": 0.72125, "discount": 1.0, "effective": 0.72125, "running_max": 0.72125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.85}, {"index": 6, "tokens": 946904, "wall_clock_seconds": 102.9, "raw": 0.852, "normalized": 0.815, "discount": 1.0, "effective": 0.815, "running_max": 0.815, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 102.35}, {"index": 7, "tokens": 1405504, "wall_clock_seconds": 101.7, "raw": 0.784, "normalized": 0.73, "discount": 1.0, "effective": 0.73, "running_max": 0.815, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.91}, {"index": 8, "tokens": 2375677, "wall_clock_seconds": 108.0, "raw": 0.865, "normalized": 0.83125, "discount": 1.0, "effective": 0.83125, "running_max": 0.83125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.14}, {"index": 9, "tokens": 3276228, "wall_clock_seconds": 109.3, "raw": 0.851, "normalized": 0.81375, "discount": 1.0, "effective": 0.81375, "running_max": 0.83125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 108.76}, {"index": 10, "tokens": 3494574, "wall_clock_seconds": 108.6, "raw": 0.869, "normalized": 0.83625, "discount": 1.0, "effective": 0.83625, "running_max": 0.83625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 108.06} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:shattered_prior", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.591449, "area_term": 0.514311, "final_best": 0.9, "best_raw": 1.0, "best_effective": 0.9, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 8846918, "usage": {"input": 375, "output": 94369, "cache_read": 8412039, "cache_write": 340135}, "cost_usd": 17.3859, "cost_source": "billed", "wall_clock_seconds": 3598.9, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1618640, "wall_clock_seconds": 1.7, "raw": 0.157558, "normalized": 0.157558, "discount": 0.9, "effective": 0.141802, "running_max": 0.141802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3260418, "wall_clock_seconds": 1.7, "raw": 0.494334, "normalized": 0.494334, "discount": 0.9, "effective": 0.444901, "running_max": 0.444901, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3668536, "wall_clock_seconds": 1.9, "raw": 0.651398, "normalized": 0.651398, "discount": 0.9, "effective": 0.586259, "running_max": 0.586259, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 4711641, "wall_clock_seconds": 2.0, "raw": 0.876369, "normalized": 0.876369, "discount": 0.9, "effective": 0.788732, "running_max": 0.788732, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 7475274, "wall_clock_seconds": 1.8, "raw": 0.950164, "normalized": 0.950164, "discount": 0.9, "effective": 0.855148, "running_max": 0.855148, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8248769, "wall_clock_seconds": 1.9, "raw": 1.0, "normalized": 1.0, "discount": 0.9, "effective": 0.9, "running_max": 0.9, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:shattered_prior--nohints", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.691815, "area_term": 0.614769, "final_best": 1.0, "best_raw": 1.0, "best_effective": 1.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 4126008, "usage": {"input": 180, "output": 52280, "cache_read": 3951545, "cache_write": 122003}, "cost_usd": 8.0924, "cost_source": "billed", "wall_clock_seconds": 1752.6, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2017474, "wall_clock_seconds": 1.7, "raw": 0.182708, "normalized": 0.182708, "discount": 1.0, "effective": 0.182708, "running_max": 0.182708, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3071470, "wall_clock_seconds": 2.1, "raw": 0.774682, "normalized": 0.774682, "discount": 1.0, "effective": 0.774682, "running_max": 0.774682, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3407474, "wall_clock_seconds": 1.8, "raw": 0.876369, "normalized": 0.876369, "discount": 1.0, "effective": 0.876369, "running_max": 0.876369, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3759499, "wall_clock_seconds": 1.7, "raw": 1.0, "normalized": 1.0, "discount": 1.0, "effective": 1.0, "running_max": 1.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:ship_tune", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.214948, "area_term": 0.197124, "final_best": 0.286244, "best_raw": 0.286244, "best_effective": 0.286244, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17511273, "usage": {"input": 590, "output": 177615, "cache_read": 16434543, "cache_write": 898525}, "cost_usd": 36.5528, "cost_source": "billed", "wall_clock_seconds": 22210.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1675671, "wall_clock_seconds": 51.8, "raw": 0.241438, "normalized": 0.241438, "discount": 1.0, "effective": 0.241438, "running_max": 0.241438, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 51.2}, {"index": 2, "tokens": 4130749, "wall_clock_seconds": 113.3, "raw": 0.260568, "normalized": 0.260568, "discount": 1.0, "effective": 0.260568, "running_max": 0.260568, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 112.62}, {"index": 3, "tokens": 7161388, "wall_clock_seconds": 115.8, "raw": 0.270869, "normalized": 0.270869, "discount": 1.0, "effective": 0.270869, "running_max": 0.270869, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 115.16}, {"index": 4, "tokens": 10795279, "wall_clock_seconds": 51.3, "raw": 0.265968, "normalized": 0.265968, "discount": 1.0, "effective": 0.265968, "running_max": 0.270869, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 50.61}, {"index": 5, "tokens": 11260014, "wall_clock_seconds": 341.3, "raw": 0.277805, "normalized": 0.277805, "discount": 1.0, "effective": 0.277805, "running_max": 0.277805, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 340.41}, {"index": 6, "tokens": 12683776, "wall_clock_seconds": 329.9, "raw": 0.275275, "normalized": 0.275275, "discount": 1.0, "effective": 0.275275, "running_max": 0.277805, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 329.11}, {"index": 7, "tokens": 15196079, "wall_clock_seconds": 364.8, "raw": 0.284752, "normalized": 0.284752, "discount": 1.0, "effective": 0.284752, "running_max": 0.284752, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 363.79}, {"index": 8, "tokens": 16228603, "wall_clock_seconds": 432.5, "raw": 0.281524, "normalized": 0.281524, "discount": 1.0, "effective": 0.281524, "running_max": 0.284752, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 431.69}, {"index": 9, "tokens": 17246194, "wall_clock_seconds": 672.8, "raw": 0.285614, "normalized": 0.285614, "discount": 1.0, "effective": 0.285614, "running_max": 0.285614, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 671.9}, {"index": 10, "tokens": 17511273, "wall_clock_seconds": 652.8, "raw": 0.286244, "normalized": 0.286244, "discount": 1.0, "effective": 0.286244, "running_max": 0.286244, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 651.97} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:splash_generalize", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.319999, "area_term": 0.306943, "final_best": 0.372222, "best_raw": 0.435, "best_effective": 0.372222, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 4071518, "usage": {"input": 264, "output": 70556, "cache_read": 3818166, "cache_write": 182532}, "cost_usd": 9.6303, "cost_source": "billed", "wall_clock_seconds": 3140.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 145987, "wall_clock_seconds": 3.7, "raw": 0.107, "normalized": 0.007778, "discount": 1.0, "effective": 0.007778, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.02}, {"index": 2, "tokens": 230276, "wall_clock_seconds": 2.4, "raw": 0.09, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.76}, {"index": 3, "tokens": 293844, "wall_clock_seconds": 0.9, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.3}, {"index": 4, "tokens": 473133, "wall_clock_seconds": 2.0, "raw": 0.149, "normalized": 0.054444, "discount": 1.0, "effective": 0.054444, "running_max": 0.054444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.35}, {"index": 5, "tokens": 803074, "wall_clock_seconds": 11.1, "raw": 0.236, "normalized": 0.151111, "discount": 1.0, "effective": 0.151111, "running_max": 0.151111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 10.57}, {"index": 6, "tokens": 1284050, "wall_clock_seconds": 86.4, "raw": 0.389, "normalized": 0.321111, "discount": 1.0, "effective": 0.321111, "running_max": 0.321111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 85.67}, {"index": 7, "tokens": 2101610, "wall_clock_seconds": 83.6, "raw": 0.384, "normalized": 0.315556, "discount": 1.0, "effective": 0.315556, "running_max": 0.321111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 82.85}, {"index": 8, "tokens": 2623067, "wall_clock_seconds": 84.4, "raw": 0.407, "normalized": 0.341111, "discount": 1.0, "effective": 0.341111, "running_max": 0.341111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 83.79}, {"index": 9, "tokens": 3104432, "wall_clock_seconds": 80.7, "raw": 0.42, "normalized": 0.355556, "discount": 1.0, "effective": 0.355556, "running_max": 0.355556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 80.03}, {"index": 10, "tokens": 4071518, "wall_clock_seconds": 88.7, "raw": 0.435, "normalized": 0.372222, "discount": 1.0, "effective": 0.372222, "running_max": 0.372222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 88.07} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:tod_pipeline--nohints", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.181362, "area_term": 0.175502, "final_best": 0.204801, "best_raw": 0.204801, "best_effective": 0.204801, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 4472943, "usage": {"input": 174, "output": 68659, "cache_read": 4270706, "cache_write": 133404}, "cost_usd": 9.3729, "cost_source": "billed", "wall_clock_seconds": 3466.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 603753, "wall_clock_seconds": 16.0, "raw": 0.08365, "normalized": 0.08365, "discount": 1.0, "effective": 0.08365, "running_max": 0.08365, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.73}, {"index": 2, "tokens": 1082297, "wall_clock_seconds": 20.2, "raw": 0.172848, "normalized": 0.172848, "discount": 1.0, "effective": 0.172848, "running_max": 0.172848, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 19.88}, {"index": 3, "tokens": 1845049, "wall_clock_seconds": 65.1, "raw": 0.191746, "normalized": 0.191746, "discount": 1.0, "effective": 0.191746, "running_max": 0.191746, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 64.78}, {"index": 4, "tokens": 2328182, "wall_clock_seconds": 2.3, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.191746, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 2.05}, {"index": 5, "tokens": 2637399, "wall_clock_seconds": 74.7, "raw": 0.192364, "normalized": 0.192364, "discount": 1.0, "effective": 0.192364, "running_max": 0.192364, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 74.42}, {"index": 6, "tokens": 2860153, "wall_clock_seconds": 70.5, "raw": 0.192512, "normalized": 0.192512, "discount": 1.0, "effective": 0.192512, "running_max": 0.192512, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 70.23}, {"index": 7, "tokens": 3093315, "wall_clock_seconds": 126.8, "raw": 0.176563, "normalized": 0.176563, "discount": 1.0, "effective": 0.176563, "running_max": 0.192512, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 126.5}, {"index": 8, "tokens": 3575142, "wall_clock_seconds": 153.5, "raw": 0.203959, "normalized": 0.203959, "discount": 1.0, "effective": 0.203959, "running_max": 0.203959, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 153.07}, {"index": 9, "tokens": 4206336, "wall_clock_seconds": 144.8, "raw": 0.204801, "normalized": 0.204801, "discount": 1.0, "effective": 0.204801, "running_max": 0.204801, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 144.49}, {"index": 10, "tokens": 4472943, "wall_clock_seconds": 209.2, "raw": 0.193262, "normalized": 0.193262, "discount": 1.0, "effective": 0.193262, "running_max": 0.204801, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 208.84} ]}, {"sample_key": "6PCUQXFcNhRxsau8RznvPo:weirdml_bonanza", "eval_id": "6PCUQXFcNhRxsau8RznvPo", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "0c8c7e8ada131fdb1029918b86da5e7825feaa6c1794aa894da1194ed525b445", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.729213, "area_term": 0.696341, "final_best": 0.860699, "best_raw": 0.860699, "best_effective": 0.860699, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11154495, "usage": {"input": 285, "output": 152190, "cache_read": 10772221, "cache_write": 229799}, "cost_usd": 21.2571, "cost_source": "billed", "wall_clock_seconds": 7231.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1093356, "wall_clock_seconds": 87.1, "raw": 0.76278, "normalized": 0.76278, "discount": 1.0, "effective": 0.76278, "running_max": 0.76278, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.43}, {"index": 2, "tokens": 1385486, "wall_clock_seconds": 69.5, "raw": 0.811375, "normalized": 0.811375, "discount": 1.0, "effective": 0.811375, "running_max": 0.811375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 68.91}, {"index": 3, "tokens": 2783531, "wall_clock_seconds": 108.0, "raw": 0.816739, "normalized": 0.816739, "discount": 1.0, "effective": 0.816739, "running_max": 0.816739, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.28}, {"index": 4, "tokens": 3950731, "wall_clock_seconds": 102.5, "raw": 0.832348, "normalized": 0.832348, "discount": 1.0, "effective": 0.832348, "running_max": 0.832348, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.76}, {"index": 5, "tokens": 4939776, "wall_clock_seconds": 106.5, "raw": 0.845727, "normalized": 0.845727, "discount": 1.0, "effective": 0.845727, "running_max": 0.845727, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.87}, {"index": 6, "tokens": 6407622, "wall_clock_seconds": 113.1, "raw": 0.860222, "normalized": 0.860222, "discount": 1.0, "effective": 0.860222, "running_max": 0.860222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 112.44}, {"index": 7, "tokens": 8537541, "wall_clock_seconds": 117.2, "raw": 0.854881, "normalized": 0.854881, "discount": 1.0, "effective": 0.854881, "running_max": 0.860222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 116.54}, {"index": 8, "tokens": 9366618, "wall_clock_seconds": 111.7, "raw": 0.857899, "normalized": 0.857899, "discount": 1.0, "effective": 0.857899, "running_max": 0.860222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 110.98}, {"index": 9, "tokens": 9796282, "wall_clock_seconds": 108.1, "raw": 0.85997, "normalized": 0.85997, "discount": 1.0, "effective": 0.85997, "running_max": 0.860222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.49}, {"index": 10, "tokens": 11154495, "wall_clock_seconds": 114.7, "raw": 0.860699, "normalized": 0.860699, "discount": 1.0, "effective": 0.860699, "running_max": 0.860699, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 114.1} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:mystery_box", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.003122, "area_term": 0.002232, "final_best": 0.00668, "best_raw": 0.062979, "best_effective": 0.00668, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 34733454, "usage": {"input": 918, "output": 215487, "cache_read": 32167131, "cache_write": 2349918}, "cost_usd": 14.4649, "cost_source": "billed", "wall_clock_seconds": 14918.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3692619, "wall_clock_seconds": 5.3, "raw": 0.044396, "normalized": 0.005112, "discount": 0.6, "effective": 0.003067, "running_max": 0.003067, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 22564706, "wall_clock_seconds": 9.4, "raw": 0.055889, "normalized": 0.018476, "discount": 0.25, "effective": 0.004619, "running_max": 0.004619, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 25290934, "wall_clock_seconds": 4.9, "raw": 0.05148, "normalized": 0.013349, "discount": 0.25, "effective": 0.003337, "running_max": 0.004619, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 26642877, "wall_clock_seconds": 6.7, "raw": 0.046105, "normalized": 0.007099, "discount": 0.25, "effective": 0.001775, "running_max": 0.004619, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 28869980, "wall_clock_seconds": 6.7, "raw": 0.054314, "normalized": 0.016644, "discount": 0.25, "effective": 0.004161, "running_max": 0.004619, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 29348631, "wall_clock_seconds": 6.9, "raw": 0.061798, "normalized": 0.025347, "discount": 0.25, "effective": 0.006337, "running_max": 0.006337, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 30746046, "wall_clock_seconds": 4.0, "raw": 0.060552, "normalized": 0.023897, "discount": 0.25, "effective": 0.005974, "running_max": 0.006337, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 32100467, "wall_clock_seconds": 4.1, "raw": 0.05806, "normalized": 0.021, "discount": 0.25, "effective": 0.00525, "running_max": 0.006337, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 33273741, "wall_clock_seconds": 14.3, "raw": 0.062884, "normalized": 0.026609, "discount": 0.25, "effective": 0.006652, "running_max": 0.006652, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 34733454, "wall_clock_seconds": 8.0, "raw": 0.062979, "normalized": 0.026719, "discount": 0.25, "effective": 0.00668, "running_max": 0.00668, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:mystery_box--nohints", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.024448, "area_term": 0.023322, "final_best": 0.028948, "best_raw": 0.064895, "best_effective": 0.028948, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5418029, "usage": {"input": 168, "output": 48192, "cache_read": 5015329, "cache_write": 354340}, "cost_usd": 2.3712, "cost_source": "billed", "wall_clock_seconds": 1922.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 641641, "wall_clock_seconds": 2.8, "raw": 0.046819, "normalized": 0.007929, "discount": 1.0, "effective": 0.007929, "running_max": 0.007929, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1049781, "wall_clock_seconds": 2.9, "raw": 0.039335, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007929, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1278134, "wall_clock_seconds": 2.8, "raw": 0.059404, "normalized": 0.022563, "discount": 1.0, "effective": 0.022563, "running_max": 0.022563, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2467961, "wall_clock_seconds": 2.8, "raw": 0.064895, "normalized": 0.028948, "discount": 1.0, "effective": 0.028948, "running_max": 0.028948, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2715416, "wall_clock_seconds": 2.8, "raw": 0.052473, "normalized": 0.014503, "discount": 1.0, "effective": 0.014503, "running_max": 0.028948, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3122200, "wall_clock_seconds": 4.0, "raw": 0.056234, "normalized": 0.018876, "discount": 1.0, "effective": 0.018876, "running_max": 0.028948, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 3563520, "wall_clock_seconds": 2.7, "raw": 0.064895, "normalized": 0.028948, "discount": 1.0, "effective": 0.028948, "running_max": 0.028948, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4528589, "wall_clock_seconds": 2.9, "raw": 0.047366, "normalized": 0.008565, "discount": 1.0, "effective": 0.008565, "running_max": 0.028948, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 4875874, "wall_clock_seconds": 2.2, "raw": 0.062019, "normalized": 0.025603, "discount": 1.0, "effective": 0.025603, "running_max": 0.028948, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5418029, "wall_clock_seconds": 5.0, "raw": 0.055996, "normalized": 0.0186, "discount": 1.0, "effective": 0.0186, "running_max": 0.028948, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:night_school", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.322076, "area_term": 0.304235, "final_best": 0.393441, "best_raw": 0.6645, "best_effective": 0.393441, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 37941256, "usage": {"input": 972, "output": 198722, "cache_read": 35939814, "cache_write": 1801748}, "cost_usd": 13.6815, "cost_source": "billed", "wall_clock_seconds": 28349.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 676727, "wall_clock_seconds": 12.7, "raw": 0.341, "normalized": 0.224706, "discount": 0.65, "effective": 0.146059, "running_max": 0.146059, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 935192, "wall_clock_seconds": 11.6, "raw": 0.397, "normalized": 0.290588, "discount": 0.65, "effective": 0.188882, "running_max": 0.188882, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1402394, "wall_clock_seconds": 14.1, "raw": 0.5025, "normalized": 0.414706, "discount": 0.65, "effective": 0.269559, "running_max": 0.269559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1978218, "wall_clock_seconds": 12.6, "raw": 0.548, "normalized": 0.468235, "discount": 0.65, "effective": 0.304353, "running_max": 0.304353, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2764060, "wall_clock_seconds": 12.2, "raw": 0.579, "normalized": 0.504706, "discount": 0.65, "effective": 0.328059, "running_max": 0.328059, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4600662, "wall_clock_seconds": 12.2, "raw": 0.628, "normalized": 0.562353, "discount": 0.65, "effective": 0.365529, "running_max": 0.365529, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 7762279, "wall_clock_seconds": 12.1, "raw": 0.6575, "normalized": 0.597059, "discount": 0.65, "effective": 0.388088, "running_max": 0.388088, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 25245832, "wall_clock_seconds": 10.7, "raw": 0.6565, "normalized": 0.595882, "discount": 0.65, "effective": 0.387324, "running_max": 0.388088, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 28477633, "wall_clock_seconds": 11.8, "raw": 0.626, "normalized": 0.56, "discount": 0.65, "effective": 0.364, "running_max": 0.388088, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 37941256, "wall_clock_seconds": 13.1, "raw": 0.6645, "normalized": 0.605294, "discount": 0.65, "effective": 0.393441, "running_max": 0.393441, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:reaction_rates", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.171093, "area_term": 0.145282, "final_best": 0.274337, "best_raw": 0.76845, "best_effective": 0.274337, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12744808, "usage": {"input": 504, "output": 95919, "cache_read": 12141370, "cache_write": 507015}, "cost_usd": 4.656, "cost_source": "billed", "wall_clock_seconds": 5965.6, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.8, "before_submission": 1}, {"id": 5, "multiplier": 0.85, "before_submission": 1}, {"id": 1, "multiplier": 0.7, "before_submission": 3}, {"id": 2, "multiplier": 0.75, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 698401, "wall_clock_seconds": 3.4, "raw": 0.0, "normalized": 0.0, "discount": 0.68, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 3.13}, {"index": 2, "tokens": 3494986, "wall_clock_seconds": 12.6, "raw": 0.288447, "normalized": 0.288447, "discount": 0.68, "effective": 0.196144, "running_max": 0.196144, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 11.95}, {"index": 3, "tokens": 7064447, "wall_clock_seconds": 140.1, "raw": 0.694324, "normalized": 0.694324, "discount": 0.357, "effective": 0.247874, "running_max": 0.247874, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 139.56}, {"index": 4, "tokens": 8100206, "wall_clock_seconds": 151.2, "raw": 0.750029, "normalized": 0.750029, "discount": 0.357, "effective": 0.26776, "running_max": 0.26776, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 150.07}, {"index": 5, "tokens": 8824005, "wall_clock_seconds": 160.2, "raw": 0.747435, "normalized": 0.747435, "discount": 0.357, "effective": 0.266834, "running_max": 0.26776, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 158.67}, {"index": 6, "tokens": 9388057, "wall_clock_seconds": 183.5, "raw": 0.736014, "normalized": 0.736014, "discount": 0.357, "effective": 0.262757, "running_max": 0.26776, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 182.16}, {"index": 7, "tokens": 9769367, "wall_clock_seconds": 162.6, "raw": 0.753424, "normalized": 0.753424, "discount": 0.357, "effective": 0.268973, "running_max": 0.268973, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 161.31}, {"index": 8, "tokens": 10735941, "wall_clock_seconds": 173.9, "raw": 0.757703, "normalized": 0.757703, "discount": 0.357, "effective": 0.2705, "running_max": 0.2705, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 170.88}, {"index": 9, "tokens": 11929158, "wall_clock_seconds": 154.0, "raw": 0.764715, "normalized": 0.764715, "discount": 0.357, "effective": 0.273003, "running_max": 0.273003, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 150.92}, {"index": 10, "tokens": 12744808, "wall_clock_seconds": 169.0, "raw": 0.76845, "normalized": 0.76845, "discount": 0.357, "effective": 0.274337, "running_max": 0.274337, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 166.38} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:reaction_rates--nohints", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.074039, "area_term": 0.062768, "final_best": 0.119124, "best_raw": 0.119124, "best_effective": 0.119124, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7278923, "usage": {"input": 207, "output": 71701, "cache_read": 7037121, "cache_write": 169894}, "cost_usd": 2.5496, "cost_source": "billed", "wall_clock_seconds": 3458.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 446089, "wall_clock_seconds": 31.1, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 24.26}, {"index": 2, "tokens": 824808, "wall_clock_seconds": 41.9, "raw": 0.017231, "normalized": 0.017231, "discount": 1.0, "effective": 0.017231, "running_max": 0.017231, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 39.51}, {"index": 3, "tokens": 3212748, "wall_clock_seconds": 604.5, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.017231, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 600.0}, {"index": 4, "tokens": 4591072, "wall_clock_seconds": 17.3, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.017231, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 11.32}, {"index": 5, "tokens": 5334952, "wall_clock_seconds": 16.4, "raw": 0.046346, "normalized": 0.046346, "discount": 1.0, "effective": 0.046346, "running_max": 0.046346, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.42}, {"index": 6, "tokens": 5642508, "wall_clock_seconds": 10.2, "raw": 0.083574, "normalized": 0.083574, "discount": 1.0, "effective": 0.083574, "running_max": 0.083574, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.24}, {"index": 7, "tokens": 5956960, "wall_clock_seconds": 58.5, "raw": 0.083574, "normalized": 0.083574, "discount": 1.0, "effective": 0.083574, "running_max": 0.083574, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 57.42}, {"index": 8, "tokens": 6605344, "wall_clock_seconds": 23.5, "raw": 0.119124, "normalized": 0.119124, "discount": 1.0, "effective": 0.119124, "running_max": 0.119124, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.88}, {"index": 9, "tokens": 6939149, "wall_clock_seconds": 70.7, "raw": 0.116502, "normalized": 0.116502, "discount": 1.0, "effective": 0.116502, "running_max": 0.119124, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 70.11}, {"index": 10, "tokens": 7278923, "wall_clock_seconds": 70.4, "raw": 0.111881, "normalized": 0.111881, "discount": 1.0, "effective": 0.111881, "running_max": 0.119124, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 69.71} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:scan_stitch", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.118658, "area_term": 0.096046, "final_best": 0.209108, "best_raw": 0.577961, "best_effective": 0.209108, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9320160, "usage": {"input": 528, "output": 101382, "cache_read": 8444343, "cache_write": 773907}, "cost_usd": 4.6385, "cost_source": "billed", "wall_clock_seconds": 3572.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 2}, {"id": 3, "multiplier": 0.75, "before_submission": 2}, {"id": 4, "multiplier": 0.35, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 523044, "wall_clock_seconds": 0.3, "raw": 0.097422, "normalized": 0.0, "discount": 0.85, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2397215, "wall_clock_seconds": 0.3, "raw": 0.082506, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 6009969, "wall_clock_seconds": 0.3, "raw": 0.568838, "normalized": 0.586047, "discount": 0.35, "effective": 0.205116, "running_max": 0.205116, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 6368210, "wall_clock_seconds": 0.4, "raw": 0.563467, "normalized": 0.579333, "discount": 0.35, "effective": 0.202767, "running_max": 0.205116, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 6608554, "wall_clock_seconds": 0.3, "raw": 0.574207, "normalized": 0.592759, "discount": 0.35, "effective": 0.207466, "running_max": 0.207466, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 7407213, "wall_clock_seconds": 0.3, "raw": 0.576542, "normalized": 0.595677, "discount": 0.35, "effective": 0.208487, "running_max": 0.208487, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 7848656, "wall_clock_seconds": 0.3, "raw": 0.545178, "normalized": 0.556473, "discount": 0.35, "effective": 0.194766, "running_max": 0.208487, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 8338656, "wall_clock_seconds": 0.3, "raw": 0.549553, "normalized": 0.561942, "discount": 0.35, "effective": 0.19668, "running_max": 0.208487, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8440908, "wall_clock_seconds": 0.3, "raw": 0.565574, "normalized": 0.581968, "discount": 0.35, "effective": 0.203689, "running_max": 0.208487, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9320160, "wall_clock_seconds": 0.3, "raw": 0.577961, "normalized": 0.597452, "discount": 0.35, "effective": 0.209108, "running_max": 0.209108, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:scan_stitch--nohints", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.01968, "area_term": 0.018896, "final_best": 0.022815, "best_raw": 0.118252, "best_effective": 0.022815, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2688719, "usage": {"input": 120, "output": 55730, "cache_read": 2515383, "cache_write": 117486}, "cost_usd": 1.3543, "cost_source": "billed", "wall_clock_seconds": 1758.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 122361, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 531754, "wall_clock_seconds": 0.3, "raw": 0.105969, "normalized": 0.007462, "discount": 1.0, "effective": 0.007462, "running_max": 0.007462, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 657505, "wall_clock_seconds": 0.4, "raw": 0.098206, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007462, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 785954, "wall_clock_seconds": 0.4, "raw": 0.101022, "normalized": 0.001278, "discount": 1.0, "effective": 0.001278, "running_max": 0.007462, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1067880, "wall_clock_seconds": 0.4, "raw": 0.078627, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007462, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1222023, "wall_clock_seconds": 0.3, "raw": 0.098388, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007462, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1385412, "wall_clock_seconds": 0.4, "raw": 0.109194, "normalized": 0.011492, "discount": 1.0, "effective": 0.011492, "running_max": 0.011492, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 1644741, "wall_clock_seconds": 0.3, "raw": 0.118252, "normalized": 0.022815, "discount": 1.0, "effective": 0.022815, "running_max": 0.022815, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 1834516, "wall_clock_seconds": 0.3, "raw": 0.098718, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.022815, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 2688719, "wall_clock_seconds": 0.3, "raw": 0.10132, "normalized": 0.00165, "discount": 1.0, "effective": 0.00165, "running_max": 0.022815, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:shapes_generalize", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.066745, "area_term": 0.064681, "final_best": 0.075, "best_raw": 0.32, "best_effective": 0.075, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7863079, "usage": {"input": 474, "output": 57924, "cache_read": 7488216, "cache_write": 316465}, "cost_usd": 2.869, "cost_source": "billed", "wall_clock_seconds": 3122.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.5, "before_submission": 7}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 510361, "wall_clock_seconds": 30.7, "raw": 0.234, "normalized": 0.0425, "discount": 1.0, "effective": 0.0425, "running_max": 0.0425, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.04}, {"index": 2, "tokens": 605198, "wall_clock_seconds": 30.6, "raw": 0.229, "normalized": 0.03625, "discount": 1.0, "effective": 0.03625, "running_max": 0.0425, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.0}, {"index": 3, "tokens": 755254, "wall_clock_seconds": 34.1, "raw": 0.184, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0425, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 33.37}, {"index": 4, "tokens": 1023886, "wall_clock_seconds": 32.3, "raw": 0.227, "normalized": 0.03375, "discount": 1.0, "effective": 0.03375, "running_max": 0.0425, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 31.08}, {"index": 5, "tokens": 1203669, "wall_clock_seconds": 29.9, "raw": 0.158, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0425, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 29.33}, {"index": 6, "tokens": 1327362, "wall_clock_seconds": 31.3, "raw": 0.243, "normalized": 0.05375, "discount": 1.0, "effective": 0.05375, "running_max": 0.05375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.62}, {"index": 7, "tokens": 2050638, "wall_clock_seconds": 31.8, "raw": 0.303, "normalized": 0.12875, "discount": 0.5, "effective": 0.064375, "running_max": 0.064375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 31.22}, {"index": 8, "tokens": 3369426, "wall_clock_seconds": 30.0, "raw": 0.31, "normalized": 0.1375, "discount": 0.5, "effective": 0.06875, "running_max": 0.06875, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 29.18}, {"index": 9, "tokens": 3590729, "wall_clock_seconds": 29.4, "raw": 0.32, "normalized": 0.15, "discount": 0.5, "effective": 0.075, "running_max": 0.075, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 28.78}, {"index": 10, "tokens": 7863079, "wall_clock_seconds": 44.5, "raw": 0.314, "normalized": 0.1425, "discount": 0.5, "effective": 0.07125, "running_max": 0.075, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 43.89} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:shattered_prior", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.335111, "area_term": 0.243889, "final_best": 0.7, "best_raw": 1.0, "best_effective": 0.7, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 42268480, "usage": {"input": 1098, "output": 226166, "cache_read": 40592734, "cache_write": 1448482}, "cost_usd": 14.0036, "cost_source": "billed", "wall_clock_seconds": 14105.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.7, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 149042, "wall_clock_seconds": 2.3, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3396602, "wall_clock_seconds": 2.1, "raw": 0.182708, "normalized": 0.182708, "discount": 0.7, "effective": 0.127895, "running_max": 0.127895, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 10812296, "wall_clock_seconds": 1.8, "raw": 0.651398, "normalized": 0.651398, "discount": 0.7, "effective": 0.455979, "running_max": 0.455979, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 16762148, "wall_clock_seconds": 2.1, "raw": 0.774682, "normalized": 0.774682, "discount": 0.7, "effective": 0.542278, "running_max": 0.542278, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 21002768, "wall_clock_seconds": 1.7, "raw": 0.87283, "normalized": 0.87283, "discount": 0.7, "effective": 0.610981, "running_max": 0.610981, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 28600442, "wall_clock_seconds": 1.8, "raw": 0.920218, "normalized": 0.920218, "discount": 0.7, "effective": 0.644152, "running_max": 0.644152, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 32203257, "wall_clock_seconds": 1.8, "raw": 0.754068, "normalized": 0.754068, "discount": 0.7, "effective": 0.527847, "running_max": 0.644152, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 33790566, "wall_clock_seconds": 1.9, "raw": 0.921547, "normalized": 0.921547, "discount": 0.7, "effective": 0.645083, "running_max": 0.645083, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 39391151, "wall_clock_seconds": 1.9, "raw": 0.950164, "normalized": 0.950164, "discount": 0.7, "effective": 0.665115, "running_max": 0.665115, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 42268480, "wall_clock_seconds": 2.5, "raw": 1.0, "normalized": 1.0, "discount": 0.7, "effective": 0.7, "running_max": 0.7, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:shattered_prior--nohints", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.054247, "area_term": 0.050326, "final_best": 0.069934, "best_raw": 0.069934, "best_effective": 0.069934, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 38706520, "usage": {"input": 846, "output": 315978, "cache_read": 36899080, "cache_write": 1490616}, "cost_usd": 14.2678, "cost_source": "billed", "wall_clock_seconds": 13478.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 263376, "wall_clock_seconds": 2.3, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7810275, "wall_clock_seconds": 1.8, "raw": 0.048494, "normalized": 0.048494, "discount": 1.0, "effective": 0.048494, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 21513634, "wall_clock_seconds": 2.4, "raw": 0.047588, "normalized": 0.047588, "discount": 1.0, "effective": 0.047588, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 27767152, "wall_clock_seconds": 2.3, "raw": 0.025013, "normalized": 0.025013, "discount": 1.0, "effective": 0.025013, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 35181300, "wall_clock_seconds": 1.7, "raw": 0.068049, "normalized": 0.068049, "discount": 1.0, "effective": 0.068049, "running_max": 0.068049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 35870883, "wall_clock_seconds": 1.7, "raw": 0.068049, "normalized": 0.068049, "discount": 1.0, "effective": 0.068049, "running_max": 0.068049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 36336631, "wall_clock_seconds": 2.0, "raw": 0.06923, "normalized": 0.06923, "discount": 1.0, "effective": 0.06923, "running_max": 0.06923, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 37044616, "wall_clock_seconds": 1.8, "raw": 0.035636, "normalized": 0.035636, "discount": 1.0, "effective": 0.035636, "running_max": 0.06923, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 38038606, "wall_clock_seconds": 2.3, "raw": 0.066551, "normalized": 0.066551, "discount": 1.0, "effective": 0.066551, "running_max": 0.06923, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 38706520, "wall_clock_seconds": 1.9, "raw": 0.069934, "normalized": 0.069934, "discount": 1.0, "effective": 0.069934, "running_max": 0.069934, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:ship_tune", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.076613, "area_term": 0.074202, "final_best": 0.086259, "best_raw": 0.086259, "best_effective": 0.086259, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17402501, "usage": {"input": 348, "output": 77142, "cache_read": 17107687, "cache_write": 217324}, "cost_usd": 4.737, "cost_source": "billed", "wall_clock_seconds": 13786.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 444612, "wall_clock_seconds": 286.3, "raw": 0.069666, "normalized": 0.069666, "discount": 1.0, "effective": 0.069666, "running_max": 0.069666, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 285.71}, {"index": 2, "tokens": 572824, "wall_clock_seconds": 254.5, "raw": 0.065545, "normalized": 0.065545, "discount": 1.0, "effective": 0.065545, "running_max": 0.069666, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 253.96}, {"index": 3, "tokens": 6287234, "wall_clock_seconds": 161.4, "raw": 0.047508, "normalized": 0.047508, "discount": 1.0, "effective": 0.047508, "running_max": 0.069666, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 160.7}, {"index": 4, "tokens": 8900608, "wall_clock_seconds": 900.6, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.069666, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 900.01}, {"index": 5, "tokens": 10587493, "wall_clock_seconds": 634.7, "raw": 0.054042, "normalized": 0.054042, "discount": 1.0, "effective": 0.054042, "running_max": 0.069666, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 633.98}, {"index": 6, "tokens": 12358005, "wall_clock_seconds": 183.1, "raw": 0.055062, "normalized": 0.055062, "discount": 1.0, "effective": 0.055062, "running_max": 0.069666, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 182.39}, {"index": 7, "tokens": 13580107, "wall_clock_seconds": 900.6, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.069666, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 900.0}, {"index": 8, "tokens": 14198585, "wall_clock_seconds": 407.6, "raw": 0.086259, "normalized": 0.086259, "discount": 1.0, "effective": 0.086259, "running_max": 0.086259, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 406.94}, {"index": 9, "tokens": 15889003, "wall_clock_seconds": 416.2, "raw": 0.084845, "normalized": 0.084845, "discount": 1.0, "effective": 0.084845, "running_max": 0.086259, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 415.53}, {"index": 10, "tokens": 17402501, "wall_clock_seconds": 409.0, "raw": 0.085844, "normalized": 0.085844, "discount": 1.0, "effective": 0.085844, "running_max": 0.086259, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 408.46} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:splash_generalize", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.346805, "area_term": 0.294617, "final_best": 0.555556, "best_raw": 0.6, "best_effective": 0.555556, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10900782, "usage": {"input": 330, "output": 77220, "cache_read": 10654267, "cache_write": 168965}, "cost_usd": 3.3261, "cost_source": "billed", "wall_clock_seconds": 3432.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 189445, "wall_clock_seconds": 1.3, "raw": 0.095, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.68}, {"index": 2, "tokens": 242180, "wall_clock_seconds": 1.1, "raw": 0.102, "normalized": 0.002222, "discount": 1.0, "effective": 0.002222, "running_max": 0.002222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.59}, {"index": 3, "tokens": 606440, "wall_clock_seconds": 2.2, "raw": 0.108, "normalized": 0.008889, "discount": 1.0, "effective": 0.008889, "running_max": 0.008889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.67}, {"index": 4, "tokens": 716704, "wall_clock_seconds": 6.7, "raw": 0.109, "normalized": 0.01, "discount": 1.0, "effective": 0.01, "running_max": 0.01, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.09}, {"index": 5, "tokens": 4457922, "wall_clock_seconds": 71.1, "raw": 0.579, "normalized": 0.532222, "discount": 1.0, "effective": 0.532222, "running_max": 0.532222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 70.47}, {"index": 6, "tokens": 5337791, "wall_clock_seconds": 2.1, "raw": 0.105, "normalized": 0.005556, "discount": 1.0, "effective": 0.005556, "running_max": 0.532222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.48}, {"index": 7, "tokens": 5864242, "wall_clock_seconds": 68.5, "raw": 0.6, "normalized": 0.555556, "discount": 1.0, "effective": 0.555556, "running_max": 0.555556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 67.68}, {"index": 8, "tokens": 6405330, "wall_clock_seconds": 70.2, "raw": 0.596, "normalized": 0.551111, "discount": 1.0, "effective": 0.551111, "running_max": 0.555556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 69.66}, {"index": 9, "tokens": 6681438, "wall_clock_seconds": 71.5, "raw": 0.572, "normalized": 0.524444, "discount": 1.0, "effective": 0.524444, "running_max": 0.555556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 70.76}, {"index": 10, "tokens": 10900782, "wall_clock_seconds": 91.8, "raw": 0.106, "normalized": 0.006667, "discount": 1.0, "effective": 0.006667, "running_max": 0.555556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 91.28} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:tod_pipeline--nohints", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.095973, "area_term": 0.095792, "final_best": 0.0967, "best_raw": 0.0967, "best_effective": 0.0967, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6500935, "usage": {"input": 195, "output": 68835, "cache_read": 6275931, "cache_write": 155974}, "cost_usd": 2.3339, "cost_source": "billed", "wall_clock_seconds": 2944.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 414116, "wall_clock_seconds": 22.0, "raw": 0.084868, "normalized": 0.084868, "discount": 1.0, "effective": 0.084868, "running_max": 0.084868, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.7}, {"index": 2, "tokens": 668911, "wall_clock_seconds": 25.5, "raw": 0.096214, "normalized": 0.096214, "discount": 1.0, "effective": 0.096214, "running_max": 0.096214, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.16}, {"index": 3, "tokens": 1972211, "wall_clock_seconds": 22.4, "raw": 0.044527, "normalized": 0.044527, "discount": 1.0, "effective": 0.044527, "running_max": 0.096214, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.11}, {"index": 4, "tokens": 2606037, "wall_clock_seconds": 15.3, "raw": 0.000237, "normalized": 0.000237, "discount": 1.0, "effective": 0.000237, "running_max": 0.096214, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.03}, {"index": 5, "tokens": 3054643, "wall_clock_seconds": 26.6, "raw": 0.0967, "normalized": 0.0967, "discount": 1.0, "effective": 0.0967, "running_max": 0.0967, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.32}, {"index": 6, "tokens": 3530337, "wall_clock_seconds": 50.6, "raw": 0.093723, "normalized": 0.093723, "discount": 1.0, "effective": 0.093723, "running_max": 0.0967, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 50.34}, {"index": 7, "tokens": 4174164, "wall_clock_seconds": 41.8, "raw": 0.095737, "normalized": 0.095737, "discount": 1.0, "effective": 0.095737, "running_max": 0.0967, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 41.3}, {"index": 8, "tokens": 5146617, "wall_clock_seconds": 41.7, "raw": 0.09581, "normalized": 0.09581, "discount": 1.0, "effective": 0.09581, "running_max": 0.0967, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 41.39}, {"index": 9, "tokens": 6188966, "wall_clock_seconds": 41.3, "raw": 0.095768, "normalized": 0.095768, "discount": 1.0, "effective": 0.095768, "running_max": 0.0967, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 41.04}, {"index": 10, "tokens": 6500935, "wall_clock_seconds": 41.7, "raw": 0.095777, "normalized": 0.095777, "discount": 1.0, "effective": 0.095777, "running_max": 0.0967, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 41.42} ]}, {"sample_key": "8rTHjvnW5LQqsHZ9Feus8j:weirdml_bonanza", "eval_id": "8rTHjvnW5LQqsHZ9Feus8j", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "ff794da8fab3449128b1a64d1dc258a128ad4a8311cdfb69391052637fc7f519", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.501013, "area_term": 0.453938, "final_best": 0.68931, "best_raw": 0.68931, "best_effective": 0.68931, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5762152, "usage": {"input": 204, "output": 63171, "cache_read": 5493010, "cache_write": 205767}, "cost_usd": 2.2451, "cost_source": "billed", "wall_clock_seconds": 6440.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1963803, "wall_clock_seconds": 120.9, "raw": 0.276714, "normalized": 0.276714, "discount": 1.0, "effective": 0.276714, "running_max": 0.276714, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.0}, {"index": 2, "tokens": 2133694, "wall_clock_seconds": 64.4, "raw": 0.40059, "normalized": 0.40059, "discount": 1.0, "effective": 0.40059, "running_max": 0.40059, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 63.8}, {"index": 3, "tokens": 2306401, "wall_clock_seconds": 84.9, "raw": 0.533833, "normalized": 0.533833, "discount": 1.0, "effective": 0.533833, "running_max": 0.533833, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 84.32}, {"index": 4, "tokens": 2582871, "wall_clock_seconds": 85.3, "raw": 0.535339, "normalized": 0.535339, "discount": 1.0, "effective": 0.535339, "running_max": 0.535339, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 84.66}, {"index": 5, "tokens": 3507661, "wall_clock_seconds": 85.0, "raw": 0.591686, "normalized": 0.591686, "discount": 1.0, "effective": 0.591686, "running_max": 0.591686, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 84.32}, {"index": 6, "tokens": 3960322, "wall_clock_seconds": 88.2, "raw": 0.654145, "normalized": 0.654145, "discount": 1.0, "effective": 0.654145, "running_max": 0.654145, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.59}, {"index": 7, "tokens": 4319842, "wall_clock_seconds": 106.5, "raw": 0.66501, "normalized": 0.66501, "discount": 1.0, "effective": 0.66501, "running_max": 0.66501, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.86}, {"index": 8, "tokens": 4954260, "wall_clock_seconds": 121.0, "raw": 0.656761, "normalized": 0.656761, "discount": 1.0, "effective": 0.656761, "running_max": 0.66501, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.0}, {"index": 9, "tokens": 5218825, "wall_clock_seconds": 104.6, "raw": 0.68931, "normalized": 0.68931, "discount": 1.0, "effective": 0.68931, "running_max": 0.68931, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.87}, {"index": 10, "tokens": 5762152, "wall_clock_seconds": 106.4, "raw": 0.688339, "normalized": 0.688339, "discount": 1.0, "effective": 0.688339, "running_max": 0.68931, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.74} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:mystery_box", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.369376, "area_term": 0.352321, "final_best": 0.437598, "best_raw": 0.667224, "best_effective": 0.437598, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7554786, "usage": {"input": 417, "output": 110075, "cache_read": 7216773, "cache_write": 227521}, "cost_usd": 15.5687, "cost_source": "billed", "wall_clock_seconds": 5992.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 403991, "wall_clock_seconds": 4.1, "raw": 0.067686, "normalized": 0.032193, "discount": 0.6, "effective": 0.019316, "running_max": 0.019316, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1100233, "wall_clock_seconds": 3.9, "raw": 0.366851, "normalized": 0.38006, "discount": 0.6, "effective": 0.228036, "running_max": 0.228036, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1473624, "wall_clock_seconds": 4.1, "raw": 0.662238, "normalized": 0.723533, "discount": 0.6, "effective": 0.43412, "running_max": 0.43412, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2332692, "wall_clock_seconds": 3.8, "raw": 0.667224, "normalized": 0.729331, "discount": 0.6, "effective": 0.437598, "running_max": 0.437598, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3226831, "wall_clock_seconds": 3.9, "raw": 0.666575, "normalized": 0.728576, "discount": 0.6, "effective": 0.437145, "running_max": 0.437598, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4661327, "wall_clock_seconds": 3.9, "raw": 0.667111, "normalized": 0.729199, "discount": 0.6, "effective": 0.43752, "running_max": 0.437598, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5651607, "wall_clock_seconds": 3.4, "raw": 0.66572, "normalized": 0.727582, "discount": 0.6, "effective": 0.436549, "running_max": 0.437598, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6102845, "wall_clock_seconds": 3.9, "raw": 0.660562, "normalized": 0.721584, "discount": 0.6, "effective": 0.43295, "running_max": 0.437598, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6253864, "wall_clock_seconds": 3.8, "raw": 0.661085, "normalized": 0.722192, "discount": 0.6, "effective": 0.433315, "running_max": 0.437598, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7554786, "wall_clock_seconds": 3.6, "raw": 0.655924, "normalized": 0.716191, "discount": 0.6, "effective": 0.429715, "running_max": 0.437598, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:mystery_box--nohints", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.029513, "area_term": 0.029513, "final_best": 0.029513, "best_raw": 0.065382, "best_effective": 0.029513, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3014407, "usage": {"input": 147, "output": 53514, "cache_read": 2804947, "cache_write": 155799}, "cost_usd": 7.4296, "cost_source": "billed", "wall_clock_seconds": 2234.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 216835, "wall_clock_seconds": 2.6, "raw": 0.057095, "normalized": 0.019878, "discount": 1.0, "effective": 0.019878, "running_max": 0.019878, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 486902, "wall_clock_seconds": 2.4, "raw": 0.065382, "normalized": 0.029513, "discount": 1.0, "effective": 0.029513, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 641224, "wall_clock_seconds": 2.6, "raw": 0.056839, "normalized": 0.01958, "discount": 1.0, "effective": 0.01958, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 809417, "wall_clock_seconds": 2.7, "raw": 0.037334, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1199206, "wall_clock_seconds": 3.2, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1501492, "wall_clock_seconds": 4.5, "raw": 0.045124, "normalized": 0.005958, "discount": 1.0, "effective": 0.005958, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1837479, "wall_clock_seconds": 2.5, "raw": 0.049105, "normalized": 0.010587, "discount": 1.0, "effective": 0.010587, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2228027, "wall_clock_seconds": 2.5, "raw": 0.045124, "normalized": 0.005958, "discount": 1.0, "effective": 0.005958, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2435685, "wall_clock_seconds": 2.9, "raw": 0.056225, "normalized": 0.018867, "discount": 1.0, "effective": 0.018867, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 3014407, "wall_clock_seconds": 4.1, "raw": 0.052697, "normalized": 0.014764, "discount": 1.0, "effective": 0.014764, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:night_school", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.746185, "area_term": 0.719937, "final_best": 0.851176, "best_raw": 0.8735, "best_effective": 0.851176, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9668456, "usage": {"input": 285, "output": 132653, "cache_read": 8930659, "cache_write": 604859}, "cost_usd": 23.1269, "cost_source": "billed", "wall_clock_seconds": 12901.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 685256, "wall_clock_seconds": 10.8, "raw": 0.5615, "normalized": 0.484118, "discount": 1.0, "effective": 0.484118, "running_max": 0.484118, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1078597, "wall_clock_seconds": 11.6, "raw": 0.65775, "normalized": 0.597353, "discount": 1.0, "effective": 0.597353, "running_max": 0.597353, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1627259, "wall_clock_seconds": 10.7, "raw": 0.755406, "normalized": 0.712243, "discount": 1.0, "effective": 0.712243, "running_max": 0.712243, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2031015, "wall_clock_seconds": 9.8, "raw": 0.8015, "normalized": 0.766471, "discount": 1.0, "effective": 0.766471, "running_max": 0.766471, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2574848, "wall_clock_seconds": 12.3, "raw": 0.8565, "normalized": 0.831176, "discount": 1.0, "effective": 0.831176, "running_max": 0.831176, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4298067, "wall_clock_seconds": 12.5, "raw": 0.869, "normalized": 0.845882, "discount": 1.0, "effective": 0.845882, "running_max": 0.845882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5119918, "wall_clock_seconds": 2.4, "raw": 0.283, "normalized": 0.156471, "discount": 1.0, "effective": 0.156471, "running_max": 0.845882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6328507, "wall_clock_seconds": 12.7, "raw": 0.8615, "normalized": 0.837059, "discount": 1.0, "effective": 0.837059, "running_max": 0.845882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8064985, "wall_clock_seconds": 12.9, "raw": 0.87, "normalized": 0.847059, "discount": 1.0, "effective": 0.847059, "running_max": 0.847059, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9668456, "wall_clock_seconds": 14.9, "raw": 0.8735, "normalized": 0.851176, "discount": 1.0, "effective": 0.851176, "running_max": 0.851176, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:reaction_rates", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.397483, "area_term": 0.359294, "final_best": 0.550241, "best_raw": 0.982573, "best_effective": 0.550241, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 23040924, "usage": {"input": 669, "output": 290182, "cache_read": 21520497, "cache_write": 1229576}, "cost_usd": 51.406, "cost_source": "billed", "wall_clock_seconds": 13820.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 2}, {"id": 4, "multiplier": 0.8, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 833322, "wall_clock_seconds": 12.5, "raw": 0.210335, "normalized": 0.210335, "discount": 1.0, "effective": 0.210335, "running_max": 0.210335, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.16}, {"index": 2, "tokens": 4137180, "wall_clock_seconds": 95.2, "raw": 0.644336, "normalized": 0.644336, "discount": 0.7, "effective": 0.451035, "running_max": 0.451035, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 94.84}, {"index": 3, "tokens": 4712349, "wall_clock_seconds": 183.0, "raw": 0.896393, "normalized": 0.896393, "discount": 0.56, "effective": 0.50198, "running_max": 0.50198, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 182.63}, {"index": 4, "tokens": 6146132, "wall_clock_seconds": 229.0, "raw": 0.920672, "normalized": 0.920672, "discount": 0.56, "effective": 0.515576, "running_max": 0.515576, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 228.64}, {"index": 5, "tokens": 10720909, "wall_clock_seconds": 238.3, "raw": 0.920815, "normalized": 0.920815, "discount": 0.56, "effective": 0.515656, "running_max": 0.515656, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 237.94}, {"index": 6, "tokens": 13264880, "wall_clock_seconds": 204.9, "raw": 0.972645, "normalized": 0.972645, "discount": 0.56, "effective": 0.544681, "running_max": 0.544681, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 204.54}, {"index": 7, "tokens": 15168356, "wall_clock_seconds": 246.1, "raw": 0.976474, "normalized": 0.976474, "discount": 0.56, "effective": 0.546826, "running_max": 0.546826, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 245.78}, {"index": 8, "tokens": 16334019, "wall_clock_seconds": 233.6, "raw": 0.981247, "normalized": 0.981247, "discount": 0.56, "effective": 0.549498, "running_max": 0.549498, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 233.25}, {"index": 9, "tokens": 21946396, "wall_clock_seconds": 311.9, "raw": 0.981657, "normalized": 0.981657, "discount": 0.56, "effective": 0.549728, "running_max": 0.549728, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 311.54}, {"index": 10, "tokens": 23040924, "wall_clock_seconds": 434.9, "raw": 0.982573, "normalized": 0.982573, "discount": 0.56, "effective": 0.550241, "running_max": 0.550241, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 434.63} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:reaction_rates--nohints", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.412512, "area_term": 0.320724, "final_best": 0.779665, "best_raw": 0.779665, "best_effective": 0.779665, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 14247955, "usage": {"input": 369, "output": 178885, "cache_read": 13470376, "cache_write": 598325}, "cost_usd": 29.8974, "cost_source": "billed", "wall_clock_seconds": 8227.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 786874, "wall_clock_seconds": 13.0, "raw": 0.064573, "normalized": 0.064573, "discount": 1.0, "effective": 0.064573, "running_max": 0.064573, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.55}, {"index": 2, "tokens": 4459520, "wall_clock_seconds": 18.5, "raw": 0.125128, "normalized": 0.125128, "discount": 1.0, "effective": 0.125128, "running_max": 0.125128, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.03}, {"index": 3, "tokens": 9036350, "wall_clock_seconds": 143.4, "raw": 0.511213, "normalized": 0.511213, "discount": 1.0, "effective": 0.511213, "running_max": 0.511213, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 142.85}, {"index": 4, "tokens": 10809551, "wall_clock_seconds": 133.1, "raw": 0.491921, "normalized": 0.491921, "discount": 1.0, "effective": 0.491921, "running_max": 0.511213, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 132.76}, {"index": 5, "tokens": 11088727, "wall_clock_seconds": 239.4, "raw": 0.750332, "normalized": 0.750332, "discount": 1.0, "effective": 0.750332, "running_max": 0.750332, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 239.05}, {"index": 6, "tokens": 11983339, "wall_clock_seconds": 254.5, "raw": 0.779665, "normalized": 0.779665, "discount": 1.0, "effective": 0.779665, "running_max": 0.779665, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 254.25}, {"index": 7, "tokens": 12140622, "wall_clock_seconds": 383.8, "raw": 0.687265, "normalized": 0.687265, "discount": 1.0, "effective": 0.687265, "running_max": 0.779665, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 383.49}, {"index": 8, "tokens": 12306294, "wall_clock_seconds": 228.0, "raw": 0.739322, "normalized": 0.739322, "discount": 1.0, "effective": 0.739322, "running_max": 0.779665, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 227.73}, {"index": 9, "tokens": 13089786, "wall_clock_seconds": 371.2, "raw": 0.778162, "normalized": 0.778162, "discount": 1.0, "effective": 0.778162, "running_max": 0.779665, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 370.74}, {"index": 10, "tokens": 14247955, "wall_clock_seconds": 274.1, "raw": 0.779465, "normalized": 0.779465, "discount": 1.0, "effective": 0.779465, "running_max": 0.779665, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 273.86} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:scan_stitch", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.355487, "area_term": 0.324028, "final_best": 0.481324, "best_raw": 0.855018, "best_effective": 0.481324, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6909258, "usage": {"input": 393, "output": 74010, "cache_read": 6615838, "cache_write": 219017}, "cost_usd": 13.058, "cost_source": "billed", "wall_clock_seconds": 3043.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 833123, "wall_clock_seconds": 0.3, "raw": 0.091473, "normalized": 0.0, "discount": 0.6375, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1866522, "wall_clock_seconds": 0.3, "raw": 0.519678, "normalized": 0.524598, "discount": 0.51, "effective": 0.267545, "running_max": 0.267545, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2301878, "wall_clock_seconds": 0.3, "raw": 0.58454, "normalized": 0.605674, "discount": 0.51, "effective": 0.308894, "running_max": 0.308894, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2771446, "wall_clock_seconds": 0.3, "raw": 0.814189, "normalized": 0.892736, "discount": 0.51, "effective": 0.455296, "running_max": 0.455296, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3610159, "wall_clock_seconds": 0.4, "raw": 0.823308, "normalized": 0.904135, "discount": 0.51, "effective": 0.461109, "running_max": 0.461109, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3829554, "wall_clock_seconds": 0.3, "raw": 0.82493, "normalized": 0.906163, "discount": 0.51, "effective": 0.462143, "running_max": 0.462143, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4642670, "wall_clock_seconds": 0.3, "raw": 0.842583, "normalized": 0.928229, "discount": 0.51, "effective": 0.473397, "running_max": 0.473397, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5135835, "wall_clock_seconds": 0.3, "raw": 0.845249, "normalized": 0.931561, "discount": 0.51, "effective": 0.475096, "running_max": 0.475096, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5929968, "wall_clock_seconds": 0.3, "raw": 0.854272, "normalized": 0.94284, "discount": 0.51, "effective": 0.480849, "running_max": 0.480849, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 6909258, "wall_clock_seconds": 0.3, "raw": 0.855018, "normalized": 0.943773, "discount": 0.51, "effective": 0.481324, "running_max": 0.481324, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:scan_stitch--nohints", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.09917, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 20892650, "usage": {"input": 582, "output": 273775, "cache_read": 19660864, "cache_write": 957429}, "cost_usd": 45.3233, "cost_source": "billed", "wall_clock_seconds": 13800.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 415940, "wall_clock_seconds": 0.3, "raw": 0.072251, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1161038, "wall_clock_seconds": 0.4, "raw": 0.09822, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2129166, "wall_clock_seconds": 0.3, "raw": 0.07092, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2726919, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4614963, "wall_clock_seconds": 0.5, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8570126, "wall_clock_seconds": 0.3, "raw": 0.094867, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 10668442, "wall_clock_seconds": 0.4, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 14595233, "wall_clock_seconds": 0.4, "raw": 0.064941, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 15941774, "wall_clock_seconds": 0.4, "raw": 0.09743, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 20892650, "wall_clock_seconds": 0.3, "raw": 0.059145, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:shapes_generalize", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.838983, "area_term": 0.830604, "final_best": 0.8725, "best_raw": 0.898, "best_effective": 0.8725, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2624372, "usage": {"input": 144, "output": 61728, "cache_read": 2467196, "cache_write": 95304}, "cost_usd": 6.7463, "cost_source": "billed", "wall_clock_seconds": 3242.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 175267, "wall_clock_seconds": 5.5, "raw": 0.184, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.92}, {"index": 2, "tokens": 237880, "wall_clock_seconds": 34.6, "raw": 0.22, "normalized": 0.025, "discount": 1.0, "effective": 0.025, "running_max": 0.025, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 33.97}, {"index": 3, "tokens": 463027, "wall_clock_seconds": 97.7, "raw": 0.25, "normalized": 0.0625, "discount": 1.0, "effective": 0.0625, "running_max": 0.0625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 97.0}, {"index": 4, "tokens": 547787, "wall_clock_seconds": 96.1, "raw": 0.538, "normalized": 0.4225, "discount": 1.0, "effective": 0.4225, "running_max": 0.4225, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.42}, {"index": 5, "tokens": 637477, "wall_clock_seconds": 96.7, "raw": 0.861, "normalized": 0.82625, "discount": 1.0, "effective": 0.82625, "running_max": 0.82625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.09}, {"index": 6, "tokens": 1231858, "wall_clock_seconds": 101.9, "raw": 0.872, "normalized": 0.84, "discount": 1.0, "effective": 0.84, "running_max": 0.84, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.15}, {"index": 7, "tokens": 1661906, "wall_clock_seconds": 108.5, "raw": 0.869, "normalized": 0.83625, "discount": 1.0, "effective": 0.83625, "running_max": 0.84, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.94}, {"index": 8, "tokens": 1985285, "wall_clock_seconds": 113.5, "raw": 0.866, "normalized": 0.8325, "discount": 1.0, "effective": 0.8325, "running_max": 0.84, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 112.93}, {"index": 9, "tokens": 2246058, "wall_clock_seconds": 105.2, "raw": 0.894, "normalized": 0.8675, "discount": 1.0, "effective": 0.8675, "running_max": 0.8675, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 104.59}, {"index": 10, "tokens": 2624372, "wall_clock_seconds": 113.0, "raw": 0.898, "normalized": 0.8725, "discount": 1.0, "effective": 0.8725, "running_max": 0.8725, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 112.44} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:shattered_prior", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.491526, "area_term": 0.389408, "final_best": 0.9, "best_raw": 1.0, "best_effective": 0.9, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 8942057, "usage": {"input": 616, "output": 90538, "cache_read": 8600838, "cache_write": 250065}, "cost_usd": 16.2597, "cost_source": "billed", "wall_clock_seconds": 2857.5, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3943896, "wall_clock_seconds": 1.8, "raw": 0.13864, "normalized": 0.13864, "discount": 0.9, "effective": 0.124776, "running_max": 0.124776, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6968451, "wall_clock_seconds": 1.7, "raw": 0.651398, "normalized": 0.651398, "discount": 0.9, "effective": 0.586259, "running_max": 0.586259, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8206393, "wall_clock_seconds": 1.8, "raw": 1.0, "normalized": 1.0, "discount": 0.9, "effective": 0.9, "running_max": 0.9, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:shattered_prior--nohints", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.670103, "area_term": 0.587629, "final_best": 1.0, "best_raw": 1.0, "best_effective": 1.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 5827839, "usage": {"input": 204, "output": 70371, "cache_read": 5609796, "cache_write": 147468}, "cost_usd": 10.9737, "cost_source": "billed", "wall_clock_seconds": 2349.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1718664, "wall_clock_seconds": 1.8, "raw": 0.158117, "normalized": 0.158117, "discount": 1.0, "effective": 0.158117, "running_max": 0.158117, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2973358, "wall_clock_seconds": 2.1, "raw": 0.494334, "normalized": 0.494334, "discount": 1.0, "effective": 0.494334, "running_max": 0.494334, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4170940, "wall_clock_seconds": 1.7, "raw": 0.876369, "normalized": 0.876369, "discount": 1.0, "effective": 0.876369, "running_max": 0.876369, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 5391212, "wall_clock_seconds": 1.7, "raw": 1.0, "normalized": 1.0, "discount": 1.0, "effective": 1.0, "running_max": 1.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:ship_detect", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.159987, "area_term": 0.143303, "final_best": 0.226722, "best_raw": 0.226722, "best_effective": 0.226722, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 102, "max_submissions": null, "tokens_total": 50245774, "usage": {"input": 1236, "output": 482836, "cache_read": 47407961, "cache_write": 2353741}, "cost_usd": 100.9839, "cost_source": "billed", "wall_clock_seconds": 33696.7, "stop_reason": "cost_limit", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 12, "tokens": 227523, "cost_weighted": 16408.9, "wall_clock_seconds": 328.1, "raw": 0.001577, "normalized": 0.001577, "effective": 0.001577, "running_max": 0.001577, "scored": true, "final": false}, {"index": 2, "model_call": 14, "tokens": 297030, "cost_weighted": 20838.0, "wall_clock_seconds": 382.0, "raw": 0.00255, "normalized": 0.00255, "effective": 0.00255, "running_max": 0.00255, "scored": true, "final": false}, {"index": 3, "model_call": 17, "tokens": 429565, "cost_weighted": 31872.3, "wall_clock_seconds": 510.7, "raw": 0.003801, "normalized": 0.003801, "effective": 0.003801, "running_max": 0.003801, "scored": true, "final": false}, {"index": 4, "model_call": 23, "tokens": 727446, "cost_weighted": 39590.0, "wall_clock_seconds": 1135.3, "raw": 0.102466, "normalized": 0.102466, "effective": 0.102466, "running_max": 0.102466, "scored": true, "final": false}, {"index": 5, "model_call": 29, "tokens": 1078139, "cost_weighted": 53901.2, "wall_clock_seconds": 1304.3, "raw": 0.103039, "normalized": 0.103039, "effective": 0.103039, "running_max": 0.103039, "scored": true, "final": false}, {"index": 6, "model_call": 32, "tokens": 1291394, "cost_weighted": 63134.9, "wall_clock_seconds": 1416.0, "raw": 0.103516, "normalized": 0.103516, "effective": 0.103516, "running_max": 0.103516, "scored": true, "final": false}, {"index": 7, "model_call": 34, "tokens": 1446177, "cost_weighted": 69556.7, "wall_clock_seconds": 1489.2, "raw": 0.104183, "normalized": 0.104183, "effective": 0.104183, "running_max": 0.104183, "scored": true, "final": false}, {"index": 8, "model_call": 35, "tokens": 1527120, "cost_weighted": 72124.6, "wall_clock_seconds": 1525.6, "raw": 0.087936, "normalized": 0.087936, "effective": 0.087936, "running_max": 0.104183, "scored": true, "final": false}, {"index": 9, "model_call": 37, "tokens": 1701154, "cost_weighted": 78969.2, "wall_clock_seconds": 1594.5, "raw": 0.088887, "normalized": 0.088887, "effective": 0.088887, "running_max": 0.104183, "scored": true, "final": false}, {"index": 10, "model_call": 39, "tokens": 1884989, "cost_weighted": 86796.4, "wall_clock_seconds": 1688.0, "raw": 0.089572, "normalized": 0.089572, "effective": 0.089572, "running_max": 0.104183, "scored": true, "final": false}, {"index": 11, "model_call": 42, "tokens": 2178177, "cost_weighted": 96973.9, "wall_clock_seconds": 1796.4, "raw": 0.089744, "normalized": 0.089744, "effective": 0.089744, "running_max": 0.104183, "scored": true, "final": false}, {"index": 12, "model_call": 43, "tokens": 2280374, "cost_weighted": 99820.5, "wall_clock_seconds": 2102.2, "raw": 0.124356, "normalized": 0.124356, "effective": 0.124356, "running_max": 0.124356, "scored": true, "final": false}, {"index": 13, "model_call": 47, "tokens": 2716392, "cost_weighted": 115881.6, "wall_clock_seconds": 2322.8, "raw": 0.133094, "normalized": 0.133094, "effective": 0.133094, "running_max": 0.133094, "scored": true, "final": false}, {"index": 14, "model_call": 52, "tokens": 3324121, "cost_weighted": 143942.2, "wall_clock_seconds": 2671.2, "raw": 0.134637, "normalized": 0.134637, "effective": 0.134637, "running_max": 0.134637, "scored": true, "final": false}, {"index": 15, "model_call": 55, "tokens": 3731159, "cost_weighted": 154789.6, "wall_clock_seconds": 4228.1, "raw": 0.161605, "normalized": 0.161605, "effective": 0.161605, "running_max": 0.161605, "scored": true, "final": false}, {"index": 16, "model_call": 59, "tokens": 4299264, "cost_weighted": 174982.5, "wall_clock_seconds": 4514.3, "raw": 0.168887, "normalized": 0.168887, "effective": 0.168887, "running_max": 0.168887, "scored": true, "final": false}, {"index": 17, "model_call": 65, "tokens": 5249407, "cost_weighted": 210667.5, "wall_clock_seconds": 4968.6, "raw": 0.169572, "normalized": 0.169572, "effective": 0.169572, "running_max": 0.169572, "scored": true, "final": false}, {"index": 18, "model_call": 68, "tokens": 5780520, "cost_weighted": 233151.0, "wall_clock_seconds": 5373.8, "raw": 0.169921, "normalized": 0.169921, "effective": 0.169921, "running_max": 0.169921, "scored": true, "final": false}, {"index": 19, "model_call": 73, "tokens": 6738009, "cost_weighted": 268926.8, "wall_clock_seconds": 5814.7, "raw": 0.167296, "normalized": 0.167296, "effective": 0.167296, "running_max": 0.169921, "scored": true, "final": false}, {"index": 20, "model_call": 74, "tokens": 6936271, "cost_weighted": 274568.8, "wall_clock_seconds": 5882.7, "raw": 0.169921, "normalized": 0.169921, "effective": 0.169921, "running_max": 0.169921, "scored": true, "final": false}, {"index": 21, "model_call": 75, "tokens": 7134737, "cost_weighted": 279003.7, "wall_clock_seconds": 7617.6, "raw": 0.15814, "normalized": 0.15814, "effective": 0.15814, "running_max": 0.169921, "scored": true, "final": false}, {"index": 22, "model_call": 76, "tokens": 7338775, "cost_weighted": 287289.7, "wall_clock_seconds": 7760.9, "raw": 0.15814, "normalized": 0.15814, "effective": 0.15814, "running_max": 0.169921, "scored": true, "final": false}, {"index": 23, "model_call": 78, "tokens": 7759298, "cost_weighted": 305384.9, "wall_clock_seconds": 8065.4, "raw": 0.194553, "normalized": 0.194553, "effective": 0.194553, "running_max": 0.194553, "scored": true, "final": false}, {"index": 24, "model_call": 80, "tokens": 8192230, "cost_weighted": 320470.0, "wall_clock_seconds": 8281.6, "raw": 0.194553, "normalized": 0.194553, "effective": 0.194553, "running_max": 0.194553, "scored": true, "final": false}, {"index": 25, "model_call": 83, "tokens": 8884789, "cost_weighted": 350876.0, "wall_clock_seconds": 8790.3, "raw": 0.200771, "normalized": 0.200771, "effective": 0.200771, "running_max": 0.200771, "scored": true, "final": false}, {"index": 26, "model_call": 116, "tokens": 11103480, "cost_weighted": 494905.3, "wall_clock_seconds": 9876.5, "raw": 0.200856, "normalized": 0.200856, "effective": 0.200856, "running_max": 0.200856, "scored": true, "final": false}, {"index": 27, "model_call": 131, "tokens": 12551126, "cost_weighted": 548339.2, "wall_clock_seconds": 10681.5, "raw": 0.200918, "normalized": 0.200918, "effective": 0.200918, "running_max": 0.200918, "scored": true, "final": false}, {"index": 28, "model_call": 134, "tokens": 12889477, "cost_weighted": 558919.6, "wall_clock_seconds": 10796.3, "raw": 0.200835, "normalized": 0.200835, "effective": 0.200835, "running_max": 0.200918, "scored": true, "final": false}, {"index": 29, "model_call": 135, "tokens": 13005082, "cost_weighted": 561749.8, "wall_clock_seconds": 10811.7, "raw": 0.200835, "normalized": 0.200835, "effective": 0.200835, "running_max": 0.200918, "scored": true, "final": false}, {"index": 30, "model_call": 136, "tokens": 13124407, "cost_weighted": 566738.7, "wall_clock_seconds": 10896.9, "raw": 0.200959, "normalized": 0.200959, "effective": 0.200959, "running_max": 0.200959, "scored": true, "final": false}, {"index": 31, "model_call": 138, "tokens": 13366969, "cost_weighted": 574327.2, "wall_clock_seconds": 10982.9, "raw": 0.20092, "normalized": 0.20092, "effective": 0.20092, "running_max": 0.200959, "scored": true, "final": false}, {"index": 32, "model_call": 140, "tokens": 13616071, "cost_weighted": 581752.4, "wall_clock_seconds": 11063.7, "raw": 0.200989, "normalized": 0.200989, "effective": 0.200989, "running_max": 0.200989, "scored": true, "final": false}, {"index": 33, "model_call": 142, "tokens": 13872482, "cost_weighted": 589918.5, "wall_clock_seconds": 11159.7, "raw": 0.200934, "normalized": 0.200934, "effective": 0.200934, "running_max": 0.200989, "scored": true, "final": false}, {"index": 34, "model_call": 146, "tokens": 14407696, "cost_weighted": 606717.4, "wall_clock_seconds": 13426.1, "raw": 0.204447, "normalized": 0.204447, "effective": 0.204447, "running_max": 0.204447, "scored": true, "final": false}, {"index": 35, "model_call": 148, "tokens": 14701265, "cost_weighted": 654643.4, "wall_clock_seconds": 13727.4, "raw": 0.204482, "normalized": 0.204482, "effective": 0.204482, "running_max": 0.204482, "scored": true, "final": false}, {"index": 36, "model_call": 152, "tokens": 15321689, "cost_weighted": 676261.8, "wall_clock_seconds": 13997.4, "raw": 0.204482, "normalized": 0.204482, "effective": 0.204482, "running_max": 0.204482, "scored": true, "final": false}, {"index": 37, "model_call": 165, "tokens": 17686666, "cost_weighted": 753928.3, "wall_clock_seconds": 14906.0, "raw": 0.204234, "normalized": 0.204234, "effective": 0.204234, "running_max": 0.204482, "scored": true, "final": false}, {"index": 38, "model_call": 169, "tokens": 18503302, "cost_weighted": 776354.4, "wall_clock_seconds": 15123.7, "raw": 0.204234, "normalized": 0.204234, "effective": 0.204234, "running_max": 0.204482, "scored": true, "final": false}, {"index": 39, "model_call": 171, "tokens": 18930052, "cost_weighted": 789434.0, "wall_clock_seconds": 16726.3, "raw": 0.20461, "normalized": 0.20461, "effective": 0.20461, "running_max": 0.20461, "scored": true, "final": false}, {"index": 40, "model_call": 174, "tokens": 19596839, "cost_weighted": 811559.4, "wall_clock_seconds": 17001.7, "raw": 0.213424, "normalized": 0.213424, "effective": 0.213424, "running_max": 0.213424, "scored": true, "final": false}, {"index": 41, "model_call": 176, "tokens": 20055010, "cost_weighted": 825367.6, "wall_clock_seconds": 17151.1, "raw": 0.21194, "normalized": 0.21194, "effective": 0.21194, "running_max": 0.213424, "scored": true, "final": false}, {"index": 42, "model_call": 180, "tokens": 21005932, "cost_weighted": 854321.1, "wall_clock_seconds": 17459.7, "raw": 0.211886, "normalized": 0.211886, "effective": 0.211886, "running_max": 0.213424, "scored": true, "final": false}, {"index": 43, "model_call": 181, "tokens": 21253452, "cost_weighted": 864456.6, "wall_clock_seconds": 17664.8, "raw": 0.211886, "normalized": 0.211886, "effective": 0.211886, "running_max": 0.213424, "scored": true, "final": false}, {"index": 44, "model_call": 193, "tokens": 21910140, "cost_weighted": 952562.1, "wall_clock_seconds": 17947.1, "raw": 0.19482, "normalized": 0.19482, "effective": 0.19482, "running_max": 0.213424, "scored": true, "final": false}, {"index": 45, "model_call": 201, "tokens": 22424802, "cost_weighted": 971826.5, "wall_clock_seconds": 18157.3, "raw": 0.19263, "normalized": 0.19263, "effective": 0.19263, "running_max": 0.213424, "scored": true, "final": false}, {"index": 46, "model_call": 202, "tokens": 22495487, "cost_weighted": 973882.8, "wall_clock_seconds": 18176.8, "raw": 0.194744, "normalized": 0.194744, "effective": 0.194744, "running_max": 0.213424, "scored": true, "final": false}, {"index": 47, "model_call": 207, "tokens": 22861695, "cost_weighted": 984469.2, "wall_clock_seconds": 18278.6, "raw": 0.204606, "normalized": 0.204606, "effective": 0.204606, "running_max": 0.213424, "scored": true, "final": false}, {"index": 48, "model_call": 208, "tokens": 22939708, "cost_weighted": 986584.0, "wall_clock_seconds": 18297.4, "raw": 0.20375, "normalized": 0.20375, "effective": 0.20375, "running_max": 0.213424, "scored": true, "final": false}, {"index": 49, "model_call": 209, "tokens": 23018297, "cost_weighted": 988400.6, "wall_clock_seconds": 18308.6, "raw": 0.204583, "normalized": 0.204583, "effective": 0.204583, "running_max": 0.213424, "scored": true, "final": false}, {"index": 50, "model_call": 213, "tokens": 23347694, "cost_weighted": 997663.6, "wall_clock_seconds": 18387.2, "raw": 0.204605, "normalized": 0.204605, "effective": 0.204605, "running_max": 0.213424, "scored": true, "final": false}, {"index": 51, "model_call": 216, "tokens": 23616942, "cost_weighted": 1006728.2, "wall_clock_seconds": 18459.9, "raw": 0.204662, "normalized": 0.204662, "effective": 0.204662, "running_max": 0.213424, "scored": true, "final": false}, {"index": 52, "model_call": 218, "tokens": 23805565, "cost_weighted": 1012494.4, "wall_clock_seconds": 18525.1, "raw": 0.20471, "normalized": 0.20471, "effective": 0.20471, "running_max": 0.213424, "scored": true, "final": false}, {"index": 53, "model_call": 222, "tokens": 24203565, "cost_weighted": 1024271.7, "wall_clock_seconds": 18622.0, "raw": 0.204721, "normalized": 0.204721, "effective": 0.204721, "running_max": 0.213424, "scored": true, "final": false}, {"index": 54, "model_call": 225, "tokens": 24523350, "cost_weighted": 1033555.9, "wall_clock_seconds": 18693.8, "raw": 0.204794, "normalized": 0.204794, "effective": 0.204794, "running_max": 0.213424, "scored": true, "final": false}, {"index": 55, "model_call": 228, "tokens": 24861870, "cost_weighted": 1043185.4, "wall_clock_seconds": 18771.8, "raw": 0.204865, "normalized": 0.204865, "effective": 0.204865, "running_max": 0.213424, "scored": true, "final": false}, {"index": 56, "model_call": 231, "tokens": 25217444, "cost_weighted": 1052434.3, "wall_clock_seconds": 18825.2, "raw": 0.204921, "normalized": 0.204921, "effective": 0.204921, "running_max": 0.213424, "scored": true, "final": false}, {"index": 57, "model_call": 239, "tokens": 26265051, "cost_weighted": 1084468.8, "wall_clock_seconds": 19144.2, "raw": 0.204169, "normalized": 0.204169, "effective": 0.204169, "running_max": 0.213424, "scored": true, "final": false}, {"index": 58, "model_call": 240, "tokens": 26409972, "cost_weighted": 1091279.0, "wall_clock_seconds": 19252.8, "raw": 0.204145, "normalized": 0.204145, "effective": 0.204145, "running_max": 0.213424, "scored": true, "final": false}, {"index": 59, "model_call": 243, "tokens": 26860183, "cost_weighted": 1107085.7, "wall_clock_seconds": 19433.4, "raw": 0.208286, "normalized": 0.208286, "effective": 0.208286, "running_max": 0.213424, "scored": true, "final": false}, {"index": 60, "model_call": 244, "tokens": 27018455, "cost_weighted": 1113323.5, "wall_clock_seconds": 19499.0, "raw": 0.207894, "normalized": 0.207894, "effective": 0.207894, "running_max": 0.213424, "scored": true, "final": false}, {"index": 61, "model_call": 252, "tokens": 28328807, "cost_weighted": 1147465.7, "wall_clock_seconds": 19777.2, "raw": 0.20839, "normalized": 0.20839, "effective": 0.20839, "running_max": 0.213424, "scored": true, "final": false}, {"index": 62, "model_call": 262, "tokens": 30071639, "cost_weighted": 1189116.5, "wall_clock_seconds": 20051.7, "raw": 0.20885, "normalized": 0.20885, "effective": 0.20885, "running_max": 0.213424, "scored": true, "final": false}, {"index": 63, "model_call": 265, "tokens": 30615239, "cost_weighted": 1203031.0, "wall_clock_seconds": 20165.6, "raw": 0.20885, "normalized": 0.20885, "effective": 0.20885, "running_max": 0.213424, "scored": true, "final": false}, {"index": 64, "model_call": 270, "tokens": 31560105, "cost_weighted": 1226953.7, "wall_clock_seconds": 20340.7, "raw": 0.20969, "normalized": 0.20969, "effective": 0.20969, "running_max": 0.213424, "scored": true, "final": false}, {"index": 65, "model_call": 281, "tokens": 33836382, "cost_weighted": 1294834.8, "wall_clock_seconds": 21061.0, "raw": 0.207441, "normalized": 0.207441, "effective": 0.207441, "running_max": 0.213424, "scored": true, "final": false}, {"index": 66, "model_call": 284, "tokens": 34504120, "cost_weighted": 1316981.7, "wall_clock_seconds": 21389.4, "raw": 0.212813, "normalized": 0.212813, "effective": 0.212813, "running_max": 0.213424, "scored": true, "final": false}, {"index": 67, "model_call": 285, "tokens": 34729810, "cost_weighted": 1321950.0, "wall_clock_seconds": 21819.2, "raw": 0.212813, "normalized": 0.212813, "effective": 0.212813, "running_max": 0.213424, "scored": true, "final": false}, {"index": 68, "model_call": 289, "tokens": 35664240, "cost_weighted": 1350710.9, "wall_clock_seconds": 22350.2, "raw": 0.213019, "normalized": 0.213019, "effective": 0.213019, "running_max": 0.213424, "scored": true, "final": false}, {"index": 69, "model_call": 290, "tokens": 35904041, "cost_weighted": 1358387.8, "wall_clock_seconds": 22445.8, "raw": 0.213019, "normalized": 0.213019, "effective": 0.213019, "running_max": 0.213424, "scored": true, "final": false}, {"index": 70, "model_call": 292, "tokens": 36391539, "cost_weighted": 1373661.8, "wall_clock_seconds": 22628.8, "raw": 0.213078, "normalized": 0.213078, "effective": 0.213078, "running_max": 0.213424, "scored": true, "final": false}, {"index": 71, "model_call": 293, "tokens": 36637927, "cost_weighted": 1438140.2, "wall_clock_seconds": 22736.7, "raw": 0.213078, "normalized": 0.213078, "effective": 0.213078, "running_max": 0.213424, "scored": true, "final": false}, {"index": 72, "model_call": 301, "tokens": 36914745, "cost_weighted": 1459686.8, "wall_clock_seconds": 22899.6, "raw": 0.213194, "normalized": 0.213194, "effective": 0.213194, "running_max": 0.213424, "scored": true, "final": false}, {"index": 73, "model_call": 305, "tokens": 37165403, "cost_weighted": 1471165.1, "wall_clock_seconds": 23030.8, "raw": 0.213337, "normalized": 0.213337, "effective": 0.213337, "running_max": 0.213424, "scored": true, "final": false}, {"index": 74, "model_call": 308, "tokens": 37379904, "cost_weighted": 1478823.5, "wall_clock_seconds": 23103.8, "raw": 0.213283, "normalized": 0.213283, "effective": 0.213283, "running_max": 0.213424, "scored": true, "final": false}, {"index": 75, "model_call": 310, "tokens": 37539530, "cost_weighted": 1486308.7, "wall_clock_seconds": 23204.6, "raw": 0.213369, "normalized": 0.213369, "effective": 0.213369, "running_max": 0.213424, "scored": true, "final": false}, {"index": 76, "model_call": 312, "tokens": 37714664, "cost_weighted": 1494032.6, "wall_clock_seconds": 23305.2, "raw": 0.21339, "normalized": 0.21339, "effective": 0.21339, "running_max": 0.213424, "scored": true, "final": false}, {"index": 77, "model_call": 314, "tokens": 37906269, "cost_weighted": 1502981.8, "wall_clock_seconds": 23431.2, "raw": 0.213563, "normalized": 0.213563, "effective": 0.213563, "running_max": 0.213563, "scored": true, "final": false}, {"index": 78, "model_call": 318, "tokens": 38327573, "cost_weighted": 1520439.6, "wall_clock_seconds": 23608.8, "raw": 0.213194, "normalized": 0.213194, "effective": 0.213194, "running_max": 0.213563, "scored": true, "final": false}, {"index": 79, "model_call": 319, "tokens": 38443322, "cost_weighted": 1528267.9, "wall_clock_seconds": 23743.7, "raw": 0.213194, "normalized": 0.213194, "effective": 0.213194, "running_max": 0.213563, "scored": true, "final": false}, {"index": 80, "model_call": 320, "tokens": 38560602, "cost_weighted": 1532865.2, "wall_clock_seconds": 23801.7, "raw": 0.220754, "normalized": 0.220754, "effective": 0.220754, "running_max": 0.220754, "scored": true, "final": false}, {"index": 81, "model_call": 321, "tokens": 38680611, "cost_weighted": 1536572.6, "wall_clock_seconds": 23827.1, "raw": 0.216864, "normalized": 0.216864, "effective": 0.216864, "running_max": 0.220754, "scored": true, "final": false}, {"index": 82, "model_call": 323, "tokens": 38931548, "cost_weighted": 1545066.6, "wall_clock_seconds": 24788.3, "raw": 0.212064, "normalized": 0.212064, "effective": 0.212064, "running_max": 0.220754, "scored": true, "final": false}, {"index": 83, "model_call": 327, "tokens": 39472886, "cost_weighted": 1566244.7, "wall_clock_seconds": 25065.2, "raw": 0.212064, "normalized": 0.212064, "effective": 0.212064, "running_max": 0.220754, "scored": true, "final": false}, {"index": 84, "model_call": 331, "tokens": 40076841, "cost_weighted": 1587692.8, "wall_clock_seconds": 25259.8, "raw": 0.212137, "normalized": 0.212137, "effective": 0.212137, "running_max": 0.220754, "scored": true, "final": false}, {"index": 85, "model_call": 334, "tokens": 40583405, "cost_weighted": 1610743.6, "wall_clock_seconds": 25597.4, "raw": 0.212137, "normalized": 0.212137, "effective": 0.212137, "running_max": 0.220754, "scored": true, "final": false}, {"index": 86, "model_call": 335, "tokens": 40758380, "cost_weighted": 1615663.9, "wall_clock_seconds": 26787.8, "raw": 0.215272, "normalized": 0.215272, "effective": 0.215272, "running_max": 0.220754, "scored": true, "final": false}, {"index": 87, "model_call": 341, "tokens": 41879263, "cost_weighted": 1650958.5, "wall_clock_seconds": 27134.5, "raw": 0.21524, "normalized": 0.21524, "effective": 0.21524, "running_max": 0.220754, "scored": true, "final": false}, {"index": 88, "model_call": 344, "tokens": 42489404, "cost_weighted": 1673599.7, "wall_clock_seconds": 27403.0, "raw": 0.21524, "normalized": 0.21524, "effective": 0.21524, "running_max": 0.220754, "scored": true, "final": false}, {"index": 89, "model_call": 350, "tokens": 43780808, "cost_weighted": 1712416.6, "wall_clock_seconds": 27722.9, "raw": 0.215384, "normalized": 0.215384, "effective": 0.215384, "running_max": 0.220754, "scored": true, "final": false}, {"index": 90, "model_call": 352, "tokens": 44238837, "cost_weighted": 1725816.3, "wall_clock_seconds": 27822.7, "raw": 0.21565, "normalized": 0.21565, "effective": 0.21565, "running_max": 0.220754, "scored": true, "final": false}, {"index": 91, "model_call": 355, "tokens": 44950945, "cost_weighted": 1749533.1, "wall_clock_seconds": 28069.6, "raw": 0.215673, "normalized": 0.215673, "effective": 0.215673, "running_max": 0.220754, "scored": true, "final": false}, {"index": 92, "model_call": 362, "tokens": 45326184, "cost_weighted": 1823278.1, "wall_clock_seconds": 28220.2, "raw": 0.216157, "normalized": 0.216157, "effective": 0.216157, "running_max": 0.220754, "scored": true, "final": false}, {"index": 93, "model_call": 365, "tokens": 45420424, "cost_weighted": 1827665.6, "wall_clock_seconds": 28262.0, "raw": 0.216342, "normalized": 0.216342, "effective": 0.216342, "running_max": 0.220754, "scored": true, "final": false}, {"index": 94, "model_call": 369, "tokens": 45573127, "cost_weighted": 1834612.0, "wall_clock_seconds": 28323.6, "raw": 0.216664, "normalized": 0.216664, "effective": 0.216664, "running_max": 0.220754, "scored": true, "final": false}, {"index": 95, "model_call": 383, "tokens": 46542572, "cost_weighted": 1870641.5, "wall_clock_seconds": 28586.0, "raw": 0.216675, "normalized": 0.216675, "effective": 0.216675, "running_max": 0.220754, "scored": true, "final": false}, {"index": 96, "model_call": 386, "tokens": 46813246, "cost_weighted": 1881422.9, "wall_clock_seconds": 28701.2, "raw": 0.216675, "normalized": 0.216675, "effective": 0.216675, "running_max": 0.220754, "scored": true, "final": false}, {"index": 97, "model_call": 401, "tokens": 48544832, "cost_weighted": 1939601.2, "wall_clock_seconds": 30057.2, "raw": 0.217173, "normalized": 0.217173, "effective": 0.217173, "running_max": 0.220754, "scored": true, "final": false}, {"index": 98, "model_call": 404, "tokens": 48944945, "cost_weighted": 1950317.5, "wall_clock_seconds": 30118.9, "raw": 0.217386, "normalized": 0.217386, "effective": 0.217386, "running_max": 0.220754, "scored": true, "final": false}, {"index": 99, "model_call": 408, "tokens": 49505276, "cost_weighted": 1965276.3, "wall_clock_seconds": 30211.7, "raw": 0.217714, "normalized": 0.217714, "effective": 0.217714, "running_max": 0.220754, "scored": true, "final": false}, {"index": 100, "model_call": 411, "tokens": 49946927, "cost_weighted": 1978946.4, "wall_clock_seconds": 30323.1, "raw": 0.217714, "normalized": 0.217714, "effective": 0.217714, "running_max": 0.220754, "scored": true, "final": false}, {"index": 101, "model_call": 412, "tokens": 50096249, "cost_weighted": 1982181.1, "wall_clock_seconds": 33634.6, "raw": 0.226722, "normalized": 0.226722, "effective": 0.226722, "running_max": 0.226722, "scored": true, "final": false}, {"index": 102, "model_call": 413, "tokens": 50245774, "cost_weighted": 2019677.7, "wall_clock_seconds": 33650.1, "raw": 0.221968, "normalized": 0.221968, "effective": 0.221968, "running_max": 0.226722, "scored": true, "final": false} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:ship_tune", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.209649, "area_term": 0.185626, "final_best": 0.305739, "best_raw": 0.305739, "best_effective": 0.305739, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13410789, "usage": {"input": 109303, "output": 134434, "cache_read": 12677584, "cache_write": 489468}, "cost_usd": 26.6107, "cost_source": "billed", "wall_clock_seconds": 10267.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2099754, "wall_clock_seconds": 60.6, "raw": 0.188602, "normalized": 0.188602, "discount": 1.0, "effective": 0.188602, "running_max": 0.188602, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 59.82}, {"index": 2, "tokens": 3501205, "wall_clock_seconds": 103.1, "raw": 0.23006, "normalized": 0.23006, "discount": 1.0, "effective": 0.23006, "running_max": 0.23006, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 102.43}, {"index": 3, "tokens": 5069058, "wall_clock_seconds": 108.3, "raw": 0.269197, "normalized": 0.269197, "discount": 1.0, "effective": 0.269197, "running_max": 0.269197, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.63}, {"index": 4, "tokens": 8925292, "wall_clock_seconds": 180.1, "raw": 0.285032, "normalized": 0.285032, "discount": 1.0, "effective": 0.285032, "running_max": 0.285032, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 179.36}, {"index": 5, "tokens": 10958237, "wall_clock_seconds": 197.1, "raw": 0.28829, "normalized": 0.28829, "discount": 1.0, "effective": 0.28829, "running_max": 0.28829, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 196.47}, {"index": 6, "tokens": 11426878, "wall_clock_seconds": 611.4, "raw": 0.293395, "normalized": 0.293395, "discount": 1.0, "effective": 0.293395, "running_max": 0.293395, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 610.6}, {"index": 7, "tokens": 11903788, "wall_clock_seconds": 204.9, "raw": 0.301144, "normalized": 0.301144, "discount": 1.0, "effective": 0.301144, "running_max": 0.301144, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 204.11}, {"index": 8, "tokens": 12387730, "wall_clock_seconds": 199.1, "raw": 0.29923, "normalized": 0.29923, "discount": 1.0, "effective": 0.29923, "running_max": 0.301144, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 198.44}, {"index": 9, "tokens": 12874995, "wall_clock_seconds": 619.5, "raw": 0.305739, "normalized": 0.305739, "discount": 1.0, "effective": 0.305739, "running_max": 0.305739, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 618.67}, {"index": 10, "tokens": 13410789, "wall_clock_seconds": 758.4, "raw": 0.303272, "normalized": 0.303272, "discount": 1.0, "effective": 0.303272, "running_max": 0.305739, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 757.6} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:splash_generalize", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.729867, "area_term": 0.693723, "final_best": 0.874444, "best_raw": 0.887, "best_effective": 0.874444, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7758372, "usage": {"input": 276, "output": 105803, "cache_read": 7484520, "cache_write": 167773}, "cost_usd": 14.8746, "cost_source": "billed", "wall_clock_seconds": 4899.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 807467, "wall_clock_seconds": 94.8, "raw": 0.557, "normalized": 0.507778, "discount": 1.0, "effective": 0.507778, "running_max": 0.507778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.05}, {"index": 2, "tokens": 1016038, "wall_clock_seconds": 94.3, "raw": 0.584, "normalized": 0.537778, "discount": 1.0, "effective": 0.537778, "running_max": 0.537778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 93.81}, {"index": 3, "tokens": 1242626, "wall_clock_seconds": 94.8, "raw": 0.564, "normalized": 0.515556, "discount": 1.0, "effective": 0.515556, "running_max": 0.537778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.15}, {"index": 4, "tokens": 1488193, "wall_clock_seconds": 95.4, "raw": 0.672, "normalized": 0.635556, "discount": 1.0, "effective": 0.635556, "running_max": 0.635556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.66}, {"index": 5, "tokens": 2270765, "wall_clock_seconds": 99.7, "raw": 0.726, "normalized": 0.695556, "discount": 1.0, "effective": 0.695556, "running_max": 0.695556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.11}, {"index": 6, "tokens": 2957599, "wall_clock_seconds": 96.1, "raw": 0.77, "normalized": 0.744444, "discount": 1.0, "effective": 0.744444, "running_max": 0.744444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.53}, {"index": 7, "tokens": 3771496, "wall_clock_seconds": 101.2, "raw": 0.82, "normalized": 0.8, "discount": 1.0, "effective": 0.8, "running_max": 0.8, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.53}, {"index": 8, "tokens": 4608770, "wall_clock_seconds": 103.7, "raw": 0.875, "normalized": 0.861111, "discount": 1.0, "effective": 0.861111, "running_max": 0.861111, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 102.41}, {"index": 9, "tokens": 5272932, "wall_clock_seconds": 104.0, "raw": 0.878, "normalized": 0.864444, "discount": 1.0, "effective": 0.864444, "running_max": 0.864444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.38}, {"index": 10, "tokens": 7758372, "wall_clock_seconds": 112.1, "raw": 0.887, "normalized": 0.874444, "discount": 1.0, "effective": 0.874444, "running_max": 0.874444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 111.16} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:tod_pipeline--nohints", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.155189, "area_term": 0.149203, "final_best": 0.179133, "best_raw": 0.179133, "best_effective": 0.179133, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5219084, "usage": {"input": 198, "output": 89777, "cache_read": 4981116, "cache_write": 147993}, "cost_usd": 11.3219, "cost_source": "billed", "wall_clock_seconds": 4288.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 522292, "wall_clock_seconds": 37.3, "raw": 0.082513, "normalized": 0.082513, "discount": 1.0, "effective": 0.082513, "running_max": 0.082513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 37.01}, {"index": 2, "tokens": 1267262, "wall_clock_seconds": 45.2, "raw": 0.122961, "normalized": 0.122961, "discount": 1.0, "effective": 0.122961, "running_max": 0.122961, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.84}, {"index": 3, "tokens": 1657236, "wall_clock_seconds": 82.2, "raw": 0.134619, "normalized": 0.134619, "discount": 1.0, "effective": 0.134619, "running_max": 0.134619, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 81.89}, {"index": 4, "tokens": 2274753, "wall_clock_seconds": 75.9, "raw": 0.114329, "normalized": 0.114329, "discount": 1.0, "effective": 0.114329, "running_max": 0.134619, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.62}, {"index": 5, "tokens": 3168779, "wall_clock_seconds": 76.4, "raw": 0.177766, "normalized": 0.177766, "discount": 1.0, "effective": 0.177766, "running_max": 0.177766, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.89}, {"index": 6, "tokens": 3845176, "wall_clock_seconds": 79.8, "raw": 0.177921, "normalized": 0.177921, "discount": 1.0, "effective": 0.177921, "running_max": 0.177921, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 79.49}, {"index": 7, "tokens": 4096525, "wall_clock_seconds": 298.4, "raw": 0.167235, "normalized": 0.167235, "discount": 1.0, "effective": 0.167235, "running_max": 0.177921, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 298.17}, {"index": 8, "tokens": 4363486, "wall_clock_seconds": 136.2, "raw": 0.178834, "normalized": 0.178834, "discount": 1.0, "effective": 0.178834, "running_max": 0.178834, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 135.94}, {"index": 9, "tokens": 4923171, "wall_clock_seconds": 112.9, "raw": 0.179133, "normalized": 0.179133, "discount": 1.0, "effective": 0.179133, "running_max": 0.179133, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 112.66}, {"index": 10, "tokens": 5219084, "wall_clock_seconds": 121.2, "raw": 0.179133, "normalized": 0.179133, "discount": 1.0, "effective": 0.179133, "running_max": 0.179133, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.91} ]}, {"sample_key": "CzycNYQWC2pKrmSr49Bvd2:weirdml_bonanza", "eval_id": "CzycNYQWC2pKrmSr49Bvd2", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "afdb10ba1bffcd14cbcaec042923fe09c990b570e5a1f9c6bea9b35f6c28d4b1", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.793195, "area_term": 0.76578, "final_best": 0.902856, "best_raw": 0.902856, "best_effective": 0.902856, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13885407, "usage": {"input": 366, "output": 189073, "cache_read": 13129086, "cache_write": 566882}, "cost_usd": 29.6724, "cost_source": "billed", "wall_clock_seconds": 7109.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 822047, "wall_clock_seconds": 69.9, "raw": 0.753403, "normalized": 0.753403, "discount": 1.0, "effective": 0.753403, "running_max": 0.753403, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 69.28}, {"index": 2, "tokens": 1843641, "wall_clock_seconds": 76.1, "raw": 0.820106, "normalized": 0.820106, "discount": 1.0, "effective": 0.820106, "running_max": 0.820106, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.45}, {"index": 3, "tokens": 2103459, "wall_clock_seconds": 78.3, "raw": 0.850629, "normalized": 0.850629, "discount": 1.0, "effective": 0.850629, "running_max": 0.850629, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 77.69}, {"index": 4, "tokens": 4068051, "wall_clock_seconds": 87.5, "raw": 0.883495, "normalized": 0.883495, "discount": 1.0, "effective": 0.883495, "running_max": 0.883495, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.82}, {"index": 5, "tokens": 5348643, "wall_clock_seconds": 106.2, "raw": 0.887506, "normalized": 0.887506, "discount": 1.0, "effective": 0.887506, "running_max": 0.887506, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.57}, {"index": 6, "tokens": 6984150, "wall_clock_seconds": 100.6, "raw": 0.888215, "normalized": 0.888215, "discount": 1.0, "effective": 0.888215, "running_max": 0.888215, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.88}, {"index": 7, "tokens": 7871182, "wall_clock_seconds": 94.5, "raw": 0.890474, "normalized": 0.890474, "discount": 1.0, "effective": 0.890474, "running_max": 0.890474, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.0}, {"index": 8, "tokens": 11648323, "wall_clock_seconds": 98.9, "raw": 0.900768, "normalized": 0.900768, "discount": 1.0, "effective": 0.900768, "running_max": 0.900768, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.11}, {"index": 9, "tokens": 12678825, "wall_clock_seconds": 112.1, "raw": 0.901497, "normalized": 0.901497, "discount": 1.0, "effective": 0.901497, "running_max": 0.901497, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 111.35}, {"index": 10, "tokens": 13885407, "wall_clock_seconds": 111.7, "raw": 0.902856, "normalized": 0.902856, "discount": 1.0, "effective": 0.902856, "running_max": 0.902856, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 111.12} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:mystery_box", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.010453, "area_term": 0.004499, "final_best": 0.034268, "best_raw": 0.157883, "best_effective": 0.034268, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 43539607, "usage": {"input": 371314, "output": 196962, "cache_read": 42971331, "cache_write": 41984862}, "cost_usd": 5.9907, "cost_source": "list_price", "wall_clock_seconds": 6081.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 26531622, "wall_clock_seconds": 6.3, "raw": 0.150148, "normalized": 0.128079, "discount": 0.25, "effective": 0.03202, "running_max": 0.03202, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 27910688, "wall_clock_seconds": 4.3, "raw": 0.116078, "normalized": 0.088463, "discount": 0.25, "effective": 0.022116, "running_max": 0.03202, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 30198487, "wall_clock_seconds": 3.6, "raw": 0.098743, "normalized": 0.068306, "discount": 0.25, "effective": 0.017077, "running_max": 0.03202, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 36211484, "wall_clock_seconds": 5.4, "raw": 0.149664, "normalized": 0.127516, "discount": 0.25, "effective": 0.031879, "running_max": 0.03202, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 38101490, "wall_clock_seconds": 4.8, "raw": 0.058359, "normalized": 0.021347, "discount": 0.25, "effective": 0.005337, "running_max": 0.03202, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 39369413, "wall_clock_seconds": 5.0, "raw": 0.146258, "normalized": 0.123556, "discount": 0.25, "effective": 0.030889, "running_max": 0.03202, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 40007355, "wall_clock_seconds": 8.5, "raw": 0.14974, "normalized": 0.127605, "discount": 0.25, "effective": 0.031901, "running_max": 0.03202, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 41302012, "wall_clock_seconds": 7.4, "raw": 0.157883, "normalized": 0.137073, "discount": 0.25, "effective": 0.034268, "running_max": 0.034268, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 41949052, "wall_clock_seconds": 5.6, "raw": 0.157532, "normalized": 0.136665, "discount": 0.25, "effective": 0.034166, "running_max": 0.034268, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 43539607, "wall_clock_seconds": 10.9, "raw": 0.157309, "normalized": 0.136406, "discount": 0.25, "effective": 0.034101, "running_max": 0.034268, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:mystery_box--nohints", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.014344, "area_term": 0.011417, "final_best": 0.026049, "best_raw": 0.062402, "best_effective": 0.026049, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11066149, "usage": {"input": 131445, "output": 66665, "cache_read": 10868039, "cache_write": 10868039}, "cost_usd": 1.6169, "cost_source": "list_price", "wall_clock_seconds": 1323.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4177485, "wall_clock_seconds": 12.0, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4634496, "wall_clock_seconds": 5.6, "raw": 0.045706, "normalized": 0.006635, "discount": 1.0, "effective": 0.006635, "running_max": 0.006635, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 5208912, "wall_clock_seconds": 5.4, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 5513462, "wall_clock_seconds": 2.9, "raw": 0.047128, "normalized": 0.008289, "discount": 1.0, "effective": 0.008289, "running_max": 0.008289, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 5913351, "wall_clock_seconds": 1.4, "raw": 0.029693, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.008289, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 6320565, "wall_clock_seconds": 3.0, "raw": 0.04253, "normalized": 0.002942, "discount": 1.0, "effective": 0.002942, "running_max": 0.008289, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 8065152, "wall_clock_seconds": 2.9, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.026049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 8873094, "wall_clock_seconds": 2.8, "raw": 0.062378, "normalized": 0.026021, "discount": 1.0, "effective": 0.026021, "running_max": 0.026049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 10316573, "wall_clock_seconds": 2.4, "raw": 0.057607, "normalized": 0.020474, "discount": 1.0, "effective": 0.020474, "running_max": 0.026049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 11066149, "wall_clock_seconds": 2.7, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.026049, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:night_school", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.037491, "area_term": 0.032047, "final_best": 0.059265, "best_raw": 0.2275, "best_effective": 0.059265, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 24326507, "usage": {"input": 225802, "output": 130364, "cache_read": 23970341, "cache_write": 23970341}, "cost_usd": 3.4556, "cost_source": "list_price", "wall_clock_seconds": 2994.1, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2843076, "wall_clock_seconds": 2.7, "raw": 0.197, "normalized": 0.055294, "discount": 0.65, "effective": 0.035941, "running_max": 0.035941, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3680952, "wall_clock_seconds": 10.7, "raw": 0.2135, "normalized": 0.074706, "discount": 0.65, "effective": 0.048559, "running_max": 0.048559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 11904792, "wall_clock_seconds": 10.6, "raw": 0.2065, "normalized": 0.066471, "discount": 0.65, "effective": 0.043206, "running_max": 0.048559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 13621229, "wall_clock_seconds": 11.9, "raw": 0.2075, "normalized": 0.067647, "discount": 0.65, "effective": 0.043971, "running_max": 0.048559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 14823597, "wall_clock_seconds": 11.0, "raw": 0.221, "normalized": 0.083529, "discount": 0.65, "effective": 0.054294, "running_max": 0.054294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 15923080, "wall_clock_seconds": 11.5, "raw": 0.211, "normalized": 0.071765, "discount": 0.65, "effective": 0.046647, "running_max": 0.054294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 19643749, "wall_clock_seconds": 12.1, "raw": 0.2275, "normalized": 0.091176, "discount": 0.65, "effective": 0.059265, "running_max": 0.059265, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 20912447, "wall_clock_seconds": 11.4, "raw": 0.22, "normalized": 0.082353, "discount": 0.65, "effective": 0.053529, "running_max": 0.059265, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 22021104, "wall_clock_seconds": 11.7, "raw": 0.222, "normalized": 0.084706, "discount": 0.65, "effective": 0.055059, "running_max": 0.059265, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:reaction_rates", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.011613, "area_term": 0.005718, "final_best": 0.035195, "best_raw": 0.140836, "best_effective": 0.035195, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 7, "max_submissions": 10, "tokens_total": 42226356, "usage": {"input": 321746, "output": 183216, "cache_read": 41721394, "cache_write": 41224340}, "cost_usd": 5.7765, "cost_source": "list_price", "wall_clock_seconds": 2938.7, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 2}, {"id": 3, "multiplier": 0.7, "before_submission": 2}, {"id": 4, "multiplier": 0.8, "before_submission": 2}, {"id": 5, "multiplier": 0.85, "before_submission": 2}, {"id": 2, "multiplier": 0.75, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 984309, "wall_clock_seconds": 1.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.08}, {"index": 2, "tokens": 16439220, "wall_clock_seconds": 1.7, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 1.02}, {"index": 3, "tokens": 23613128, "wall_clock_seconds": 31.3, "raw": 0.137838, "normalized": 0.137838, "discount": 0.2499, "effective": 0.034446, "running_max": 0.034446, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 30.45}, {"index": 4, "tokens": 26034470, "wall_clock_seconds": 70.9, "raw": 0.140836, "normalized": 0.140836, "discount": 0.2499, "effective": 0.035195, "running_max": 0.035195, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 70.02}, {"index": 5, "tokens": 36707751, "wall_clock_seconds": 37.4, "raw": 0.05901, "normalized": 0.05901, "discount": 0.2499, "effective": 0.014747, "running_max": 0.035195, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 36.85}, {"index": 6, "tokens": 38682014, "wall_clock_seconds": 75.1, "raw": 0.046169, "normalized": 0.046169, "discount": 0.2499, "effective": 0.011538, "running_max": 0.035195, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 74.32}, {"index": 7, "tokens": 40715619, "wall_clock_seconds": 72.5, "raw": 0.1169, "normalized": 0.1169, "discount": 0.2499, "effective": 0.029213, "running_max": 0.035195, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 71.89} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:reaction_rates--nohints", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.07465, "area_term": 0.040431, "final_best": 0.211524, "best_raw": 0.211524, "best_effective": 0.211524, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 48785326, "usage": {"input": 328403, "output": 218170, "cache_read": 48238753, "cache_write": 47466852}, "cost_usd": 6.6617, "cost_source": "list_price", "wall_clock_seconds": 3633.0, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2336227, "wall_clock_seconds": 2.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.98}, {"index": 2, "tokens": 15507366, "wall_clock_seconds": 9.9, "raw": 0.143807, "normalized": 0.143807, "discount": 1.0, "effective": 0.143807, "running_max": 0.143807, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.42}, {"index": 3, "tokens": 27835938, "wall_clock_seconds": 8.7, "raw": 0.094728, "normalized": 0.094728, "discount": 1.0, "effective": 0.094728, "running_max": 0.143807, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.97}, {"index": 4, "tokens": 31949563, "wall_clock_seconds": 7.8, "raw": 0.143807, "normalized": 0.143807, "discount": 1.0, "effective": 0.143807, "running_max": 0.143807, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.35}, {"index": 5, "tokens": 38421491, "wall_clock_seconds": 8.9, "raw": 0.211524, "normalized": 0.211524, "discount": 1.0, "effective": 0.211524, "running_max": 0.211524, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.22}, {"index": 6, "tokens": 46967232, "wall_clock_seconds": 8.2, "raw": 0.188492, "normalized": 0.188492, "discount": 1.0, "effective": 0.188492, "running_max": 0.211524, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.62} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:scan_stitch", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.000973, "area_term": 1.8e-05, "final_best": 0.004796, "best_raw": 0.110963, "best_effective": 0.004796, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 50219976, "usage": {"input": 445254, "output": 261137, "cache_read": 49513585, "cache_write": 45884326}, "cost_usd": 6.9401, "cost_source": "list_price", "wall_clock_seconds": 4110.0, "stop_reason": "token_limit", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 12165101, "wall_clock_seconds": 0.3, "raw": 0.089677, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 17793512, "wall_clock_seconds": 0.3, "raw": 0.08793, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 46338083, "wall_clock_seconds": 0.3, "raw": 0.085843, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 47752366, "wall_clock_seconds": 0.3, "raw": 0.082933, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 49159164, "wall_clock_seconds": 0.4, "raw": 0.110963, "normalized": 0.013703, "discount": 0.35, "effective": 0.004796, "running_max": 0.004796, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:scan_stitch--nohints", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.099305, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13911057, "usage": {"input": 164570, "output": 89187, "cache_read": 13657300, "cache_write": 13575986}, "cost_usd": 2.0483, "cost_source": "list_price", "wall_clock_seconds": 1819.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4784984, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4959354, "wall_clock_seconds": 0.2, "raw": 0.075717, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 6655756, "wall_clock_seconds": 0.3, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 7470796, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 8758065, "wall_clock_seconds": 0.3, "raw": 0.099305, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 10875488, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 12419937, "wall_clock_seconds": 0.2, "raw": 0.07485, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 12824685, "wall_clock_seconds": 0.3, "raw": 0.06175, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 13499876, "wall_clock_seconds": 0.2, "raw": 0.034376, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 13911057, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:shapes_generalize", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.04819, "area_term": 0.036175, "final_best": 0.09625, "best_raw": 0.277, "best_effective": 0.09625, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12938635, "usage": {"input": 204938, "output": 130157, "cache_read": 12603540, "cache_write": 11830809}, "cost_usd": 2.0804, "cost_source": "list_price", "wall_clock_seconds": 1743.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5630849, "wall_clock_seconds": 3.0, "raw": 0.185, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.45}, {"index": 2, "tokens": 6593261, "wall_clock_seconds": 1.0, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.27}, {"index": 3, "tokens": 6924303, "wall_clock_seconds": 0.9, "raw": 0.173, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.26}, {"index": 4, "tokens": 7389969, "wall_clock_seconds": 1.4, "raw": 0.188, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.86}, {"index": 5, "tokens": 7858717, "wall_clock_seconds": 1.1, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.43}, {"index": 6, "tokens": 8844559, "wall_clock_seconds": 2.9, "raw": 0.239, "normalized": 0.04875, "discount": 1.0, "effective": 0.04875, "running_max": 0.04875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.26}, {"index": 7, "tokens": 9486970, "wall_clock_seconds": 5.1, "raw": 0.239, "normalized": 0.04875, "discount": 1.0, "effective": 0.04875, "running_max": 0.04875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.57}, {"index": 8, "tokens": 9895450, "wall_clock_seconds": 1.7, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.05}, {"index": 9, "tokens": 11447483, "wall_clock_seconds": 3.3, "raw": 0.277, "normalized": 0.09625, "discount": 1.0, "effective": 0.09625, "running_max": 0.09625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.69}, {"index": 10, "tokens": 12938635, "wall_clock_seconds": 2.8, "raw": 0.276, "normalized": 0.095, "discount": 1.0, "effective": 0.095, "running_max": 0.09625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.24} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:shattered_prior", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.04873, "area_term": 0.04873, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 10619217, "usage": {"input": 266858, "output": 81050, "cache_read": 10271309, "cache_write": 9764366}, "cost_usd": 1.6816, "cost_source": "list_price", "wall_clock_seconds": 1972.9, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.7, "before_submission": 2}, {"id": 3, "multiplier": 0.65, "before_submission": 2}, {"id": 4, "multiplier": 0.55, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 259975, "wall_clock_seconds": 1.8, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5588858, "wall_clock_seconds": 1.8, "raw": 0.002081, "normalized": 0.002081, "discount": 0.55, "effective": 0.001145, "running_max": 0.04873, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 5780167, "wall_clock_seconds": 1.9, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:shattered_prior--nohints", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.000866, "area_term": 0.000556, "final_best": 0.002109, "best_raw": 0.002109, "best_effective": 0.002109, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 27902962, "usage": {"input": 271141, "output": 172024, "cache_read": 27459797, "cache_write": 25693423}, "cost_usd": 3.9793, "cost_source": "list_price", "wall_clock_seconds": 2756.1, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 14756248, "wall_clock_seconds": 1.6, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.002081, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 24973842, "wall_clock_seconds": 1.8, "raw": 0.002109, "normalized": 0.002109, "discount": 1.0, "effective": 0.002109, "running_max": 0.002109, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 25530438, "wall_clock_seconds": 1.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.002109, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:ship_tune", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.043269, "area_term": 0.032896, "final_best": 0.084764, "best_raw": 0.084764, "best_effective": 0.084764, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 33767444, "usage": {"input": 335443, "output": 148502, "cache_read": 33283499, "cache_write": 31158952}, "cost_usd": 4.6041, "cost_source": "list_price", "wall_clock_seconds": 4021.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 6909597, "wall_clock_seconds": 80.7, "raw": 0.067978, "normalized": 0.067978, "discount": 1.0, "effective": 0.067978, "running_max": 0.067978, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 80.04}, {"index": 2, "tokens": 14069181, "wall_clock_seconds": 50.8, "raw": 0.076798, "normalized": 0.076798, "discount": 1.0, "effective": 0.076798, "running_max": 0.076798, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 50.24}, {"index": 3, "tokens": 16187448, "wall_clock_seconds": 49.9, "raw": 0.077132, "normalized": 0.077132, "discount": 1.0, "effective": 0.077132, "running_max": 0.077132, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 49.16}, {"index": 4, "tokens": 17707136, "wall_clock_seconds": 57.0, "raw": 0.078555, "normalized": 0.078555, "discount": 1.0, "effective": 0.078555, "running_max": 0.078555, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 56.46}, {"index": 5, "tokens": 23523162, "wall_clock_seconds": 59.4, "raw": 0.078371, "normalized": 0.078371, "discount": 1.0, "effective": 0.078371, "running_max": 0.078555, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.73}, {"index": 6, "tokens": 25377801, "wall_clock_seconds": 62.1, "raw": 0.081932, "normalized": 0.081932, "discount": 1.0, "effective": 0.081932, "running_max": 0.081932, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 61.02}, {"index": 7, "tokens": 27300291, "wall_clock_seconds": 60.6, "raw": 0.084267, "normalized": 0.084267, "discount": 1.0, "effective": 0.084267, "running_max": 0.084267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 59.86}, {"index": 8, "tokens": 29295294, "wall_clock_seconds": 57.8, "raw": 0.084294, "normalized": 0.084294, "discount": 1.0, "effective": 0.084294, "running_max": 0.084294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 57.18}, {"index": 9, "tokens": 31624393, "wall_clock_seconds": 60.1, "raw": 0.084514, "normalized": 0.084514, "discount": 1.0, "effective": 0.084514, "running_max": 0.084514, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 59.48}, {"index": 10, "tokens": 33767444, "wall_clock_seconds": 59.3, "raw": 0.084764, "normalized": 0.084764, "discount": 1.0, "effective": 0.084764, "running_max": 0.084764, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.42} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:splash_generalize", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.278222, "area_term": 0.198889, "final_best": 0.595556, "best_raw": 0.636, "best_effective": 0.595556, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 39024197, "usage": {"input": 545396, "output": 367766, "cache_read": 38111035, "cache_write": 22184066}, "cost_usd": 5.5716, "cost_source": "list_price", "wall_clock_seconds": 7154.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 9246981, "wall_clock_seconds": 28.0, "raw": 0.544, "normalized": 0.493333, "discount": 1.0, "effective": 0.493333, "running_max": 0.493333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.43}, {"index": 2, "tokens": 19385133, "wall_clock_seconds": 60.8, "raw": 0.574, "normalized": 0.526667, "discount": 1.0, "effective": 0.526667, "running_max": 0.526667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 59.83}, {"index": 3, "tokens": 20723916, "wall_clock_seconds": 59.4, "raw": 0.617, "normalized": 0.574444, "discount": 1.0, "effective": 0.574444, "running_max": 0.574444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.82}, {"index": 4, "tokens": 22081294, "wall_clock_seconds": 2.2, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.574444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.48}, {"index": 5, "tokens": 27357758, "wall_clock_seconds": 67.3, "raw": 0.624, "normalized": 0.582222, "discount": 1.0, "effective": 0.582222, "running_max": 0.582222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 66.82}, {"index": 6, "tokens": 29136757, "wall_clock_seconds": 66.3, "raw": 0.619, "normalized": 0.576667, "discount": 1.0, "effective": 0.576667, "running_max": 0.582222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 65.63}, {"index": 7, "tokens": 33605828, "wall_clock_seconds": 87.7, "raw": 0.633, "normalized": 0.592222, "discount": 1.0, "effective": 0.592222, "running_max": 0.592222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.19}, {"index": 8, "tokens": 35543322, "wall_clock_seconds": 85.9, "raw": 0.632, "normalized": 0.591111, "discount": 1.0, "effective": 0.591111, "running_max": 0.592222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 85.31}, {"index": 9, "tokens": 37264462, "wall_clock_seconds": 88.5, "raw": 0.636, "normalized": 0.595556, "discount": 1.0, "effective": 0.595556, "running_max": 0.595556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.92}, {"index": 10, "tokens": 39024197, "wall_clock_seconds": 102.5, "raw": 0.636, "normalized": 0.595556, "discount": 1.0, "effective": 0.595556, "running_max": 0.595556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.96} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:tod_pipeline--nohints", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.0, "best_effective": 0.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 20552286, "usage": {"input": 354136, "output": 89898, "cache_read": 20108252, "cache_write": 18238858}, "cost_usd": 2.8714, "cost_source": "list_price", "wall_clock_seconds": 1888.9, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 9320383, "wall_clock_seconds": 9.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.74}, {"index": 2, "tokens": 14918812, "wall_clock_seconds": 9.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.57}, {"index": 3, "tokens": 17559664, "wall_clock_seconds": 10.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 10.15} ]}, {"sample_key": "FLXrcQPjhCd4aUeR6TNY7X:weirdml_bonanza", "eval_id": "FLXrcQPjhCd4aUeR6TNY7X", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "94ceee7", "source_log_sha256": "d2a537ebf29e71df31602f466557e33e416f07af783fbd9296c3741a8caad674", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.279398, "area_term": 0.230464, "final_best": 0.475135, "best_raw": 0.475135, "best_effective": 0.475135, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 7, "max_submissions": 10, "tokens_total": 30220477, "usage": {"input": 353669, "output": 173162, "cache_read": 29693646, "cache_write": 27904396}, "cost_usd": 4.3052, "cost_source": "list_price", "wall_clock_seconds": 3379.1, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1764387, "wall_clock_seconds": 1.3, "raw": 0.163696, "normalized": 0.163696, "discount": 1.0, "effective": 0.163696, "running_max": 0.163696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.79}, {"index": 2, "tokens": 9252812, "wall_clock_seconds": 55.7, "raw": 0.460259, "normalized": 0.460259, "discount": 1.0, "effective": 0.460259, "running_max": 0.460259, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 55.11}, {"index": 3, "tokens": 12458735, "wall_clock_seconds": 63.1, "raw": 0.4551, "normalized": 0.4551, "discount": 1.0, "effective": 0.4551, "running_max": 0.460259, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 62.48}, {"index": 4, "tokens": 17217826, "wall_clock_seconds": 53.9, "raw": 0.455506, "normalized": 0.455506, "discount": 1.0, "effective": 0.455506, "running_max": 0.460259, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 53.38}, {"index": 5, "tokens": 20095324, "wall_clock_seconds": 63.6, "raw": 0.475135, "normalized": 0.475135, "discount": 1.0, "effective": 0.475135, "running_max": 0.475135, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 63.09}, {"index": 6, "tokens": 24280396, "wall_clock_seconds": 65.3, "raw": 0.460057, "normalized": 0.460057, "discount": 1.0, "effective": 0.460057, "running_max": 0.475135, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 64.73}, {"index": 7, "tokens": 27688352, "wall_clock_seconds": 74.3, "raw": 0.471835, "normalized": 0.471835, "discount": 1.0, "effective": 0.471835, "running_max": 0.475135, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 73.58} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:mystery_box", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.018301, "area_term": 0.010093, "final_best": 0.051133, "best_raw": 0.215898, "best_effective": 0.051133, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 34305264, "usage": {"input": 5734, "output": 396601, "cache_read": 33392997, "cache_write": 509932}, "cost_usd": 38.4343, "cost_source": "list_price", "wall_clock_seconds": 10658.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1938286, "wall_clock_seconds": 5.6, "raw": 0.042522, "normalized": 0.002933, "discount": 0.6, "effective": 0.00176, "running_max": 0.00176, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 17429877, "wall_clock_seconds": 4.4, "raw": 0.120256, "normalized": 0.093321, "discount": 0.25, "effective": 0.02333, "running_max": 0.02333, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 19387146, "wall_clock_seconds": 4.1, "raw": 0.089336, "normalized": 0.057367, "discount": 0.25, "effective": 0.014342, "running_max": 0.02333, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 23976086, "wall_clock_seconds": 5.5, "raw": 0.180505, "normalized": 0.163378, "discount": 0.25, "effective": 0.040844, "running_max": 0.040844, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 27105772, "wall_clock_seconds": 3.7, "raw": 0.190042, "normalized": 0.174467, "discount": 0.25, "effective": 0.043617, "running_max": 0.043617, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 29430488, "wall_clock_seconds": 3.6, "raw": 0.161481, "normalized": 0.141257, "discount": 0.25, "effective": 0.035314, "running_max": 0.043617, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 30386025, "wall_clock_seconds": 4.0, "raw": 0.201849, "normalized": 0.188197, "discount": 0.25, "effective": 0.047049, "running_max": 0.047049, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 31838400, "wall_clock_seconds": 4.2, "raw": 0.211476, "normalized": 0.199391, "discount": 0.25, "effective": 0.049848, "running_max": 0.049848, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 33309458, "wall_clock_seconds": 4.2, "raw": 0.204984, "normalized": 0.191842, "discount": 0.25, "effective": 0.047961, "running_max": 0.049848, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 34305264, "wall_clock_seconds": 3.8, "raw": 0.215898, "normalized": 0.204532, "discount": 0.25, "effective": 0.051133, "running_max": 0.051133, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:mystery_box--nohints", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.016822, "area_term": 0.015555, "final_best": 0.021894, "best_raw": 0.058829, "best_effective": 0.021894, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7265559, "usage": {"input": 7792, "output": 215307, "cache_read": 6754250, "cache_write": 288210}, "cost_usd": 18.296, "cost_source": "list_price", "wall_clock_seconds": 3871.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 508652, "wall_clock_seconds": 4.2, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1771821, "wall_clock_seconds": 5.5, "raw": 0.045533, "normalized": 0.006434, "discount": 1.0, "effective": 0.006434, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2279238, "wall_clock_seconds": 8.9, "raw": 0.049994, "normalized": 0.011621, "discount": 1.0, "effective": 0.011621, "running_max": 0.011621, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3446483, "wall_clock_seconds": 7.0, "raw": 0.038885, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.011621, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4022249, "wall_clock_seconds": 3.6, "raw": 0.058829, "normalized": 0.021894, "discount": 1.0, "effective": 0.021894, "running_max": 0.021894, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 5120495, "wall_clock_seconds": 10.1, "raw": 0.052697, "normalized": 0.014764, "discount": 1.0, "effective": 0.014764, "running_max": 0.021894, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5605733, "wall_clock_seconds": 3.2, "raw": 0.046462, "normalized": 0.007514, "discount": 1.0, "effective": 0.007514, "running_max": 0.021894, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6130720, "wall_clock_seconds": 2.1, "raw": 0.031098, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.021894, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6689080, "wall_clock_seconds": 2.4, "raw": 0.036769, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.021894, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7265559, "wall_clock_seconds": 3.8, "raw": 0.041625, "normalized": 0.001889, "discount": 1.0, "effective": 0.001889, "running_max": 0.021894, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:night_school", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.437859, "area_term": 0.346248, "final_best": 0.804301, "best_raw": 0.833656, "best_effective": 0.804301, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 15613613, "usage": {"input": 6159, "output": 215343, "cache_read": 15074451, "cache_write": 317660}, "cost_usd": 20.9506, "cost_source": "list_price", "wall_clock_seconds": 37970.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5824551, "wall_clock_seconds": 10.7, "raw": 0.4555, "normalized": 0.359412, "discount": 1.0, "effective": 0.359412, "running_max": 0.359412, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6909639, "wall_clock_seconds": 11.9, "raw": 0.617594, "normalized": 0.55011, "discount": 1.0, "effective": 0.55011, "running_max": 0.55011, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 7609895, "wall_clock_seconds": 10.9, "raw": 0.666063, "normalized": 0.607132, "discount": 1.0, "effective": 0.607132, "running_max": 0.607132, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 8093292, "wall_clock_seconds": 11.8, "raw": 0.741313, "normalized": 0.695662, "discount": 1.0, "effective": 0.695662, "running_max": 0.695662, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 9908807, "wall_clock_seconds": 13.4, "raw": 0.833656, "normalized": 0.804301, "discount": 1.0, "effective": 0.804301, "running_max": 0.804301, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 14348752, "wall_clock_seconds": 11.8, "raw": 0.826344, "normalized": 0.795699, "discount": 1.0, "effective": 0.795699, "running_max": 0.804301, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:reaction_rates", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.151352, "area_term": 0.108596, "final_best": 0.322378, "best_raw": 0.575674, "best_effective": 0.322378, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 36456553, "usage": {"input": 21161, "output": 299255, "cache_read": 35669459, "cache_write": 466678}, "cost_usd": 33.4253, "cost_source": "list_price", "wall_clock_seconds": 37948.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 2}, {"id": 4, "multiplier": 0.8, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1291070, "wall_clock_seconds": 391.8, "raw": 0.028144, "normalized": 0.028144, "discount": 1.0, "effective": 0.028144, "running_max": 0.028144, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 391.05}, {"index": 2, "tokens": 10138185, "wall_clock_seconds": 40.5, "raw": 0.321493, "normalized": 0.321493, "discount": 0.56, "effective": 0.180036, "running_max": 0.180036, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 40.17}, {"index": 3, "tokens": 13870942, "wall_clock_seconds": 40.0, "raw": 0.520238, "normalized": 0.520238, "discount": 0.56, "effective": 0.291333, "running_max": 0.291333, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 39.77}, {"index": 4, "tokens": 19089495, "wall_clock_seconds": 140.0, "raw": 0.51641, "normalized": 0.51641, "discount": 0.56, "effective": 0.28919, "running_max": 0.291333, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 139.74}, {"index": 5, "tokens": 20442104, "wall_clock_seconds": 73.1, "raw": 0.176536, "normalized": 0.176536, "discount": 0.56, "effective": 0.09886, "running_max": 0.291333, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 67.67}, {"index": 6, "tokens": 21842736, "wall_clock_seconds": 277.6, "raw": 0.138135, "normalized": 0.138135, "discount": 0.56, "effective": 0.077355, "running_max": 0.291333, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 277.29}, {"index": 7, "tokens": 25461150, "wall_clock_seconds": 276.8, "raw": 0.519807, "normalized": 0.519807, "discount": 0.56, "effective": 0.291092, "running_max": 0.291333, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 276.51}, {"index": 8, "tokens": 31035586, "wall_clock_seconds": 319.6, "raw": 0.524315, "normalized": 0.524315, "discount": 0.56, "effective": 0.293616, "running_max": 0.293616, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 319.21}, {"index": 9, "tokens": 33700056, "wall_clock_seconds": 356.7, "raw": 0.568213, "normalized": 0.568213, "discount": 0.56, "effective": 0.318199, "running_max": 0.318199, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 356.39}, {"index": 10, "tokens": 36456553, "wall_clock_seconds": 377.9, "raw": 0.575674, "normalized": 0.575674, "discount": 0.56, "effective": 0.322378, "running_max": 0.322378, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 377.56} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:reaction_rates--nohints", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.086223, "area_term": 0.065257, "final_best": 0.170088, "best_raw": 0.170088, "best_effective": 0.170088, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 20449707, "usage": {"input": 13238, "output": 234722, "cache_read": 19827417, "cache_write": 374330}, "cost_usd": 24.3119, "cost_source": "list_price", "wall_clock_seconds": 18859.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1851394, "wall_clock_seconds": 36.7, "raw": 0.033624, "normalized": 0.033624, "discount": 1.0, "effective": 0.033624, "running_max": 0.033624, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 36.08}, {"index": 2, "tokens": 9435704, "wall_clock_seconds": 141.7, "raw": 0.056037, "normalized": 0.056037, "discount": 1.0, "effective": 0.056037, "running_max": 0.056037, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 141.06}, {"index": 3, "tokens": 11371722, "wall_clock_seconds": 109.3, "raw": 0.102895, "normalized": 0.102895, "discount": 1.0, "effective": 0.102895, "running_max": 0.102895, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 108.78}, {"index": 4, "tokens": 13483643, "wall_clock_seconds": 101.3, "raw": 0.13064, "normalized": 0.13064, "discount": 1.0, "effective": 0.13064, "running_max": 0.13064, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.71}, {"index": 5, "tokens": 14114143, "wall_clock_seconds": 104.4, "raw": 0.152534, "normalized": 0.152534, "discount": 1.0, "effective": 0.152534, "running_max": 0.152534, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.88}, {"index": 6, "tokens": 15090139, "wall_clock_seconds": 99.6, "raw": 0.153547, "normalized": 0.153547, "discount": 1.0, "effective": 0.153547, "running_max": 0.153547, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.95}, {"index": 7, "tokens": 17154662, "wall_clock_seconds": 157.8, "raw": 0.169882, "normalized": 0.169882, "discount": 1.0, "effective": 0.169882, "running_max": 0.169882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 157.21}, {"index": 8, "tokens": 17864228, "wall_clock_seconds": 164.6, "raw": 0.089985, "normalized": 0.089985, "discount": 1.0, "effective": 0.089985, "running_max": 0.169882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 164.01}, {"index": 9, "tokens": 18957132, "wall_clock_seconds": 272.2, "raw": 0.166972, "normalized": 0.166972, "discount": 1.0, "effective": 0.166972, "running_max": 0.169882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 271.7}, {"index": 10, "tokens": 20449707, "wall_clock_seconds": 147.8, "raw": 0.170088, "normalized": 0.170088, "discount": 1.0, "effective": 0.170088, "running_max": 0.170088, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 147.34} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:scan_stitch", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.020448, "area_term": 0.006323, "final_best": 0.076952, "best_raw": 0.27589, "best_effective": 0.076952, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 37844262, "usage": {"input": 6642, "output": 366644, "cache_read": 37000473, "cache_write": 470503}, "cost_usd": 37.0588, "cost_source": "list_price", "wall_clock_seconds": 10365.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 2}, {"id": 4, "multiplier": 0.35, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 18300671, "wall_clock_seconds": 0.3, "raw": 0.081234, "normalized": 0.0, "discount": 0.68, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 21623460, "wall_clock_seconds": 0.3, "raw": 0.08227, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 24083062, "wall_clock_seconds": 0.4, "raw": 0.087377, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 26330452, "wall_clock_seconds": 0.3, "raw": 0.091755, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 31163890, "wall_clock_seconds": 0.3, "raw": 0.090797, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 33324605, "wall_clock_seconds": 0.3, "raw": 0.238063, "normalized": 0.172579, "discount": 0.35, "effective": 0.060403, "running_max": 0.060403, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 34653191, "wall_clock_seconds": 0.3, "raw": 0.196199, "normalized": 0.120248, "discount": 0.35, "effective": 0.042087, "running_max": 0.060403, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 36007096, "wall_clock_seconds": 0.4, "raw": 0.236299, "normalized": 0.170374, "discount": 0.35, "effective": 0.059631, "running_max": 0.060403, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 36920499, "wall_clock_seconds": 0.3, "raw": 0.235319, "normalized": 0.169149, "discount": 0.35, "effective": 0.059202, "running_max": 0.060403, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 37844262, "wall_clock_seconds": 0.3, "raw": 0.27589, "normalized": 0.219862, "discount": 0.35, "effective": 0.076952, "running_max": 0.076952, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:scan_stitch--nohints", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.098227, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 4852483, "usage": {"input": 3649, "output": 203631, "cache_read": 4374723, "cache_write": 270480}, "cost_usd": 16.7213, "cost_source": "list_price", "wall_clock_seconds": 3306.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2224790, "wall_clock_seconds": 0.3, "raw": 0.066094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2391283, "wall_clock_seconds": 0.3, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2564640, "wall_clock_seconds": 0.4, "raw": 0.07548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2749769, "wall_clock_seconds": 0.5, "raw": 0.098227, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2942107, "wall_clock_seconds": 0.3, "raw": 0.087261, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3564073, "wall_clock_seconds": 0.4, "raw": 0.093265, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4039038, "wall_clock_seconds": 0.3, "raw": 0.086927, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4299064, "wall_clock_seconds": 0.3, "raw": 0.063658, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 4569136, "wall_clock_seconds": 0.3, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 4852483, "wall_clock_seconds": 0.3, "raw": 0.066168, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:shapes_generalize", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.528365, "area_term": 0.440456, "final_best": 0.88, "best_raw": 0.904, "best_effective": 0.88, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10818105, "usage": {"input": 12628, "output": 140226, "cache_read": 10447242, "cache_write": 218009}, "cost_usd": 14.1096, "cost_source": "list_price", "wall_clock_seconds": 16759.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1562344, "wall_clock_seconds": 5.3, "raw": 0.219, "normalized": 0.02375, "discount": 1.0, "effective": 0.02375, "running_max": 0.02375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.76}, {"index": 2, "tokens": 1923080, "wall_clock_seconds": 1.0, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.02375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.42}, {"index": 3, "tokens": 3316341, "wall_clock_seconds": 6.4, "raw": 0.38, "normalized": 0.225, "discount": 1.0, "effective": 0.225, "running_max": 0.225, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.83}, {"index": 4, "tokens": 4229088, "wall_clock_seconds": 6.6, "raw": 0.245, "normalized": 0.05625, "discount": 1.0, "effective": 0.05625, "running_max": 0.225, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.99}, {"index": 5, "tokens": 5177206, "wall_clock_seconds": 5.7, "raw": 0.235, "normalized": 0.04375, "discount": 1.0, "effective": 0.04375, "running_max": 0.225, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.1}, {"index": 6, "tokens": 5854392, "wall_clock_seconds": 6.1, "raw": 0.893, "normalized": 0.86625, "discount": 1.0, "effective": 0.86625, "running_max": 0.86625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.53}, {"index": 7, "tokens": 7701310, "wall_clock_seconds": 7.0, "raw": 0.899, "normalized": 0.87375, "discount": 1.0, "effective": 0.87375, "running_max": 0.87375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.41}, {"index": 8, "tokens": 9120132, "wall_clock_seconds": 8.1, "raw": 0.904, "normalized": 0.88, "discount": 1.0, "effective": 0.88, "running_max": 0.88, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.49}, {"index": 9, "tokens": 10166425, "wall_clock_seconds": 6.5, "raw": 0.901, "normalized": 0.87625, "discount": 1.0, "effective": 0.87625, "running_max": 0.88, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.91}, {"index": 10, "tokens": 10818105, "wall_clock_seconds": 6.1, "raw": 0.9, "normalized": 0.875, "discount": 1.0, "effective": 0.875, "running_max": 0.88, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.61} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:shattered_prior", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.148476, "area_term": 0.071107, "final_best": 0.457954, "best_raw": 0.832644, "best_effective": 0.457954, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 40510230, "usage": {"input": 7054, "output": 330902, "cache_read": 39667846, "cache_write": 504428}, "cost_usd": 36.6212, "cost_source": "list_price", "wall_clock_seconds": 27857.3, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 1}, {"id": 2, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.65, "before_submission": 1}, {"id": 4, "multiplier": 0.55, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 23312363, "wall_clock_seconds": 1.7, "raw": 0.626467, "normalized": 0.626467, "discount": 0.55, "effective": 0.344557, "running_max": 0.344557, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 24579759, "wall_clock_seconds": 2.1, "raw": 0.653773, "normalized": 0.653773, "discount": 0.55, "effective": 0.359575, "running_max": 0.359575, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 27194453, "wall_clock_seconds": 1.8, "raw": 0.776776, "normalized": 0.776776, "discount": 0.55, "effective": 0.427227, "running_max": 0.427227, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 32739605, "wall_clock_seconds": 1.8, "raw": 0.829267, "normalized": 0.829267, "discount": 0.55, "effective": 0.456097, "running_max": 0.456097, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 39024784, "wall_clock_seconds": 1.7, "raw": 0.832644, "normalized": 0.832644, "discount": 0.55, "effective": 0.457954, "running_max": 0.457954, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:shattered_prior--nohints", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.011057, "area_term": 0.001332, "final_best": 0.049959, "best_raw": 0.049959, "best_effective": 0.049959, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 44791469, "usage": {"input": 6865, "output": 377760, "cache_read": 43830996, "cache_write": 575848}, "cost_usd": 41.4314, "cost_source": "list_price", "wall_clock_seconds": 10602.0, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 26293648, "wall_clock_seconds": 2.1, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 38687873, "wall_clock_seconds": 2.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 44224393, "wall_clock_seconds": 1.9, "raw": 0.049959, "normalized": 0.049959, "discount": 1.0, "effective": 0.049959, "running_max": 0.049959, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:ship_detect", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.10579, "area_term": 0.073986, "final_best": 0.233003, "best_raw": 0.233003, "best_effective": 0.233003, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 11, "max_submissions": null, "tokens_total": 90753022, "usage": {"input": 29162, "output": 176274, "cache_read": 88128717, "cache_write": 2418869}, "cost_usd": 79.5149, "cost_source": "list_price", "wall_clock_seconds": 35724.5, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 10, "tokens": 311139, "cost_weighted": 26459.5, "wall_clock_seconds": 556.8, "raw": 0.060511, "normalized": 0.060511, "effective": 0.060511, "running_max": 0.060511, "scored": true, "final": false}, {"index": 2, "model_call": 50, "tokens": 3441550, "cost_weighted": 114042.9, "wall_clock_seconds": 5208.9, "raw": 0.050146, "normalized": 0.050146, "effective": 0.050146, "running_max": 0.060511, "scored": true, "final": false}, {"index": 3, "model_call": 379, "tokens": 58871648, "cost_weighted": 948875.9, "wall_clock_seconds": 12503.1, "raw": 0.131532, "normalized": 0.131532, "effective": 0.131532, "running_max": 0.131532, "scored": true, "final": false}, {"index": 4, "model_call": 507, "tokens": 86414723, "cost_weighted": 1541744.3, "wall_clock_seconds": 18411.9, "raw": 0.205118, "normalized": 0.205118, "effective": 0.205118, "running_max": 0.205118, "scored": true, "final": false}, {"index": 5, "model_call": 513, "tokens": 87747092, "cost_weighted": 1555863.0, "wall_clock_seconds": 23274.8, "raw": 0.211339, "normalized": 0.211339, "effective": 0.211339, "running_max": 0.211339, "scored": true, "final": false}, {"index": 6, "model_call": 515, "tokens": 88198233, "cost_weighted": 1559718.6, "wall_clock_seconds": 26338.0, "raw": 0.215456, "normalized": 0.215456, "effective": 0.215456, "running_max": 0.215456, "scored": true, "final": false}, {"index": 7, "model_call": 516, "tokens": 88424286, "cost_weighted": 1561224.5, "wall_clock_seconds": 29063.3, "raw": 0.222062, "normalized": 0.222062, "effective": 0.222062, "running_max": 0.222062, "scored": true, "final": false}, {"index": 8, "model_call": 517, "tokens": 88651300, "cost_weighted": 1563344.4, "wall_clock_seconds": 29817.3, "raw": 0.223363, "normalized": 0.223363, "effective": 0.223363, "running_max": 0.223363, "scored": true, "final": false}, {"index": 9, "model_call": 519, "tokens": 89107390, "cost_weighted": 1567272.5, "wall_clock_seconds": 33106.7, "raw": 0.228718, "normalized": 0.228718, "effective": 0.228718, "running_max": 0.228718, "scored": true, "final": false}, {"index": 10, "model_call": 520, "tokens": 89335791, "cost_weighted": 1568653.1, "wall_clock_seconds": 35408.7, "raw": 0.233003, "normalized": 0.233003, "effective": 0.233003, "running_max": 0.233003, "scored": true, "final": false}, {"index": 11, "model_call": 525, "tokens": 90509077, "cost_weighted": 1587335.5, "wall_clock_seconds": 35663.8, "raw": 0.229286, "normalized": 0.229286, "effective": 0.229286, "running_max": 0.233003, "scored": true, "final": false} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:ship_tune", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.101589, "area_term": 0.069418, "final_best": 0.230274, "best_raw": 0.230274, "best_effective": 0.230274, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 33269369, "usage": {"input": 70190, "output": 451442, "cache_read": 31590995, "cache_write": 1156742}, "cost_usd": 54.3066, "cost_source": "list_price", "wall_clock_seconds": 19579.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1233020, "wall_clock_seconds": 432.3, "raw": 0.053015, "normalized": 0.053015, "discount": 1.0, "effective": 0.053015, "running_max": 0.053015, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 431.31}, {"index": 2, "tokens": 24618884, "wall_clock_seconds": 61.0, "raw": 0.182631, "normalized": 0.182631, "discount": 1.0, "effective": 0.182631, "running_max": 0.182631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 60.35}, {"index": 3, "tokens": 25385644, "wall_clock_seconds": 78.9, "raw": 0.221978, "normalized": 0.221978, "discount": 1.0, "effective": 0.221978, "running_max": 0.221978, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 78.34}, {"index": 4, "tokens": 26363790, "wall_clock_seconds": 354.9, "raw": 0.228448, "normalized": 0.228448, "discount": 1.0, "effective": 0.228448, "running_max": 0.228448, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 354.15}, {"index": 5, "tokens": 27421804, "wall_clock_seconds": 520.6, "raw": 0.227353, "normalized": 0.227353, "discount": 1.0, "effective": 0.227353, "running_max": 0.228448, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 519.96}, {"index": 6, "tokens": 29603955, "wall_clock_seconds": 121.6, "raw": 0.218267, "normalized": 0.218267, "discount": 1.0, "effective": 0.218267, "running_max": 0.228448, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.92}, {"index": 7, "tokens": 30684207, "wall_clock_seconds": 174.4, "raw": 0.225868, "normalized": 0.225868, "discount": 1.0, "effective": 0.225868, "running_max": 0.228448, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 173.43}, {"index": 8, "tokens": 31592120, "wall_clock_seconds": 361.4, "raw": 0.225127, "normalized": 0.225127, "discount": 1.0, "effective": 0.225127, "running_max": 0.228448, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 360.58}, {"index": 9, "tokens": 32535475, "wall_clock_seconds": 418.5, "raw": 0.224171, "normalized": 0.224171, "discount": 1.0, "effective": 0.224171, "running_max": 0.228448, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 417.72}, {"index": 10, "tokens": 33269369, "wall_clock_seconds": 675.2, "raw": 0.230274, "normalized": 0.230274, "discount": 1.0, "effective": 0.230274, "running_max": 0.230274, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 674.45} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:splash_generalize", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.51785, "area_term": 0.473424, "final_best": 0.695556, "best_raw": 0.726, "best_effective": 0.695556, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8128499, "usage": {"input": 4637, "output": 150069, "cache_read": 7752379, "cache_write": 221414}, "cost_usd": 13.9162, "cost_source": "list_price", "wall_clock_seconds": 5128.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1735124, "wall_clock_seconds": 62.0, "raw": 0.409, "normalized": 0.343333, "discount": 1.0, "effective": 0.343333, "running_max": 0.343333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 61.44}, {"index": 2, "tokens": 1995238, "wall_clock_seconds": 62.1, "raw": 0.41, "normalized": 0.344444, "discount": 1.0, "effective": 0.344444, "running_max": 0.344444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 61.45}, {"index": 3, "tokens": 2403047, "wall_clock_seconds": 70.6, "raw": 0.689, "normalized": 0.654444, "discount": 1.0, "effective": 0.654444, "running_max": 0.654444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 70.03}, {"index": 4, "tokens": 2638900, "wall_clock_seconds": 87.1, "raw": 0.673, "normalized": 0.636667, "discount": 1.0, "effective": 0.636667, "running_max": 0.654444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.42}, {"index": 5, "tokens": 3024480, "wall_clock_seconds": 87.4, "raw": 0.639, "normalized": 0.598889, "discount": 1.0, "effective": 0.598889, "running_max": 0.654444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.78}, {"index": 6, "tokens": 4172969, "wall_clock_seconds": 86.9, "raw": 0.691, "normalized": 0.656667, "discount": 1.0, "effective": 0.656667, "running_max": 0.656667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 86.33}, {"index": 7, "tokens": 5149157, "wall_clock_seconds": 101.4, "raw": 0.671, "normalized": 0.634444, "discount": 1.0, "effective": 0.634444, "running_max": 0.656667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.78}, {"index": 8, "tokens": 5854538, "wall_clock_seconds": 94.8, "raw": 0.67, "normalized": 0.633333, "discount": 1.0, "effective": 0.633333, "running_max": 0.656667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.29}, {"index": 9, "tokens": 6834279, "wall_clock_seconds": 80.7, "raw": 0.72, "normalized": 0.688889, "discount": 1.0, "effective": 0.688889, "running_max": 0.688889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 80.02}, {"index": 10, "tokens": 8128499, "wall_clock_seconds": 79.2, "raw": 0.726, "normalized": 0.695556, "discount": 1.0, "effective": 0.695556, "running_max": 0.695556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 78.56} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:tod_pipeline--nohints", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.053428, "area_term": 0.031819, "final_best": 0.139866, "best_raw": 0.139866, "best_effective": 0.139866, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 36468815, "usage": {"input": 16627, "output": 314633, "cache_read": 35639372, "cache_write": 498183}, "cost_usd": 34.7714, "cost_source": "list_price", "wall_clock_seconds": 9218.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 17537908, "wall_clock_seconds": 38.3, "raw": 0.139866, "normalized": 0.139866, "discount": 1.0, "effective": 0.139866, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 37.95}, {"index": 2, "tokens": 24901779, "wall_clock_seconds": 27.4, "raw": 0.108198, "normalized": 0.108198, "discount": 1.0, "effective": 0.108198, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.1}, {"index": 3, "tokens": 27498469, "wall_clock_seconds": 28.2, "raw": 0.107255, "normalized": 0.107255, "discount": 1.0, "effective": 0.107255, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.87}, {"index": 4, "tokens": 29750404, "wall_clock_seconds": 28.2, "raw": 0.106849, "normalized": 0.106849, "discount": 1.0, "effective": 0.106849, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.8}, {"index": 5, "tokens": 31618203, "wall_clock_seconds": 27.9, "raw": 0.11971, "normalized": 0.11971, "discount": 1.0, "effective": 0.11971, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.41}, {"index": 6, "tokens": 32571892, "wall_clock_seconds": 27.9, "raw": 0.105717, "normalized": 0.105717, "discount": 1.0, "effective": 0.105717, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.55}, {"index": 7, "tokens": 33533876, "wall_clock_seconds": 28.2, "raw": 0.11219, "normalized": 0.11219, "discount": 1.0, "effective": 0.11219, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.86}, {"index": 8, "tokens": 34016451, "wall_clock_seconds": 28.7, "raw": 0.114536, "normalized": 0.114536, "discount": 1.0, "effective": 0.114536, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.31}, {"index": 9, "tokens": 34989229, "wall_clock_seconds": 27.9, "raw": 0.128604, "normalized": 0.128604, "discount": 1.0, "effective": 0.128604, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.49}, {"index": 10, "tokens": 36468815, "wall_clock_seconds": 28.3, "raw": 0.11461, "normalized": 0.11461, "discount": 1.0, "effective": 0.11461, "running_max": 0.139866, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.91} ]}, {"sample_key": "bDkqdgMtfU3evr47BSJgHZ:weirdml_bonanza", "eval_id": "bDkqdgMtfU3evr47BSJgHZ", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-14", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "86c0151", "source_log_sha256": "31427ed26ab38253c777b11c7f018218094a956f5e3da7af0e29d09ecc02fb4f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.480884, "area_term": 0.377215, "final_best": 0.895562, "best_raw": 0.895562, "best_effective": 0.895562, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 18712569, "usage": {"input": 31762, "output": 197060, "cache_read": 18157432, "cache_write": 326315}, "cost_usd": 21.2363, "cost_source": "list_price", "wall_clock_seconds": 8137.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 6747936, "wall_clock_seconds": 96.5, "raw": 0.783571, "normalized": 0.783571, "discount": 1.0, "effective": 0.783571, "running_max": 0.783571, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.84}, {"index": 2, "tokens": 9748585, "wall_clock_seconds": 99.0, "raw": 0.826959, "normalized": 0.826959, "discount": 1.0, "effective": 0.826959, "running_max": 0.826959, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 98.33}, {"index": 3, "tokens": 10501857, "wall_clock_seconds": 94.4, "raw": 0.862447, "normalized": 0.862447, "discount": 1.0, "effective": 0.862447, "running_max": 0.862447, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 93.72}, {"index": 4, "tokens": 11542008, "wall_clock_seconds": 103.7, "raw": 0.86906, "normalized": 0.86906, "discount": 1.0, "effective": 0.86906, "running_max": 0.86906, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 102.92}, {"index": 5, "tokens": 12078481, "wall_clock_seconds": 89.8, "raw": 0.769236, "normalized": 0.769236, "discount": 1.0, "effective": 0.769236, "running_max": 0.86906, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 89.11}, {"index": 6, "tokens": 12907154, "wall_clock_seconds": 104.8, "raw": 0.876334, "normalized": 0.876334, "discount": 1.0, "effective": 0.876334, "running_max": 0.876334, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 104.21}, {"index": 7, "tokens": 14048754, "wall_clock_seconds": 106.4, "raw": 0.882726, "normalized": 0.882726, "discount": 1.0, "effective": 0.882726, "running_max": 0.882726, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 105.71}, {"index": 8, "tokens": 14931941, "wall_clock_seconds": 105.5, "raw": 0.885479, "normalized": 0.885479, "discount": 1.0, "effective": 0.885479, "running_max": 0.885479, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 104.91}, {"index": 9, "tokens": 17414426, "wall_clock_seconds": 95.6, "raw": 0.893871, "normalized": 0.893871, "discount": 1.0, "effective": 0.893871, "running_max": 0.893871, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 94.69}, {"index": 10, "tokens": 18712569, "wall_clock_seconds": 101.4, "raw": 0.895562, "normalized": 0.895562, "discount": 1.0, "effective": 0.895562, "running_max": 0.895562, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.72} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:mystery_box", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.34922, "area_term": 0.327269, "final_best": 0.437023, "best_raw": 0.6664, "best_effective": 0.437023, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8023061, "usage": {"input": 402, "output": 118843, "cache_read": 7672991, "cache_write": 230825}, "cost_usd": 16.5045, "cost_source": "billed", "wall_clock_seconds": 4952.7, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 560844, "wall_clock_seconds": 3.4, "raw": 0.066671, "normalized": 0.031012, "discount": 0.6, "effective": 0.018607, "running_max": 0.018607, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1240310, "wall_clock_seconds": 4.2, "raw": 0.389144, "normalized": 0.405982, "discount": 0.6, "effective": 0.243589, "running_max": 0.243589, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1806408, "wall_clock_seconds": 5.8, "raw": 0.552843, "normalized": 0.596329, "discount": 0.6, "effective": 0.357797, "running_max": 0.357797, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2458910, "wall_clock_seconds": 3.6, "raw": 0.591184, "normalized": 0.640911, "discount": 0.6, "effective": 0.384547, "running_max": 0.384547, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3675124, "wall_clock_seconds": 3.8, "raw": 0.6531, "normalized": 0.712907, "discount": 0.6, "effective": 0.427744, "running_max": 0.427744, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 5776553, "wall_clock_seconds": 3.5, "raw": 0.663258, "normalized": 0.724718, "discount": 0.6, "effective": 0.434831, "running_max": 0.434831, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 6415101, "wall_clock_seconds": 3.9, "raw": 0.663205, "normalized": 0.724657, "discount": 0.6, "effective": 0.434794, "running_max": 0.434831, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 7283486, "wall_clock_seconds": 3.7, "raw": 0.663186, "normalized": 0.724634, "discount": 0.6, "effective": 0.434781, "running_max": 0.434831, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 7463317, "wall_clock_seconds": 4.1, "raw": 0.660989, "normalized": 0.72208, "discount": 0.6, "effective": 0.433248, "running_max": 0.434831, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 8023061, "wall_clock_seconds": 3.6, "raw": 0.6664, "normalized": 0.728372, "discount": 0.6, "effective": 0.437023, "running_max": 0.437023, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:mystery_box--nohints", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.030234, "area_term": 0.029872, "final_best": 0.031681, "best_raw": 0.067245, "best_effective": 0.031681, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6011867, "usage": {"input": 225, "output": 106836, "cache_read": 5734731, "cache_write": 170075}, "cost_usd": 13.2047, "cost_source": "billed", "wall_clock_seconds": 3999.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 166555, "wall_clock_seconds": 2.8, "raw": 0.057095, "normalized": 0.019878, "discount": 1.0, "effective": 0.019878, "running_max": 0.019878, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 552847, "wall_clock_seconds": 4.3, "raw": 0.058001, "normalized": 0.020931, "discount": 1.0, "effective": 0.020931, "running_max": 0.020931, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 843278, "wall_clock_seconds": 2.5, "raw": 0.045111, "normalized": 0.005943, "discount": 1.0, "effective": 0.005943, "running_max": 0.020931, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1074383, "wall_clock_seconds": 2.9, "raw": 0.067245, "normalized": 0.031681, "discount": 1.0, "effective": 0.031681, "running_max": 0.031681, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1567997, "wall_clock_seconds": 2.9, "raw": 0.045338, "normalized": 0.006207, "discount": 1.0, "effective": 0.006207, "running_max": 0.031681, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1814098, "wall_clock_seconds": 2.9, "raw": 0.04963, "normalized": 0.011197, "discount": 1.0, "effective": 0.011197, "running_max": 0.031681, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 2579243, "wall_clock_seconds": 2.6, "raw": 0.043167, "normalized": 0.003682, "discount": 1.0, "effective": 0.003682, "running_max": 0.031681, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 3608811, "wall_clock_seconds": 4.1, "raw": 0.044566, "normalized": 0.005309, "discount": 1.0, "effective": 0.005309, "running_max": 0.031681, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5019049, "wall_clock_seconds": 3.4, "raw": 0.058679, "normalized": 0.02172, "discount": 1.0, "effective": 0.02172, "running_max": 0.031681, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 6011867, "wall_clock_seconds": 3.5, "raw": 0.063706, "normalized": 0.027565, "discount": 1.0, "effective": 0.027565, "running_max": 0.031681, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:night_school", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.749996, "area_term": 0.7247, "final_best": 0.851176, "best_raw": 0.8735, "best_effective": 0.851176, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5277421, "usage": {"input": 207, "output": 88221, "cache_read": 5036937, "cache_write": 152056}, "cost_usd": 11.3508, "cost_source": "billed", "wall_clock_seconds": 3989.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 760512, "wall_clock_seconds": 12.4, "raw": 0.4075, "normalized": 0.302941, "discount": 1.0, "effective": 0.302941, "running_max": 0.302941, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1097257, "wall_clock_seconds": 14.9, "raw": 0.816, "normalized": 0.783529, "discount": 1.0, "effective": 0.783529, "running_max": 0.783529, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1416008, "wall_clock_seconds": 14.4, "raw": 0.8645, "normalized": 0.840588, "discount": 1.0, "effective": 0.840588, "running_max": 0.840588, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2140012, "wall_clock_seconds": 17.4, "raw": 0.87, "normalized": 0.847059, "discount": 1.0, "effective": 0.847059, "running_max": 0.847059, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2233397, "wall_clock_seconds": 17.3, "raw": 0.8605, "normalized": 0.835882, "discount": 1.0, "effective": 0.835882, "running_max": 0.847059, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3283125, "wall_clock_seconds": 15.8, "raw": 0.8715, "normalized": 0.848824, "discount": 1.0, "effective": 0.848824, "running_max": 0.848824, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 3754106, "wall_clock_seconds": 16.2, "raw": 0.864, "normalized": 0.84, "discount": 1.0, "effective": 0.84, "running_max": 0.848824, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 3873075, "wall_clock_seconds": 16.3, "raw": 0.8685, "normalized": 0.845294, "discount": 1.0, "effective": 0.845294, "running_max": 0.848824, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 4530210, "wall_clock_seconds": 13.7, "raw": 0.8515, "normalized": 0.825294, "discount": 1.0, "effective": 0.825294, "running_max": 0.848824, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5277421, "wall_clock_seconds": 15.6, "raw": 0.8735, "normalized": 0.851176, "discount": 1.0, "effective": 0.851176, "running_max": 0.851176, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:reaction_rates", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.670171, "area_term": 0.605189, "final_best": 0.930099, "best_raw": 0.930099, "best_effective": 0.930099, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17249020, "usage": {"input": 777, "output": 184528, "cache_read": 16149102, "cache_write": 914613}, "cost_usd": 36.8159, "cost_source": "billed", "wall_clock_seconds": 12567.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 542106, "wall_clock_seconds": 16.5, "raw": 0.214152, "normalized": 0.214152, "discount": 1.0, "effective": 0.214152, "running_max": 0.214152, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.99}, {"index": 2, "tokens": 2490114, "wall_clock_seconds": 101.9, "raw": 0.649669, "normalized": 0.649669, "discount": 1.0, "effective": 0.649669, "running_max": 0.649669, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.32}, {"index": 3, "tokens": 3767067, "wall_clock_seconds": 142.3, "raw": 0.741098, "normalized": 0.741098, "discount": 1.0, "effective": 0.741098, "running_max": 0.741098, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 141.72}, {"index": 4, "tokens": 4028893, "wall_clock_seconds": 117.2, "raw": 0.647321, "normalized": 0.647321, "discount": 1.0, "effective": 0.647321, "running_max": 0.741098, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 116.53}, {"index": 5, "tokens": 6291856, "wall_clock_seconds": 134.8, "raw": 0.688225, "normalized": 0.688225, "discount": 1.0, "effective": 0.688225, "running_max": 0.741098, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 134.22}, {"index": 6, "tokens": 10961869, "wall_clock_seconds": 165.3, "raw": 0.782775, "normalized": 0.782775, "discount": 1.0, "effective": 0.782775, "running_max": 0.782775, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 164.77}, {"index": 7, "tokens": 11397507, "wall_clock_seconds": 151.6, "raw": 0.632034, "normalized": 0.632034, "discount": 1.0, "effective": 0.632034, "running_max": 0.782775, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 150.86}, {"index": 8, "tokens": 11838927, "wall_clock_seconds": 243.6, "raw": 0.924358, "normalized": 0.924358, "discount": 1.0, "effective": 0.924358, "running_max": 0.924358, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 243.09}, {"index": 9, "tokens": 12282824, "wall_clock_seconds": 241.0, "raw": 0.930099, "normalized": 0.930099, "discount": 1.0, "effective": 0.930099, "running_max": 0.930099, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 240.43}, {"index": 10, "tokens": 17249020, "wall_clock_seconds": 260.9, "raw": 0.925537, "normalized": 0.925537, "discount": 1.0, "effective": 0.925537, "running_max": 0.930099, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 260.3} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:reaction_rates--nohints", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.534661, "area_term": 0.481987, "final_best": 0.745358, "best_raw": 0.745358, "best_effective": 0.745358, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6853111, "usage": {"input": 180, "output": 82526, "cache_read": 6579655, "cache_write": 190750}, "cost_usd": 13.0921, "cost_source": "billed", "wall_clock_seconds": 3844.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1104394, "wall_clock_seconds": 12.4, "raw": 0.249161, "normalized": 0.249161, "discount": 1.0, "effective": 0.249161, "running_max": 0.249161, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.08}, {"index": 2, "tokens": 3321509, "wall_clock_seconds": 65.6, "raw": 0.580651, "normalized": 0.580651, "discount": 1.0, "effective": 0.580651, "running_max": 0.580651, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 65.21}, {"index": 3, "tokens": 3614079, "wall_clock_seconds": 76.4, "raw": 0.593532, "normalized": 0.593532, "discount": 1.0, "effective": 0.593532, "running_max": 0.593532, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 76.08}, {"index": 4, "tokens": 3917472, "wall_clock_seconds": 45.0, "raw": 0.385681, "normalized": 0.385681, "discount": 1.0, "effective": 0.385681, "running_max": 0.593532, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.62}, {"index": 5, "tokens": 4228432, "wall_clock_seconds": 139.8, "raw": 0.614567, "normalized": 0.614567, "discount": 1.0, "effective": 0.614567, "running_max": 0.614567, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 139.4}, {"index": 6, "tokens": 5398873, "wall_clock_seconds": 136.9, "raw": 0.672407, "normalized": 0.672407, "discount": 1.0, "effective": 0.672407, "running_max": 0.672407, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 136.56}, {"index": 7, "tokens": 5747688, "wall_clock_seconds": 156.5, "raw": 0.719246, "normalized": 0.719246, "discount": 1.0, "effective": 0.719246, "running_max": 0.719246, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 156.21}, {"index": 8, "tokens": 6104429, "wall_clock_seconds": 175.1, "raw": 0.745358, "normalized": 0.745358, "discount": 1.0, "effective": 0.745358, "running_max": 0.745358, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 174.77}, {"index": 9, "tokens": 6471627, "wall_clock_seconds": 149.2, "raw": 0.730084, "normalized": 0.730084, "discount": 1.0, "effective": 0.730084, "running_max": 0.745358, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 148.9}, {"index": 10, "tokens": 6853111, "wall_clock_seconds": 170.5, "raw": 0.744559, "normalized": 0.744559, "discount": 1.0, "effective": 0.744559, "running_max": 0.745358, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 170.13} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:scan_stitch", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.379929, "area_term": 0.35447, "final_best": 0.481766, "best_raw": 0.855711, "best_effective": 0.481766, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5820295, "usage": {"input": 306, "output": 67205, "cache_read": 5587405, "cache_write": 165379}, "cost_usd": 11.018, "cost_source": "billed", "wall_clock_seconds": 2419.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1583797, "wall_clock_seconds": 0.3, "raw": 0.764127, "normalized": 0.830159, "discount": 0.51, "effective": 0.423381, "running_max": 0.423381, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1894907, "wall_clock_seconds": 0.3, "raw": 0.779774, "normalized": 0.849717, "discount": 0.51, "effective": 0.433356, "running_max": 0.433356, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2673719, "wall_clock_seconds": 0.3, "raw": 0.842961, "normalized": 0.928701, "discount": 0.51, "effective": 0.473638, "running_max": 0.473638, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3064847, "wall_clock_seconds": 0.3, "raw": 0.845399, "normalized": 0.931748, "discount": 0.51, "effective": 0.475192, "running_max": 0.475192, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4336618, "wall_clock_seconds": 0.3, "raw": 0.853269, "normalized": 0.941586, "discount": 0.51, "effective": 0.480209, "running_max": 0.480209, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4729562, "wall_clock_seconds": 0.3, "raw": 0.84999, "normalized": 0.937487, "discount": 0.51, "effective": 0.478119, "running_max": 0.480209, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4997834, "wall_clock_seconds": 0.3, "raw": 0.855648, "normalized": 0.94456, "discount": 0.51, "effective": 0.481726, "running_max": 0.481726, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5268309, "wall_clock_seconds": 0.3, "raw": 0.849257, "normalized": 0.936571, "discount": 0.51, "effective": 0.477651, "running_max": 0.481726, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5542006, "wall_clock_seconds": 0.3, "raw": 0.855711, "normalized": 0.944639, "discount": 0.51, "effective": 0.481766, "running_max": 0.481766, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5820295, "wall_clock_seconds": 0.3, "raw": 0.84992, "normalized": 0.9374, "discount": 0.51, "effective": 0.478074, "running_max": 0.481766, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:scan_stitch--nohints", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.237464, "area_term": 0.063883, "final_best": 0.931789, "best_raw": 0.845431, "best_effective": 0.931789, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 36662391, "usage": {"input": 948, "output": 437289, "cache_read": 34596133, "cache_write": 1628021}, "cost_usd": 76.8203, "cost_source": "billed", "wall_clock_seconds": 15224.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 822703, "wall_clock_seconds": 0.2, "raw": 0.097837, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1001511, "wall_clock_seconds": 0.3, "raw": 0.088513, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2609574, "wall_clock_seconds": 0.3, "raw": 0.088433, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2922551, "wall_clock_seconds": 0.3, "raw": 0.098261, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3853678, "wall_clock_seconds": 0.3, "raw": 0.093944, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4509070, "wall_clock_seconds": 0.2, "raw": 0.087653, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5823560, "wall_clock_seconds": 0.3, "raw": 0.087653, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 10122125, "wall_clock_seconds": 0.3, "raw": 0.103161, "normalized": 0.003951, "discount": 1.0, "effective": 0.003951, "running_max": 0.003951, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 18766415, "wall_clock_seconds": 0.3, "raw": 0.097052, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.003951, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 36662391, "wall_clock_seconds": 0.3, "raw": 0.845431, "normalized": 0.931789, "discount": 1.0, "effective": 0.931789, "running_max": 0.931789, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:shapes_generalize", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.800052, "area_term": 0.783502, "final_best": 0.86625, "best_raw": 0.893, "best_effective": 0.86625, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 1777420, "usage": {"input": 132, "output": 32397, "cache_read": 1680823, "cache_write": 64068}, "cost_usd": 4.1028, "cost_source": "billed", "wall_clock_seconds": 2021.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 145786, "wall_clock_seconds": 5.2, "raw": 0.222, "normalized": 0.0275, "discount": 1.0, "effective": 0.0275, "running_max": 0.0275, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.5}, {"index": 2, "tokens": 224760, "wall_clock_seconds": 5.1, "raw": 0.226, "normalized": 0.0325, "discount": 1.0, "effective": 0.0325, "running_max": 0.0325, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.34}, {"index": 3, "tokens": 412900, "wall_clock_seconds": 97.6, "raw": 0.361, "normalized": 0.20125, "discount": 1.0, "effective": 0.20125, "running_max": 0.20125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.75}, {"index": 4, "tokens": 486195, "wall_clock_seconds": 95.7, "raw": 0.4, "normalized": 0.25, "discount": 1.0, "effective": 0.25, "running_max": 0.25, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.17}, {"index": 5, "tokens": 565043, "wall_clock_seconds": 92.7, "raw": 0.237, "normalized": 0.04625, "discount": 1.0, "effective": 0.04625, "running_max": 0.25, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 92.09}, {"index": 6, "tokens": 783362, "wall_clock_seconds": 96.6, "raw": 0.4, "normalized": 0.25, "discount": 1.0, "effective": 0.25, "running_max": 0.25, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.9}, {"index": 7, "tokens": 877399, "wall_clock_seconds": 91.6, "raw": 0.844, "normalized": 0.805, "discount": 1.0, "effective": 0.805, "running_max": 0.805, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 91.06}, {"index": 8, "tokens": 1236886, "wall_clock_seconds": 85.6, "raw": 0.857, "normalized": 0.82125, "discount": 1.0, "effective": 0.82125, "running_max": 0.82125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 85.01}, {"index": 9, "tokens": 1587175, "wall_clock_seconds": 91.0, "raw": 0.877, "normalized": 0.84625, "discount": 1.0, "effective": 0.84625, "running_max": 0.84625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 89.84}, {"index": 10, "tokens": 1777420, "wall_clock_seconds": 111.5, "raw": 0.893, "normalized": 0.86625, "discount": 1.0, "effective": 0.86625, "running_max": 0.86625, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 110.48} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:shattered_prior", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.414404, "area_term": 0.355505, "final_best": 0.65, "best_raw": 1.0, "best_effective": 0.65, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 8607226, "usage": {"input": 501, "output": 97435, "cache_read": 8214059, "cache_write": 295231}, "cost_usd": 16.7812, "cost_source": "billed", "wall_clock_seconds": 3376.9, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 1}, {"id": 3, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1363308, "wall_clock_seconds": 1.7, "raw": 0.0, "normalized": 0.0, "discount": 0.9, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3021069, "wall_clock_seconds": 1.7, "raw": 0.182708, "normalized": 0.182708, "discount": 0.65, "effective": 0.11876, "running_max": 0.11876, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3244795, "wall_clock_seconds": 1.7, "raw": 0.366033, "normalized": 0.366033, "discount": 0.65, "effective": 0.237922, "running_max": 0.237922, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3591602, "wall_clock_seconds": 1.6, "raw": 0.494334, "normalized": 0.494334, "discount": 0.65, "effective": 0.321317, "running_max": 0.321317, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3957079, "wall_clock_seconds": 1.9, "raw": 0.651398, "normalized": 0.651398, "discount": 0.65, "effective": 0.423409, "running_max": 0.423409, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4345568, "wall_clock_seconds": 1.7, "raw": 0.774682, "normalized": 0.774682, "discount": 0.65, "effective": 0.503544, "running_max": 0.503544, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4885962, "wall_clock_seconds": 1.6, "raw": 0.876369, "normalized": 0.876369, "discount": 0.65, "effective": 0.56964, "running_max": 0.56964, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 7530576, "wall_clock_seconds": 2.4, "raw": 0.950164, "normalized": 0.950164, "discount": 0.65, "effective": 0.617607, "running_max": 0.617607, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8061515, "wall_clock_seconds": 2.0, "raw": 1.0, "normalized": 1.0, "discount": 0.65, "effective": 0.65, "running_max": 0.65, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:shattered_prior--nohints", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.592923, "area_term": 0.491154, "final_best": 1.0, "best_raw": 1.0, "best_effective": 1.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 10079389, "usage": {"input": 267, "output": 93078, "cache_read": 9789231, "cache_write": 196813}, "cost_usd": 16.906, "cost_source": "billed", "wall_clock_seconds": 3048.4, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1847873, "wall_clock_seconds": 1.8, "raw": 0.158118, "normalized": 0.158118, "discount": 1.0, "effective": 0.158118, "running_max": 0.158118, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5340220, "wall_clock_seconds": 1.9, "raw": 0.651398, "normalized": 0.651398, "discount": 1.0, "effective": 0.651398, "running_max": 0.651398, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 7340199, "wall_clock_seconds": 2.2, "raw": 0.876369, "normalized": 0.876369, "discount": 1.0, "effective": 0.876369, "running_max": 0.876369, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 9494635, "wall_clock_seconds": 1.7, "raw": 1.0, "normalized": 1.0, "discount": 1.0, "effective": 1.0, "running_max": 1.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:ship_detect", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.155518, "area_term": 0.13339, "final_best": 0.244033, "best_raw": 0.244033, "best_effective": 0.244033, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 56, "max_submissions": null, "tokens_total": 55440316, "usage": {"input": 1347, "output": 436528, "cache_read": 52970671, "cache_write": 2031770}, "cost_usd": 100.2077, "cost_source": "billed", "wall_clock_seconds": 30315.8, "stop_reason": "cost_limit", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 18, "tokens": 346984, "cost_weighted": 19748.7, "wall_clock_seconds": 325.6, "raw": 0.00683, "normalized": 0.00683, "effective": 0.00683, "running_max": 0.00683, "scored": true, "final": false}, {"index": 2, "model_call": 21, "tokens": 449465, "cost_weighted": 24457.9, "wall_clock_seconds": 369.9, "raw": 0.012511, "normalized": 0.012511, "effective": 0.012511, "running_max": 0.012511, "scored": true, "final": false}, {"index": 3, "model_call": 23, "tokens": 529415, "cost_weighted": 27998.8, "wall_clock_seconds": 406.3, "raw": 0.018007, "normalized": 0.018007, "effective": 0.018007, "running_max": 0.018007, "scored": true, "final": false}, {"index": 4, "model_call": 26, "tokens": 667536, "cost_weighted": 37318.6, "wall_clock_seconds": 518.1, "raw": 0.029369, "normalized": 0.029369, "effective": 0.029369, "running_max": 0.029369, "scored": true, "final": false}, {"index": 5, "model_call": 29, "tokens": 824392, "cost_weighted": 43273.7, "wall_clock_seconds": 563.3, "raw": 0.033936, "normalized": 0.033936, "effective": 0.033936, "running_max": 0.033936, "scored": true, "final": false}, {"index": 6, "model_call": 31, "tokens": 942397, "cost_weighted": 47962.3, "wall_clock_seconds": 607.7, "raw": 0.038749, "normalized": 0.038749, "effective": 0.038749, "running_max": 0.038749, "scored": true, "final": false}, {"index": 7, "model_call": 35, "tokens": 1195797, "cost_weighted": 55900.0, "wall_clock_seconds": 682.7, "raw": 0.043601, "normalized": 0.043601, "effective": 0.043601, "running_max": 0.043601, "scored": true, "final": false}, {"index": 8, "model_call": 38, "tokens": 1407078, "cost_weighted": 63868.2, "wall_clock_seconds": 754.2, "raw": 0.048608, "normalized": 0.048608, "effective": 0.048608, "running_max": 0.048608, "scored": true, "final": false}, {"index": 9, "model_call": 39, "tokens": 1480662, "cost_weighted": 65716.3, "wall_clock_seconds": 1104.1, "raw": 0.10689, "normalized": 0.10689, "effective": 0.10689, "running_max": 0.10689, "scored": true, "final": false}, {"index": 10, "model_call": 62, "tokens": 3697260, "cost_weighted": 140409.7, "wall_clock_seconds": 1767.2, "raw": 0.130782, "normalized": 0.130782, "effective": 0.130782, "running_max": 0.130782, "scored": true, "final": false}, {"index": 11, "model_call": 66, "tokens": 4209457, "cost_weighted": 156067.8, "wall_clock_seconds": 2101.7, "raw": 0.138657, "normalized": 0.138657, "effective": 0.138657, "running_max": 0.138657, "scored": true, "final": false}, {"index": 12, "model_call": 69, "tokens": 4630085, "cost_weighted": 174034.0, "wall_clock_seconds": 2330.8, "raw": 0.137454, "normalized": 0.137454, "effective": 0.137454, "running_max": 0.138657, "scored": true, "final": false}, {"index": 13, "model_call": 75, "tokens": 5575427, "cost_weighted": 208704.7, "wall_clock_seconds": 3355.7, "raw": 0.164085, "normalized": 0.164085, "effective": 0.164085, "running_max": 0.164085, "scored": true, "final": false}, {"index": 14, "model_call": 78, "tokens": 6091032, "cost_weighted": 223398.7, "wall_clock_seconds": 3485.3, "raw": 0.111813, "normalized": 0.111813, "effective": 0.111813, "running_max": 0.164085, "scored": true, "final": false}, {"index": 15, "model_call": 87, "tokens": 7816378, "cost_weighted": 276052.7, "wall_clock_seconds": 4172.9, "raw": 0.155846, "normalized": 0.155846, "effective": 0.155846, "running_max": 0.164085, "scored": true, "final": false}, {"index": 16, "model_call": 90, "tokens": 8439111, "cost_weighted": 296583.0, "wall_clock_seconds": 4409.0, "raw": 0.161172, "normalized": 0.161172, "effective": 0.161172, "running_max": 0.164085, "scored": true, "final": false}, {"index": 17, "model_call": 94, "tokens": 9295341, "cost_weighted": 320407.6, "wall_clock_seconds": 6156.5, "raw": 0.163478, "normalized": 0.163478, "effective": 0.163478, "running_max": 0.164085, "scored": true, "final": false}, {"index": 18, "model_call": 96, "tokens": 9731894, "cost_weighted": 332143.4, "wall_clock_seconds": 6593.8, "raw": 0.149699, "normalized": 0.149699, "effective": 0.149699, "running_max": 0.164085, "scored": true, "final": false}, {"index": 19, "model_call": 98, "tokens": 10177841, "cost_weighted": 346170.0, "wall_clock_seconds": 6753.9, "raw": 0.159351, "normalized": 0.159351, "effective": 0.159351, "running_max": 0.164085, "scored": true, "final": false}, {"index": 20, "model_call": 104, "tokens": 11566904, "cost_weighted": 382151.3, "wall_clock_seconds": 7171.9, "raw": 0.185321, "normalized": 0.185321, "effective": 0.185321, "running_max": 0.185321, "scored": true, "final": false}, {"index": 21, "model_call": 113, "tokens": 12424639, "cost_weighted": 474415.1, "wall_clock_seconds": 7684.2, "raw": 0.169111, "normalized": 0.169111, "effective": 0.169111, "running_max": 0.185321, "scored": true, "final": false}, {"index": 22, "model_call": 130, "tokens": 13341186, "cost_weighted": 513367.0, "wall_clock_seconds": 8049.4, "raw": 0.191082, "normalized": 0.191082, "effective": 0.191082, "running_max": 0.191082, "scored": true, "final": false}, {"index": 23, "model_call": 133, "tokens": 13573950, "cost_weighted": 524453.3, "wall_clock_seconds": 8189.0, "raw": 0.19255, "normalized": 0.19255, "effective": 0.19255, "running_max": 0.19255, "scored": true, "final": false}, {"index": 24, "model_call": 145, "tokens": 14678326, "cost_weighted": 565555.2, "wall_clock_seconds": 8654.5, "raw": 0.182241, "normalized": 0.182241, "effective": 0.182241, "running_max": 0.19255, "scored": true, "final": false}, {"index": 25, "model_call": 146, "tokens": 14781665, "cost_weighted": 568531.1, "wall_clock_seconds": 8679.8, "raw": 0.183345, "normalized": 0.183345, "effective": 0.183345, "running_max": 0.19255, "scored": true, "final": false}, {"index": 26, "model_call": 157, "tokens": 16109985, "cost_weighted": 615194.4, "wall_clock_seconds": 9168.4, "raw": 0.192394, "normalized": 0.192394, "effective": 0.192394, "running_max": 0.19255, "scored": true, "final": false}, {"index": 27, "model_call": 165, "tokens": 17239529, "cost_weighted": 653130.2, "wall_clock_seconds": 9643.8, "raw": 0.19872, "normalized": 0.19872, "effective": 0.19872, "running_max": 0.19872, "scored": true, "final": false}, {"index": 28, "model_call": 168, "tokens": 17706705, "cost_weighted": 674235.0, "wall_clock_seconds": 9984.3, "raw": 0.198813, "normalized": 0.198813, "effective": 0.198813, "running_max": 0.198813, "scored": true, "final": false}, {"index": 29, "model_call": 173, "tokens": 18527937, "cost_weighted": 703600.1, "wall_clock_seconds": 10659.2, "raw": 0.204734, "normalized": 0.204734, "effective": 0.204734, "running_max": 0.204734, "scored": true, "final": false}, {"index": 30, "model_call": 174, "tokens": 18698354, "cost_weighted": 707895.6, "wall_clock_seconds": 11558.0, "raw": 0.203827, "normalized": 0.203827, "effective": 0.203827, "running_max": 0.204734, "scored": true, "final": false}, {"index": 31, "model_call": 187, "tokens": 21188998, "cost_weighted": 786046.2, "wall_clock_seconds": 12396.7, "raw": 0.20882, "normalized": 0.20882, "effective": 0.20882, "running_max": 0.20882, "scored": true, "final": false}, {"index": 32, "model_call": 188, "tokens": 21406142, "cost_weighted": 795535.0, "wall_clock_seconds": 12553.4, "raw": 0.204343, "normalized": 0.204343, "effective": 0.204343, "running_max": 0.20882, "scored": true, "final": false}, {"index": 33, "model_call": 195, "tokens": 23016044, "cost_weighted": 849712.8, "wall_clock_seconds": 13196.2, "raw": 0.204583, "normalized": 0.204583, "effective": 0.204583, "running_max": 0.20882, "scored": true, "final": false}, {"index": 34, "model_call": 211, "tokens": 24039097, "cost_weighted": 956080.7, "wall_clock_seconds": 13805.4, "raw": 0.204838, "normalized": 0.204838, "effective": 0.204838, "running_max": 0.20882, "scored": true, "final": false}, {"index": 35, "model_call": 236, "tokens": 26029717, "cost_weighted": 1017402.0, "wall_clock_seconds": 14294.1, "raw": 0.213402, "normalized": 0.213402, "effective": 0.213402, "running_max": 0.213402, "scored": true, "final": false}, {"index": 36, "model_call": 237, "tokens": 26130907, "cost_weighted": 1020669.0, "wall_clock_seconds": 14328.9, "raw": 0.219977, "normalized": 0.219977, "effective": 0.219977, "running_max": 0.219977, "scored": true, "final": false}, {"index": 37, "model_call": 239, "tokens": 26344466, "cost_weighted": 1030506.2, "wall_clock_seconds": 14439.6, "raw": 0.220138, "normalized": 0.220138, "effective": 0.220138, "running_max": 0.220138, "scored": true, "final": false}, {"index": 38, "model_call": 251, "tokens": 27791251, "cost_weighted": 1073973.1, "wall_clock_seconds": 14793.2, "raw": 0.220171, "normalized": 0.220171, "effective": 0.220171, "running_max": 0.220171, "scored": true, "final": false}, {"index": 39, "model_call": 254, "tokens": 28200634, "cost_weighted": 1086285.5, "wall_clock_seconds": 14903.8, "raw": 0.220149, "normalized": 0.220149, "effective": 0.220149, "running_max": 0.220171, "scored": true, "final": false}, {"index": 40, "model_call": 260, "tokens": 29068363, "cost_weighted": 1116574.7, "wall_clock_seconds": 16713.0, "raw": 0.234515, "normalized": 0.234515, "effective": 0.234515, "running_max": 0.234515, "scored": true, "final": false}, {"index": 41, "model_call": 295, "tokens": 35152996, "cost_weighted": 1268187.5, "wall_clock_seconds": 17826.3, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.234515, "scored": true, "final": false}, {"index": 42, "model_call": 299, "tokens": 35920389, "cost_weighted": 1289650.5, "wall_clock_seconds": 18066.0, "raw": 0.213606, "normalized": 0.213606, "effective": 0.213606, "running_max": 0.234515, "scored": true, "final": false}, {"index": 43, "model_call": 300, "tokens": 36117678, "cost_weighted": 1296279.3, "wall_clock_seconds": 18151.8, "raw": 0.210441, "normalized": 0.210441, "effective": 0.210441, "running_max": 0.234515, "scored": true, "final": false}, {"index": 44, "model_call": 303, "tokens": 36734842, "cost_weighted": 1322289.2, "wall_clock_seconds": 18452.9, "raw": 0.234603, "normalized": 0.234603, "effective": 0.234603, "running_max": 0.234603, "scored": true, "final": false}, {"index": 45, "model_call": 325, "tokens": 41710421, "cost_weighted": 1445275.1, "wall_clock_seconds": 20992.6, "raw": 0.234036, "normalized": 0.234036, "effective": 0.234036, "running_max": 0.234603, "scored": true, "final": false}, {"index": 46, "model_call": 328, "tokens": 42440489, "cost_weighted": 1465649.4, "wall_clock_seconds": 21143.1, "raw": 0.234966, "normalized": 0.234966, "effective": 0.234966, "running_max": 0.234966, "scored": true, "final": false}, {"index": 47, "model_call": 348, "tokens": 43232798, "cost_weighted": 1551671.2, "wall_clock_seconds": 21409.7, "raw": 0.234741, "normalized": 0.234741, "effective": 0.234741, "running_max": 0.234966, "scored": true, "final": false}, {"index": 48, "model_call": 370, "tokens": 44321246, "cost_weighted": 1581434.6, "wall_clock_seconds": 21662.0, "raw": 0.234989, "normalized": 0.234989, "effective": 0.234989, "running_max": 0.234989, "scored": true, "final": false}, {"index": 49, "model_call": 384, "tokens": 45442103, "cost_weighted": 1622521.4, "wall_clock_seconds": 22065.3, "raw": 0.235277, "normalized": 0.235277, "effective": 0.235277, "running_max": 0.235277, "scored": true, "final": false}, {"index": 50, "model_call": 386, "tokens": 45629471, "cost_weighted": 1627816.5, "wall_clock_seconds": 22120.2, "raw": 0.234915, "normalized": 0.234915, "effective": 0.234915, "running_max": 0.235277, "scored": true, "final": false}, {"index": 51, "model_call": 389, "tokens": 45922667, "cost_weighted": 1639072.8, "wall_clock_seconds": 22282.9, "raw": 0.234826, "normalized": 0.234826, "effective": 0.234826, "running_max": 0.235277, "scored": true, "final": false}, {"index": 52, "model_call": 395, "tokens": 46566482, "cost_weighted": 1664453.9, "wall_clock_seconds": 23430.9, "raw": 0.2352, "normalized": 0.2352, "effective": 0.2352, "running_max": 0.235277, "scored": true, "final": false}, {"index": 53, "model_call": 414, "tokens": 49017176, "cost_weighted": 1738322.2, "wall_clock_seconds": 24083.7, "raw": 0.235622, "normalized": 0.235622, "effective": 0.235622, "running_max": 0.235622, "scored": true, "final": false}, {"index": 54, "model_call": 424, "tokens": 50563328, "cost_weighted": 1778660.6, "wall_clock_seconds": 24348.9, "raw": 0.235943, "normalized": 0.235943, "effective": 0.235943, "running_max": 0.235943, "scored": true, "final": false}, {"index": 55, "model_call": 429, "tokens": 51400039, "cost_weighted": 1801023.1, "wall_clock_seconds": 24498.0, "raw": 0.236105, "normalized": 0.236105, "effective": 0.236105, "running_max": 0.236105, "scored": true, "final": false}, {"index": 56, "model_call": 435, "tokens": 52462084, "cost_weighted": 1834376.1, "wall_clock_seconds": 26889.6, "raw": 0.244033, "normalized": 0.244033, "effective": 0.244033, "running_max": 0.244033, "scored": true, "final": false} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:ship_tune", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.201965, "area_term": 0.180616, "final_best": 0.287358, "best_raw": 0.287358, "best_effective": 0.287358, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12963431, "usage": {"input": 563, "output": 189030, "cache_read": 12181885, "cache_write": 591953}, "cost_usd": 29.0384, "cost_source": "billed", "wall_clock_seconds": 15809.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2194055, "wall_clock_seconds": 51.0, "raw": 0.22335, "normalized": 0.22335, "discount": 1.0, "effective": 0.22335, "running_max": 0.22335, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 50.23}, {"index": 2, "tokens": 3857610, "wall_clock_seconds": 101.0, "raw": 0.252368, "normalized": 0.252368, "discount": 1.0, "effective": 0.252368, "running_max": 0.252368, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.21}, {"index": 3, "tokens": 6452002, "wall_clock_seconds": 259.1, "raw": 0.260597, "normalized": 0.260597, "discount": 1.0, "effective": 0.260597, "running_max": 0.260597, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 258.04}, {"index": 4, "tokens": 9893237, "wall_clock_seconds": 261.9, "raw": 0.242706, "normalized": 0.242706, "discount": 1.0, "effective": 0.242706, "running_max": 0.260597, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 259.06}, {"index": 5, "tokens": 10023173, "wall_clock_seconds": 241.7, "raw": 0.276411, "normalized": 0.276411, "discount": 1.0, "effective": 0.276411, "running_max": 0.276411, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 240.65}, {"index": 6, "tokens": 10226805, "wall_clock_seconds": 282.7, "raw": 0.284246, "normalized": 0.284246, "discount": 1.0, "effective": 0.284246, "running_max": 0.284246, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 281.32}, {"index": 7, "tokens": 11592168, "wall_clock_seconds": 269.2, "raw": 0.285576, "normalized": 0.285576, "discount": 1.0, "effective": 0.285576, "running_max": 0.285576, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 268.28}, {"index": 8, "tokens": 12092837, "wall_clock_seconds": 343.6, "raw": 0.287358, "normalized": 0.287358, "discount": 1.0, "effective": 0.287358, "running_max": 0.287358, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 342.66}, {"index": 9, "tokens": 12628549, "wall_clock_seconds": 384.8, "raw": 0.285767, "normalized": 0.285767, "discount": 1.0, "effective": 0.285767, "running_max": 0.287358, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 384.0}, {"index": 10, "tokens": 12963431, "wall_clock_seconds": 323.2, "raw": 0.286229, "normalized": 0.286229, "discount": 1.0, "effective": 0.286229, "running_max": 0.287358, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 322.43} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:splash_generalize", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.770616, "area_term": 0.740215, "final_best": 0.892222, "best_raw": 0.903, "best_effective": 0.892222, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7520203, "usage": {"input": 249, "output": 120511, "cache_read": 7221080, "cache_write": 178363}, "cost_usd": 15.4787, "cost_source": "billed", "wall_clock_seconds": 4750.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 96725, "wall_clock_seconds": 6.2, "raw": 0.107, "normalized": 0.007778, "discount": 1.0, "effective": 0.007778, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.55}, {"index": 2, "tokens": 142074, "wall_clock_seconds": 6.0, "raw": 0.09, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.53}, {"index": 3, "tokens": 219066, "wall_clock_seconds": 6.3, "raw": 0.117, "normalized": 0.018889, "discount": 1.0, "effective": 0.018889, "running_max": 0.018889, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.7}, {"index": 4, "tokens": 411449, "wall_clock_seconds": 3.1, "raw": 0.266, "normalized": 0.184444, "discount": 1.0, "effective": 0.184444, "running_max": 0.184444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.46}, {"index": 5, "tokens": 1214687, "wall_clock_seconds": 59.3, "raw": 0.841, "normalized": 0.823333, "discount": 1.0, "effective": 0.823333, "running_max": 0.823333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.68}, {"index": 6, "tokens": 1947240, "wall_clock_seconds": 83.6, "raw": 0.842, "normalized": 0.824444, "discount": 1.0, "effective": 0.824444, "running_max": 0.824444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 82.94}, {"index": 7, "tokens": 2639960, "wall_clock_seconds": 88.0, "raw": 0.881, "normalized": 0.867778, "discount": 1.0, "effective": 0.867778, "running_max": 0.867778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 87.4}, {"index": 8, "tokens": 3080454, "wall_clock_seconds": 91.8, "raw": 0.879, "normalized": 0.865556, "discount": 1.0, "effective": 0.865556, "running_max": 0.867778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 91.23}, {"index": 9, "tokens": 5657766, "wall_clock_seconds": 84.0, "raw": 0.903, "normalized": 0.892222, "discount": 1.0, "effective": 0.892222, "running_max": 0.892222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 83.34}, {"index": 10, "tokens": 7520203, "wall_clock_seconds": 89.2, "raw": 0.903, "normalized": 0.892222, "discount": 1.0, "effective": 0.892222, "running_max": 0.892222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 88.51} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:tod_pipeline--nohints", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.199426, "area_term": 0.182091, "final_best": 0.268768, "best_raw": 0.268768, "best_effective": 0.268768, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6073941, "usage": {"input": 213, "output": 74422, "cache_read": 5860576, "cache_write": 138730}, "cost_usd": 11.3179, "cost_source": "billed", "wall_clock_seconds": 3902.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 515818, "wall_clock_seconds": 26.8, "raw": 0.071639, "normalized": 0.071639, "discount": 1.0, "effective": 0.071639, "running_max": 0.071639, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.49}, {"index": 2, "tokens": 1353947, "wall_clock_seconds": 51.6, "raw": 0.111359, "normalized": 0.111359, "discount": 1.0, "effective": 0.111359, "running_max": 0.111359, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 51.32}, {"index": 3, "tokens": 1757612, "wall_clock_seconds": 126.1, "raw": 0.109409, "normalized": 0.109409, "discount": 1.0, "effective": 0.109409, "running_max": 0.111359, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 125.85}, {"index": 4, "tokens": 3271965, "wall_clock_seconds": 117.3, "raw": 0.098885, "normalized": 0.098885, "discount": 1.0, "effective": 0.098885, "running_max": 0.111359, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 117.0}, {"index": 5, "tokens": 3843905, "wall_clock_seconds": 291.8, "raw": 0.111945, "normalized": 0.111945, "discount": 1.0, "effective": 0.111945, "running_max": 0.111945, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 291.45}, {"index": 6, "tokens": 4845198, "wall_clock_seconds": 152.1, "raw": 0.268768, "normalized": 0.268768, "discount": 1.0, "effective": 0.268768, "running_max": 0.268768, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 151.8}, {"index": 7, "tokens": 5112901, "wall_clock_seconds": 116.9, "raw": 0.264592, "normalized": 0.264592, "discount": 1.0, "effective": 0.264592, "running_max": 0.268768, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 116.6}, {"index": 8, "tokens": 5521887, "wall_clock_seconds": 151.6, "raw": 0.243098, "normalized": 0.243098, "discount": 1.0, "effective": 0.243098, "running_max": 0.268768, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 151.34}, {"index": 9, "tokens": 5796536, "wall_clock_seconds": 150.0, "raw": 0.268768, "normalized": 0.268768, "discount": 1.0, "effective": 0.268768, "running_max": 0.268768, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 149.73}, {"index": 10, "tokens": 6073941, "wall_clock_seconds": 147.7, "raw": 0.248184, "normalized": 0.248184, "discount": 1.0, "effective": 0.248184, "running_max": 0.268768, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 147.45} ]}, {"sample_key": "GSxqVwUVA6yjELAnWnNMZx:weirdml_bonanza", "eval_id": "GSxqVwUVA6yjELAnWnNMZx", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "07ab9da", "source_log_sha256": "df7953bace249a2dc112abc8e3ea18870826bdf03ad8446e4a18fd65d694573f", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.671234, "area_term": 0.613221, "final_best": 0.903286, "best_raw": 0.903286, "best_effective": 0.903286, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11535220, "usage": {"input": 267, "output": 150788, "cache_read": 11144125, "cache_write": 240040}, "cost_usd": 21.6867, "cost_source": "billed", "wall_clock_seconds": 5460.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1943110, "wall_clock_seconds": 69.7, "raw": 0.427976, "normalized": 0.427976, "discount": 1.0, "effective": 0.427976, "running_max": 0.427976, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 69.13}, {"index": 2, "tokens": 2131484, "wall_clock_seconds": 89.7, "raw": 0.799163, "normalized": 0.799163, "discount": 1.0, "effective": 0.799163, "running_max": 0.799163, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 89.2}, {"index": 3, "tokens": 2995998, "wall_clock_seconds": 67.7, "raw": 0.845039, "normalized": 0.845039, "discount": 1.0, "effective": 0.845039, "running_max": 0.845039, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 67.04}, {"index": 4, "tokens": 4121083, "wall_clock_seconds": 108.5, "raw": 0.848869, "normalized": 0.848869, "discount": 1.0, "effective": 0.848869, "running_max": 0.848869, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.86}, {"index": 5, "tokens": 4670719, "wall_clock_seconds": 85.4, "raw": 0.886239, "normalized": 0.886239, "discount": 1.0, "effective": 0.886239, "running_max": 0.886239, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 84.67}, {"index": 6, "tokens": 5269680, "wall_clock_seconds": 80.5, "raw": 0.892545, "normalized": 0.892545, "discount": 1.0, "effective": 0.892545, "running_max": 0.892545, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 79.94}, {"index": 7, "tokens": 6403448, "wall_clock_seconds": 81.0, "raw": 0.901216, "normalized": 0.901216, "discount": 1.0, "effective": 0.901216, "running_max": 0.901216, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 80.46}, {"index": 8, "tokens": 8205067, "wall_clock_seconds": 108.0, "raw": 0.902798, "normalized": 0.902798, "discount": 1.0, "effective": 0.902798, "running_max": 0.902798, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.44}, {"index": 9, "tokens": 9895891, "wall_clock_seconds": 101.5, "raw": 0.90148, "normalized": 0.90148, "discount": 1.0, "effective": 0.90148, "running_max": 0.902798, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.02}, {"index": 10, "tokens": 11535220, "wall_clock_seconds": 98.3, "raw": 0.903286, "normalized": 0.903286, "discount": 1.0, "effective": 0.903286, "running_max": 0.903286, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 97.64} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:mystery_box", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.344609, "area_term": 0.322185, "final_best": 0.434307, "best_raw": 0.662507, "best_effective": 0.434307, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7063619, "usage": {"input": 384, "output": 95454, "cache_read": 6750503, "cache_write": 217278}, "cost_usd": 14.243, "cost_source": "billed", "wall_clock_seconds": 3791.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1445759, "wall_clock_seconds": 9.2, "raw": 0.530087, "normalized": 0.569869, "discount": 0.6, "effective": 0.341921, "running_max": 0.341921, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2043643, "wall_clock_seconds": 4.1, "raw": 0.588014, "normalized": 0.637226, "discount": 0.6, "effective": 0.382335, "running_max": 0.382335, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2817225, "wall_clock_seconds": 4.6, "raw": 0.613228, "normalized": 0.666544, "discount": 0.6, "effective": 0.399926, "running_max": 0.399926, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3243580, "wall_clock_seconds": 3.6, "raw": 0.654046, "normalized": 0.714007, "discount": 0.6, "effective": 0.428404, "running_max": 0.428404, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4108444, "wall_clock_seconds": 8.6, "raw": 0.661658, "normalized": 0.722859, "discount": 0.6, "effective": 0.433715, "running_max": 0.433715, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 5570504, "wall_clock_seconds": 3.4, "raw": 0.659897, "normalized": 0.72081, "discount": 0.6, "effective": 0.432486, "running_max": 0.433715, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 6056756, "wall_clock_seconds": 3.6, "raw": 0.659652, "normalized": 0.720525, "discount": 0.6, "effective": 0.432315, "running_max": 0.433715, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6388432, "wall_clock_seconds": 3.6, "raw": 0.661616, "normalized": 0.72281, "discount": 0.6, "effective": 0.433686, "running_max": 0.433715, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6554800, "wall_clock_seconds": 3.7, "raw": 0.662004, "normalized": 0.723261, "discount": 0.6, "effective": 0.433956, "running_max": 0.433956, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7063619, "wall_clock_seconds": 3.7, "raw": 0.662507, "normalized": 0.723845, "discount": 0.6, "effective": 0.434307, "running_max": 0.434307, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:mystery_box--nohints", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.031727, "area_term": 0.031475, "final_best": 0.032736, "best_raw": 0.068153, "best_effective": 0.032736, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3032142, "usage": {"input": 156, "output": 73388, "cache_read": 2832245, "cache_write": 126353}, "cost_usd": 8.0826, "cost_source": "billed", "wall_clock_seconds": 2468.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 287550, "wall_clock_seconds": 2.7, "raw": 0.057686, "normalized": 0.020565, "discount": 1.0, "effective": 0.020565, "running_max": 0.020565, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 352866, "wall_clock_seconds": 3.1, "raw": 0.065382, "normalized": 0.029513, "discount": 1.0, "effective": 0.029513, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 691067, "wall_clock_seconds": 2.5, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 740078, "wall_clock_seconds": 2.7, "raw": 0.061887, "normalized": 0.02545, "discount": 1.0, "effective": 0.02545, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1086715, "wall_clock_seconds": 2.8, "raw": 0.045501, "normalized": 0.006397, "discount": 1.0, "effective": 0.006397, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1290531, "wall_clock_seconds": 3.8, "raw": 0.054654, "normalized": 0.017039, "discount": 1.0, "effective": 0.017039, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1362977, "wall_clock_seconds": 2.5, "raw": 0.055573, "normalized": 0.018108, "discount": 1.0, "effective": 0.018108, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 1934578, "wall_clock_seconds": 3.0, "raw": 0.054404, "normalized": 0.016749, "discount": 1.0, "effective": 0.016749, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2325253, "wall_clock_seconds": 2.6, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.029513, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 3032142, "wall_clock_seconds": 1.8, "raw": 0.068153, "normalized": 0.032736, "discount": 1.0, "effective": 0.032736, "running_max": 0.032736, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:night_school", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.781869, "area_term": 0.761453, "final_best": 0.863529, "best_raw": 0.884, "best_effective": 0.863529, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 4493684, "usage": {"input": 183, "output": 67883, "cache_read": 4011861, "cache_write": 413757}, "cost_usd": 12.5798, "cost_source": "billed", "wall_clock_seconds": 16995.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 714173, "wall_clock_seconds": 12.6, "raw": 0.463, "normalized": 0.368235, "discount": 1.0, "effective": 0.368235, "running_max": 0.368235, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 823476, "wall_clock_seconds": 13.5, "raw": 0.7005, "normalized": 0.647647, "discount": 1.0, "effective": 0.647647, "running_max": 0.647647, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1128626, "wall_clock_seconds": 11.3, "raw": 0.858, "normalized": 0.832941, "discount": 1.0, "effective": 0.832941, "running_max": 0.832941, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1737499, "wall_clock_seconds": 10.1, "raw": 0.869, "normalized": 0.845882, "discount": 1.0, "effective": 0.845882, "running_max": 0.845882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2598180, "wall_clock_seconds": 15.6, "raw": 0.869, "normalized": 0.845882, "discount": 1.0, "effective": 0.845882, "running_max": 0.845882, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 2922296, "wall_clock_seconds": 13.4, "raw": 0.8765, "normalized": 0.854706, "discount": 1.0, "effective": 0.854706, "running_max": 0.854706, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 3031444, "wall_clock_seconds": 12.5, "raw": 0.875, "normalized": 0.852941, "discount": 1.0, "effective": 0.852941, "running_max": 0.854706, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 3369861, "wall_clock_seconds": 12.5, "raw": 0.884, "normalized": 0.863529, "discount": 1.0, "effective": 0.863529, "running_max": 0.863529, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 3484139, "wall_clock_seconds": 12.6, "raw": 0.842, "normalized": 0.814118, "discount": 1.0, "effective": 0.814118, "running_max": 0.863529, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 4493684, "wall_clock_seconds": 12.4, "raw": 0.884, "normalized": 0.863529, "discount": 1.0, "effective": 0.863529, "running_max": 0.863529, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:reaction_rates", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.249216, "area_term": 0.214475, "final_best": 0.388181, "best_raw": 0.990257, "best_effective": 0.388181, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17843695, "usage": {"input": 786, "output": 209861, "cache_read": 16871282, "cache_write": 761766}, "cost_usd": 36.8943, "cost_source": "billed", "wall_clock_seconds": 12015.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.7, "before_submission": 1}, {"id": 4, "multiplier": 0.8, "before_submission": 1}, {"id": 1, "multiplier": 0.7, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1304564, "wall_clock_seconds": 13.9, "raw": 0.036803, "normalized": 0.036803, "discount": 0.56, "effective": 0.02061, "running_max": 0.02061, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 13.53}, {"index": 2, "tokens": 2607193, "wall_clock_seconds": 20.6, "raw": 0.373252, "normalized": 0.373252, "discount": 0.56, "effective": 0.209021, "running_max": 0.209021, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 20.2}, {"index": 3, "tokens": 6129084, "wall_clock_seconds": 90.5, "raw": 0.923665, "normalized": 0.923665, "discount": 0.392, "effective": 0.362077, "running_max": 0.362077, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 90.14}, {"index": 4, "tokens": 9092157, "wall_clock_seconds": 128.0, "raw": 0.930804, "normalized": 0.930804, "discount": 0.392, "effective": 0.364875, "running_max": 0.364875, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 127.74}, {"index": 5, "tokens": 11182376, "wall_clock_seconds": 273.9, "raw": 0.932439, "normalized": 0.932439, "discount": 0.392, "effective": 0.365516, "running_max": 0.365516, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 273.57}, {"index": 6, "tokens": 13881506, "wall_clock_seconds": 171.4, "raw": 0.990032, "normalized": 0.990032, "discount": 0.392, "effective": 0.388092, "running_max": 0.388092, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 171.1}, {"index": 7, "tokens": 14662023, "wall_clock_seconds": 260.3, "raw": 0.990127, "normalized": 0.990127, "discount": 0.392, "effective": 0.38813, "running_max": 0.38813, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 260.0}, {"index": 8, "tokens": 15528364, "wall_clock_seconds": 432.7, "raw": 0.990231, "normalized": 0.990231, "discount": 0.392, "effective": 0.38817, "running_max": 0.38817, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 432.45}, {"index": 9, "tokens": 16102763, "wall_clock_seconds": 446.2, "raw": 0.990256, "normalized": 0.990256, "discount": 0.392, "effective": 0.388181, "running_max": 0.388181, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 445.86}, {"index": 10, "tokens": 17843695, "wall_clock_seconds": 484.1, "raw": 0.990257, "normalized": 0.990257, "discount": 0.392, "effective": 0.388181, "running_max": 0.388181, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 483.82} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:reaction_rates--nohints", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.425237, "area_term": 0.365794, "final_best": 0.663007, "best_raw": 0.663007, "best_effective": 0.663007, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11674685, "usage": {"input": 252, "output": 111509, "cache_read": 11332895, "cache_write": 230029}, "cost_usd": 19.7862, "cost_source": "billed", "wall_clock_seconds": 5242.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 711355, "wall_clock_seconds": 22.9, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.49}, {"index": 2, "tokens": 1861070, "wall_clock_seconds": 37.5, "raw": 0.276066, "normalized": 0.276066, "discount": 1.0, "effective": 0.276066, "running_max": 0.276066, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 37.24}, {"index": 3, "tokens": 5997164, "wall_clock_seconds": 119.3, "raw": 0.519844, "normalized": 0.519844, "discount": 1.0, "effective": 0.519844, "running_max": 0.519844, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 118.97}, {"index": 4, "tokens": 7844405, "wall_clock_seconds": 135.9, "raw": 0.555144, "normalized": 0.555144, "discount": 1.0, "effective": 0.555144, "running_max": 0.555144, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 135.56}, {"index": 5, "tokens": 8236654, "wall_clock_seconds": 185.8, "raw": 0.65522, "normalized": 0.65522, "discount": 1.0, "effective": 0.65522, "running_max": 0.65522, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 185.53}, {"index": 6, "tokens": 8638546, "wall_clock_seconds": 189.5, "raw": 0.642547, "normalized": 0.642547, "discount": 1.0, "effective": 0.642547, "running_max": 0.65522, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 189.14}, {"index": 7, "tokens": 9047514, "wall_clock_seconds": 188.7, "raw": 0.662547, "normalized": 0.662547, "discount": 1.0, "effective": 0.662547, "running_max": 0.662547, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 188.41}, {"index": 8, "tokens": 9465087, "wall_clock_seconds": 297.0, "raw": 0.54989, "normalized": 0.54989, "discount": 1.0, "effective": 0.54989, "running_max": 0.662547, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 296.7}, {"index": 9, "tokens": 9889460, "wall_clock_seconds": 189.7, "raw": 0.642478, "normalized": 0.642478, "discount": 1.0, "effective": 0.642478, "running_max": 0.662547, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 189.44}, {"index": 10, "tokens": 11674685, "wall_clock_seconds": 195.4, "raw": 0.663007, "normalized": 0.663007, "discount": 1.0, "effective": 0.663007, "running_max": 0.663007, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 195.04} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:scan_stitch", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.288414, "area_term": 0.249139, "final_best": 0.445515, "best_raw": 0.798847, "best_effective": 0.445515, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8386960, "usage": {"input": 378, "output": 94531, "cache_read": 8005145, "cache_write": 286906}, "cost_usd": 16.3218, "cost_source": "billed", "wall_clock_seconds": 3758.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2087610, "wall_clock_seconds": 0.3, "raw": 0.10775, "normalized": 0.009688, "discount": 0.51, "effective": 0.004941, "running_max": 0.004941, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3480494, "wall_clock_seconds": 0.3, "raw": 0.564305, "normalized": 0.580382, "discount": 0.51, "effective": 0.295995, "running_max": 0.295995, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4322428, "wall_clock_seconds": 0.3, "raw": 0.751843, "normalized": 0.814803, "discount": 0.51, "effective": 0.41555, "running_max": 0.41555, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 5963541, "wall_clock_seconds": 0.3, "raw": 0.795708, "normalized": 0.869635, "discount": 0.51, "effective": 0.443514, "running_max": 0.443514, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 6316664, "wall_clock_seconds": 0.3, "raw": 0.796163, "normalized": 0.870204, "discount": 0.51, "effective": 0.443804, "running_max": 0.443804, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 6493770, "wall_clock_seconds": 0.3, "raw": 0.793642, "normalized": 0.867053, "discount": 0.51, "effective": 0.442197, "running_max": 0.443804, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 7424045, "wall_clock_seconds": 0.4, "raw": 0.798847, "normalized": 0.873559, "discount": 0.51, "effective": 0.445515, "running_max": 0.445515, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 7807048, "wall_clock_seconds": 0.4, "raw": 0.79765, "normalized": 0.872062, "discount": 0.51, "effective": 0.444752, "running_max": 0.445515, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 7998927, "wall_clock_seconds": 0.3, "raw": 0.791924, "normalized": 0.864906, "discount": 0.51, "effective": 0.441102, "running_max": 0.445515, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 8386960, "wall_clock_seconds": 0.2, "raw": 0.798625, "normalized": 0.873281, "discount": 0.51, "effective": 0.445373, "running_max": 0.445515, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:scan_stitch--nohints", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.006602, "area_term": 0.00558, "final_best": 0.010693, "best_raw": 0.108554, "best_effective": 0.010693, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12507357, "usage": {"input": 390, "output": 142066, "cache_read": 12039464, "cache_write": 325437}, "cost_usd": 23.2146, "cost_source": "billed", "wall_clock_seconds": 5049.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 717391, "wall_clock_seconds": 0.3, "raw": 0.064939, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1309809, "wall_clock_seconds": 0.4, "raw": 0.083199, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2160484, "wall_clock_seconds": 0.3, "raw": 0.072632, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2565390, "wall_clock_seconds": 0.3, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4280517, "wall_clock_seconds": 0.3, "raw": 0.08233, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4521872, "wall_clock_seconds": 0.3, "raw": 0.108554, "normalized": 0.010693, "discount": 1.0, "effective": 0.010693, "running_max": 0.010693, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 6737759, "wall_clock_seconds": 0.4, "raw": 0.088073, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.010693, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 7362990, "wall_clock_seconds": 0.4, "raw": 0.079606, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.010693, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 9634562, "wall_clock_seconds": 0.3, "raw": 0.099428, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.010693, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 12507357, "wall_clock_seconds": 0.3, "raw": 0.103243, "normalized": 0.004054, "discount": 1.0, "effective": 0.004054, "running_max": 0.010693, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:shapes_generalize", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.728642, "area_term": 0.708927, "final_best": 0.8075, "best_raw": 0.846, "best_effective": 0.8075, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6103381, "usage": {"input": 321, "output": 98890, "cache_read": 5828949, "cache_write": 175221}, "cost_usd": 12.9669, "cost_source": "billed", "wall_clock_seconds": 6075.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 146432, "wall_clock_seconds": 7.2, "raw": 0.195, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 6.55}, {"index": 2, "tokens": 264382, "wall_clock_seconds": 98.2, "raw": 0.186, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 96.4}, {"index": 3, "tokens": 508551, "wall_clock_seconds": 99.4, "raw": 0.24, "normalized": 0.05, "discount": 1.0, "effective": 0.05, "running_max": 0.05, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 97.82}, {"index": 4, "tokens": 712051, "wall_clock_seconds": 90.7, "raw": 0.491, "normalized": 0.36375, "discount": 1.0, "effective": 0.36375, "running_max": 0.36375, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 89.65}, {"index": 5, "tokens": 846444, "wall_clock_seconds": 92.9, "raw": 0.784, "normalized": 0.73, "discount": 1.0, "effective": 0.73, "running_max": 0.73, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 90.81}, {"index": 6, "tokens": 1310454, "wall_clock_seconds": 92.7, "raw": 0.783, "normalized": 0.72875, "discount": 1.0, "effective": 0.72875, "running_max": 0.73, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 92.01}, {"index": 7, "tokens": 2160002, "wall_clock_seconds": 114.4, "raw": 0.762, "normalized": 0.7025, "discount": 1.0, "effective": 0.7025, "running_max": 0.73, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 113.69}, {"index": 8, "tokens": 2655533, "wall_clock_seconds": 96.4, "raw": 0.818, "normalized": 0.7725, "discount": 1.0, "effective": 0.7725, "running_max": 0.7725, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.74}, {"index": 9, "tokens": 4495978, "wall_clock_seconds": 62.9, "raw": 0.842, "normalized": 0.8025, "discount": 1.0, "effective": 0.8025, "running_max": 0.8025, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 62.16}, {"index": 10, "tokens": 6103381, "wall_clock_seconds": 59.2, "raw": 0.846, "normalized": 0.8075, "discount": 1.0, "effective": 0.8075, "running_max": 0.8075, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 58.39} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:shattered_prior", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.661358, "area_term": 0.601697, "final_best": 0.9, "best_raw": 1.0, "best_effective": 0.9, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 3784558, "usage": {"input": 231, "output": 53874, "cache_read": 3570624, "cache_write": 159829}, "cost_usd": 8.2645, "cost_source": "billed", "wall_clock_seconds": 1741.6, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2015756, "wall_clock_seconds": 1.7, "raw": 0.651481, "normalized": 0.651481, "discount": 0.9, "effective": 0.586333, "running_max": 0.586333, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2715791, "wall_clock_seconds": 1.8, "raw": 0.876369, "normalized": 0.876369, "discount": 0.9, "effective": 0.788732, "running_max": 0.788732, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3415488, "wall_clock_seconds": 1.9, "raw": 1.0, "normalized": 1.0, "discount": 0.9, "effective": 0.9, "running_max": 0.9, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:shattered_prior--nohints", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.708275, "area_term": 0.635344, "final_best": 1.0, "best_raw": 1.0, "best_effective": 1.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 15182297, "usage": {"input": 348, "output": 108398, "cache_read": 14858950, "cache_write": 214601}, "cost_usd": 22.9648, "cost_source": "billed", "wall_clock_seconds": 4969.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 843259, "wall_clock_seconds": 1.7, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1622479, "wall_clock_seconds": 2.0, "raw": 0.182708, "normalized": 0.182708, "discount": 1.0, "effective": 0.182708, "running_max": 0.182708, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2084210, "wall_clock_seconds": 2.2, "raw": 0.494334, "normalized": 0.494334, "discount": 1.0, "effective": 0.494334, "running_max": 0.494334, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2489742, "wall_clock_seconds": 1.7, "raw": 0.651398, "normalized": 0.651398, "discount": 1.0, "effective": 0.651398, "running_max": 0.651398, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3192441, "wall_clock_seconds": 1.7, "raw": 0.876369, "normalized": 0.876369, "discount": 1.0, "effective": 0.876369, "running_max": 0.876369, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8250153, "wall_clock_seconds": 2.0, "raw": 0.890851, "normalized": 0.890851, "discount": 1.0, "effective": 0.890851, "running_max": 0.890851, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 10258280, "wall_clock_seconds": 2.0, "raw": 0.945457, "normalized": 0.945457, "discount": 1.0, "effective": 0.945457, "running_max": 0.945457, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 12644330, "wall_clock_seconds": 1.8, "raw": 0.997525, "normalized": 0.997525, "discount": 1.0, "effective": 0.997525, "running_max": 0.997525, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 14113582, "wall_clock_seconds": 1.8, "raw": 0.999893, "normalized": 0.999893, "discount": 1.0, "effective": 0.999893, "running_max": 0.999893, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 15182297, "wall_clock_seconds": 1.9, "raw": 1.0, "normalized": 1.0, "discount": 1.0, "effective": 1.0, "running_max": 1.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:ship_detect", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.116634, "area_term": 0.097219, "final_best": 0.194293, "best_raw": 0.194293, "best_effective": 0.194293, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 32, "max_submissions": null, "tokens_total": 51739088, "usage": {"input": 1242, "output": 468959, "cache_read": 49070889, "cache_write": 2197998}, "cost_usd": 100.0062, "cost_source": "billed", "wall_clock_seconds": 30241.9, "stop_reason": "cost_limit", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 29, "tokens": 879305, "cost_weighted": 41776.4, "wall_clock_seconds": 587.5, "raw": 0.004064, "normalized": 0.004064, "effective": 0.004064, "running_max": 0.004064, "scored": true, "final": false}, {"index": 2, "model_call": 53, "tokens": 2631009, "cost_weighted": 97485.5, "wall_clock_seconds": 1158.9, "raw": 0.092538, "normalized": 0.092538, "effective": 0.092538, "running_max": 0.092538, "scored": true, "final": false}, {"index": 3, "model_call": 62, "tokens": 3493201, "cost_weighted": 124363.7, "wall_clock_seconds": 2006.4, "raw": 0.113693, "normalized": 0.113693, "effective": 0.113693, "running_max": 0.113693, "scored": true, "final": false}, {"index": 4, "model_call": 68, "tokens": 4138523, "cost_weighted": 147626.3, "wall_clock_seconds": 2323.4, "raw": 0.121345, "normalized": 0.121345, "effective": 0.121345, "running_max": 0.121345, "scored": true, "final": false}, {"index": 5, "model_call": 77, "tokens": 5234394, "cost_weighted": 182911.8, "wall_clock_seconds": 2745.4, "raw": 0.124438, "normalized": 0.124438, "effective": 0.124438, "running_max": 0.124438, "scored": true, "final": false}, {"index": 6, "model_call": 82, "tokens": 5902619, "cost_weighted": 203322.5, "wall_clock_seconds": 3233.8, "raw": 0.127103, "normalized": 0.127103, "effective": 0.127103, "running_max": 0.127103, "scored": true, "final": false}, {"index": 7, "model_call": 95, "tokens": 7816141, "cost_weighted": 256379.8, "wall_clock_seconds": 3646.8, "raw": 0.142713, "normalized": 0.142713, "effective": 0.142713, "running_max": 0.142713, "scored": true, "final": false}, {"index": 8, "model_call": 109, "tokens": 10318677, "cost_weighted": 345777.5, "wall_clock_seconds": 4715.8, "raw": 0.132543, "normalized": 0.132543, "effective": 0.132543, "running_max": 0.142713, "scored": true, "final": false}, {"index": 9, "model_call": 123, "tokens": 13294369, "cost_weighted": 439909.9, "wall_clock_seconds": 5734.2, "raw": 0.140424, "normalized": 0.140424, "effective": 0.140424, "running_max": 0.142713, "scored": true, "final": false}, {"index": 10, "model_call": 124, "tokens": 13525620, "cost_weighted": 448036.0, "wall_clock_seconds": 5817.0, "raw": 0.15274, "normalized": 0.15274, "effective": 0.15274, "running_max": 0.15274, "scored": true, "final": false}, {"index": 11, "model_call": 153, "tokens": 15783985, "cost_weighted": 589356.8, "wall_clock_seconds": 6701.2, "raw": 0.15315, "normalized": 0.15315, "effective": 0.15315, "running_max": 0.15315, "scored": true, "final": false}, {"index": 12, "model_call": 183, "tokens": 18506391, "cost_weighted": 675313.3, "wall_clock_seconds": 7461.2, "raw": 0.164771, "normalized": 0.164771, "effective": 0.164771, "running_max": 0.164771, "scored": true, "final": false}, {"index": 13, "model_call": 187, "tokens": 18989231, "cost_weighted": 689450.3, "wall_clock_seconds": 7583.5, "raw": 0.167885, "normalized": 0.167885, "effective": 0.167885, "running_max": 0.167885, "scored": true, "final": false}, {"index": 14, "model_call": 201, "tokens": 20953305, "cost_weighted": 752509.2, "wall_clock_seconds": 8183.5, "raw": 0.16888, "normalized": 0.16888, "effective": 0.16888, "running_max": 0.16888, "scored": true, "final": false}, {"index": 15, "model_call": 206, "tokens": 21740920, "cost_weighted": 775584.5, "wall_clock_seconds": 8391.8, "raw": 0.170696, "normalized": 0.170696, "effective": 0.170696, "running_max": 0.170696, "scored": true, "final": false}, {"index": 16, "model_call": 240, "tokens": 28431475, "cost_weighted": 968892.0, "wall_clock_seconds": 10392.8, "raw": 0.170871, "normalized": 0.170871, "effective": 0.170871, "running_max": 0.170871, "scored": true, "final": false}, {"index": 17, "model_call": 264, "tokens": 30403277, "cost_weighted": 1105650.3, "wall_clock_seconds": 11298.6, "raw": 0.171689, "normalized": 0.171689, "effective": 0.171689, "running_max": 0.171689, "scored": true, "final": false}, {"index": 18, "model_call": 273, "tokens": 31082582, "cost_weighted": 1127915.2, "wall_clock_seconds": 11514.3, "raw": 0.171769, "normalized": 0.171769, "effective": 0.171769, "running_max": 0.171769, "scored": true, "final": false}, {"index": 19, "model_call": 275, "tokens": 31257809, "cost_weighted": 1133456.7, "wall_clock_seconds": 11569.4, "raw": 0.171905, "normalized": 0.171905, "effective": 0.171905, "running_max": 0.171905, "scored": true, "final": false}, {"index": 20, "model_call": 279, "tokens": 31639576, "cost_weighted": 1148190.4, "wall_clock_seconds": 11758.0, "raw": 0.172081, "normalized": 0.172081, "effective": 0.172081, "running_max": 0.172081, "scored": true, "final": false}, {"index": 21, "model_call": 283, "tokens": 32061410, "cost_weighted": 1162930.7, "wall_clock_seconds": 11896.0, "raw": 0.172161, "normalized": 0.172161, "effective": 0.172161, "running_max": 0.172161, "scored": true, "final": false}, {"index": 22, "model_call": 288, "tokens": 32640119, "cost_weighted": 1182367.4, "wall_clock_seconds": 12095.1, "raw": 0.172204, "normalized": 0.172204, "effective": 0.172204, "running_max": 0.172204, "scored": true, "final": false}, {"index": 23, "model_call": 306, "tokens": 35252017, "cost_weighted": 1258950.3, "wall_clock_seconds": 12695.6, "raw": 0.173074, "normalized": 0.173074, "effective": 0.173074, "running_max": 0.173074, "scored": true, "final": false}, {"index": 24, "model_call": 312, "tokens": 36323121, "cost_weighted": 1297759.6, "wall_clock_seconds": 14418.1, "raw": null, "normalized": null, "effective": null, "running_max": null, "scored": false, "final": false}, {"index": 25, "model_call": 324, "tokens": 38804493, "cost_weighted": 1387830.5, "wall_clock_seconds": 15586.6, "raw": 0.177837, "normalized": 0.177837, "effective": 0.177837, "running_max": 0.177837, "scored": true, "final": false}, {"index": 26, "model_call": 347, "tokens": 40916660, "cost_weighted": 1543417.7, "wall_clock_seconds": 16708.2, "raw": 0.177826, "normalized": 0.177826, "effective": 0.177826, "running_max": 0.177837, "scored": true, "final": false}, {"index": 27, "model_call": 349, "tokens": 41093261, "cost_weighted": 1547585.6, "wall_clock_seconds": 19660.7, "raw": 0.188233, "normalized": 0.188233, "effective": 0.188233, "running_max": 0.188233, "scored": true, "final": false}, {"index": 28, "model_call": 377, "tokens": 44694881, "cost_weighted": 1681034.1, "wall_clock_seconds": 20681.2, "raw": 0.189833, "normalized": 0.189833, "effective": 0.189833, "running_max": 0.189833, "scored": true, "final": false}, {"index": 29, "model_call": 387, "tokens": 46439718, "cost_weighted": 1777711.8, "wall_clock_seconds": 24555.3, "raw": 0.18714, "normalized": 0.18714, "effective": 0.18714, "running_max": 0.189833, "scored": true, "final": false}, {"index": 30, "model_call": 394, "tokens": 47807191, "cost_weighted": 1814495.6, "wall_clock_seconds": 26123.4, "raw": 0.18978, "normalized": 0.18978, "effective": 0.18978, "running_max": 0.189833, "scored": true, "final": false}, {"index": 31, "model_call": 398, "tokens": 48643739, "cost_weighted": 1841308.7, "wall_clock_seconds": 26887.7, "raw": 0.19275, "normalized": 0.19275, "effective": 0.19275, "running_max": 0.19275, "scored": true, "final": false}, {"index": 32, "model_call": 404, "tokens": 49974844, "cost_weighted": 1882932.0, "wall_clock_seconds": 27845.9, "raw": 0.192936, "normalized": 0.192936, "effective": 0.192936, "running_max": 0.192936, "scored": true, "final": false}, {"index": 33, "model_call": 407, "tokens": 50687292, "cost_weighted": 1910286.8, "wall_clock_seconds": 29917.4, "raw": 0.194293, "normalized": 0.194293, "effective": 0.194293, "running_max": 0.194293, "scored": true, "final": false} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:ship_tune", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.201797, "area_term": 0.180707, "final_best": 0.28616, "best_raw": 0.28616, "best_effective": 0.28616, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9899251, "usage": {"input": 363, "output": 120657, "cache_read": 9269276, "cache_write": 508955}, "cost_usd": 21.6677, "cost_source": "billed", "wall_clock_seconds": 13090.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2006655, "wall_clock_seconds": 100.2, "raw": 0.226345, "normalized": 0.226345, "discount": 1.0, "effective": 0.226345, "running_max": 0.226345, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.64}, {"index": 2, "tokens": 2659574, "wall_clock_seconds": 157.6, "raw": 0.22789, "normalized": 0.22789, "discount": 1.0, "effective": 0.22789, "running_max": 0.22789, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 156.91}, {"index": 3, "tokens": 5060535, "wall_clock_seconds": 497.2, "raw": 0.228851, "normalized": 0.228851, "discount": 1.0, "effective": 0.228851, "running_max": 0.228851, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 496.43}, {"index": 4, "tokens": 5629984, "wall_clock_seconds": 130.7, "raw": 0.229419, "normalized": 0.229419, "discount": 1.0, "effective": 0.229419, "running_max": 0.229419, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 130.1}, {"index": 5, "tokens": 6786801, "wall_clock_seconds": 196.2, "raw": 0.228876, "normalized": 0.228876, "discount": 1.0, "effective": 0.228876, "running_max": 0.229419, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 195.29}, {"index": 6, "tokens": 8579547, "wall_clock_seconds": 238.6, "raw": 0.228768, "normalized": 0.228768, "discount": 1.0, "effective": 0.228768, "running_max": 0.229419, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 237.65}, {"index": 7, "tokens": 8900537, "wall_clock_seconds": 125.4, "raw": 0.26249, "normalized": 0.26249, "discount": 1.0, "effective": 0.26249, "running_max": 0.26249, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 124.5}, {"index": 8, "tokens": 9224383, "wall_clock_seconds": 124.2, "raw": 0.273783, "normalized": 0.273783, "discount": 1.0, "effective": 0.273783, "running_max": 0.273783, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 123.54}, {"index": 9, "tokens": 9554380, "wall_clock_seconds": 126.7, "raw": 0.278735, "normalized": 0.278735, "discount": 1.0, "effective": 0.278735, "running_max": 0.278735, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 125.88}, {"index": 10, "tokens": 9899251, "wall_clock_seconds": 131.9, "raw": 0.28616, "normalized": 0.28616, "discount": 1.0, "effective": 0.28616, "running_max": 0.28616, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 131.21} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:splash_generalize", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.31875, "area_term": 0.298993, "final_best": 0.397778, "best_raw": 0.816, "best_effective": 0.397778, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10515461, "usage": {"input": 396, "output": 124819, "cache_read": 10125419, "cache_write": 264827}, "cost_usd": 19.6807, "cost_source": "billed", "wall_clock_seconds": 4567.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.5, "before_submission": 7}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 114395, "wall_clock_seconds": 2.2, "raw": 0.107, "normalized": 0.007778, "discount": 1.0, "effective": 0.007778, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.61}, {"index": 2, "tokens": 189842, "wall_clock_seconds": 1.0, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.43}, {"index": 3, "tokens": 403033, "wall_clock_seconds": 1.5, "raw": 0.082, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.99}, {"index": 4, "tokens": 650196, "wall_clock_seconds": 23.5, "raw": 0.203, "normalized": 0.114444, "discount": 1.0, "effective": 0.114444, "running_max": 0.114444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.87}, {"index": 5, "tokens": 746757, "wall_clock_seconds": 23.8, "raw": 0.21, "normalized": 0.122222, "discount": 1.0, "effective": 0.122222, "running_max": 0.122222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 23.28}, {"index": 6, "tokens": 1265738, "wall_clock_seconds": 29.3, "raw": 0.296, "normalized": 0.217778, "discount": 1.0, "effective": 0.217778, "running_max": 0.217778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.73}, {"index": 7, "tokens": 2815949, "wall_clock_seconds": 39.9, "raw": 0.752, "normalized": 0.724444, "discount": 0.5, "effective": 0.362222, "running_max": 0.362222, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 39.32}, {"index": 8, "tokens": 4376385, "wall_clock_seconds": 45.7, "raw": 0.796, "normalized": 0.773333, "discount": 0.5, "effective": 0.386667, "running_max": 0.386667, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 45.09}, {"index": 9, "tokens": 6693128, "wall_clock_seconds": 70.5, "raw": 0.802, "normalized": 0.78, "discount": 0.5, "effective": 0.39, "running_max": 0.39, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 69.98}, {"index": 10, "tokens": 10515461, "wall_clock_seconds": 76.6, "raw": 0.816, "normalized": 0.795556, "discount": 0.5, "effective": 0.397778, "running_max": 0.397778, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 76.1} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:tod_pipeline--nohints", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.292614, "area_term": 0.268708, "final_best": 0.388237, "best_raw": 0.388237, "best_effective": 0.388237, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12968811, "usage": {"input": 381, "output": 196935, "cache_read": 12194663, "cache_write": 576832}, "cost_usd": 29.2556, "cost_source": "billed", "wall_clock_seconds": 12418.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 597944, "wall_clock_seconds": 21.4, "raw": 0.079233, "normalized": 0.079233, "discount": 1.0, "effective": 0.079233, "running_max": 0.079233, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.2}, {"index": 2, "tokens": 1096925, "wall_clock_seconds": 18.0, "raw": 0.195741, "normalized": 0.195741, "discount": 1.0, "effective": 0.195741, "running_max": 0.195741, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 17.7}, {"index": 3, "tokens": 1882526, "wall_clock_seconds": 24.9, "raw": 0.219477, "normalized": 0.219477, "discount": 1.0, "effective": 0.219477, "running_max": 0.219477, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 24.57}, {"index": 4, "tokens": 2362596, "wall_clock_seconds": 44.3, "raw": 0.197976, "normalized": 0.197976, "discount": 1.0, "effective": 0.197976, "running_max": 0.219477, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.01}, {"index": 5, "tokens": 2704054, "wall_clock_seconds": 55.7, "raw": 0.197989, "normalized": 0.197989, "discount": 1.0, "effective": 0.197989, "running_max": 0.219477, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 55.49}, {"index": 6, "tokens": 4165005, "wall_clock_seconds": 135.8, "raw": 0.196425, "normalized": 0.196425, "discount": 1.0, "effective": 0.196425, "running_max": 0.219477, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 135.59}, {"index": 7, "tokens": 4895111, "wall_clock_seconds": 147.6, "raw": 0.180884, "normalized": 0.180884, "discount": 1.0, "effective": 0.180884, "running_max": 0.219477, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 147.35}, {"index": 8, "tokens": 5359596, "wall_clock_seconds": 242.2, "raw": 0.37356, "normalized": 0.37356, "discount": 1.0, "effective": 0.37356, "running_max": 0.37356, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 242.01}, {"index": 9, "tokens": 6366745, "wall_clock_seconds": 248.4, "raw": 0.373474, "normalized": 0.373474, "discount": 1.0, "effective": 0.373474, "running_max": 0.37356, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 248.12}, {"index": 10, "tokens": 12968811, "wall_clock_seconds": 465.5, "raw": 0.388237, "normalized": 0.388237, "discount": 1.0, "effective": 0.388237, "running_max": 0.388237, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 465.23} ]}, {"sample_key": "gYumm372EGDVGpswFdzf3u:weirdml_bonanza", "eval_id": "gYumm372EGDVGpswFdzf3u", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-15", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "55286da", "source_log_sha256": "a53edb344f26dc1af28debaaee6a7c6c8f9621507d8a818aa8075fc8b1228488", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.745089, "area_term": 0.707065, "final_best": 0.897186, "best_raw": 0.897186, "best_effective": 0.897186, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 16201944, "usage": {"input": 468, "output": 222981, "cache_read": 15376020, "cache_write": 602475}, "cost_usd": 34.0607, "cost_source": "billed", "wall_clock_seconds": 8320.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1027960, "wall_clock_seconds": 121.0, "raw": 0.558623, "normalized": 0.558623, "discount": 1.0, "effective": 0.558623, "running_max": 0.558623, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.0}, {"index": 2, "tokens": 1165952, "wall_clock_seconds": 83.2, "raw": 0.736841, "normalized": 0.736841, "discount": 1.0, "effective": 0.736841, "running_max": 0.736841, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 82.27}, {"index": 3, "tokens": 1692894, "wall_clock_seconds": 95.8, "raw": 0.785653, "normalized": 0.785653, "discount": 1.0, "effective": 0.785653, "running_max": 0.785653, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 95.05}, {"index": 4, "tokens": 3465873, "wall_clock_seconds": 111.4, "raw": 0.833835, "normalized": 0.833835, "discount": 1.0, "effective": 0.833835, "running_max": 0.833835, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 110.83}, {"index": 5, "tokens": 4633261, "wall_clock_seconds": 121.2, "raw": 0.793418, "normalized": 0.793418, "discount": 1.0, "effective": 0.793418, "running_max": 0.833835, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.0}, {"index": 6, "tokens": 5059175, "wall_clock_seconds": 106.9, "raw": 0.866506, "normalized": 0.866506, "discount": 1.0, "effective": 0.866506, "running_max": 0.866506, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.27}, {"index": 7, "tokens": 6532238, "wall_clock_seconds": 107.6, "raw": 0.880935, "normalized": 0.880935, "discount": 1.0, "effective": 0.880935, "running_max": 0.880935, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.98}, {"index": 8, "tokens": 7676974, "wall_clock_seconds": 114.4, "raw": 0.880829, "normalized": 0.880829, "discount": 1.0, "effective": 0.880829, "running_max": 0.880935, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 113.82}, {"index": 9, "tokens": 8512673, "wall_clock_seconds": 114.1, "raw": 0.8792, "normalized": 0.8792, "discount": 1.0, "effective": 0.8792, "running_max": 0.880935, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 113.43}, {"index": 10, "tokens": 16201944, "wall_clock_seconds": 107.5, "raw": 0.897186, "normalized": 0.897186, "discount": 1.0, "effective": 0.897186, "running_max": 0.897186, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 106.78} ]}, {"sample_key": "4gYLxdTMQtySqrWE44jVSG:ship_detect", "eval_id": "4gYLxdTMQtySqrWE44jVSG", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "6331f2d", "source_log_sha256": "251ec162f4acedce517d74eb9e4e0acf60b2812d5d03c3dc28b12b9a4e8c1300", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.027943, "area_term": 0.020029, "final_best": 0.0596, "best_raw": 0.0596, "best_effective": 0.0596, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 2, "max_submissions": null, "tokens_total": 12547313, "usage": {"input": 170028, "output": 56564, "cache_read": 12320721, "cache_write": 12124731}, "cost_usd": 1.7693, "cost_source": "list_price", "wall_clock_seconds": 1692.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 72, "tokens": 4013472, "cost_weighted": 164486.1, "wall_clock_seconds": 690.0, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.0, "scored": true, "final": false}, {"index": 2, "model_call": 134, "tokens": 11215472, "cost_weighted": 425500.7, "wall_clock_seconds": 1539.5, "raw": 0.0596, "normalized": 0.0596, "effective": 0.0596, "running_max": 0.0596, "scored": true, "final": false} ]}, {"sample_key": "5iZxVE3CJQ8ofh6YDMoqGg:scan_stitch--nohints", "eval_id": "5iZxVE3CJQ8ofh6YDMoqGg", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "caefdb0", "source_log_sha256": "ad30c7fe7dc2c57bee726b42aebb9da9b7ac9a4ab475faf10bc50be6c79c22d2", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.000194, "area_term": 0.000103, "final_best": 0.000558, "best_raw": 0.100446, "best_effective": 0.000558, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 22131247, "usage": {"input": 684, "output": 276415, "cache_read": 21028274, "cache_write": 825874}, "cost_usd": 45.1793, "cost_source": "billed", "wall_clock_seconds": 9532.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1265637, "wall_clock_seconds": 0.3, "raw": 0.095175, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3394625, "wall_clock_seconds": 0.3, "raw": 0.085364, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4966828, "wall_clock_seconds": 0.3, "raw": 0.078054, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 9065920, "wall_clock_seconds": 0.3, "raw": 0.090169, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 14274105, "wall_clock_seconds": 0.3, "raw": 0.099735, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 17862452, "wall_clock_seconds": 0.3, "raw": 0.085364, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 20204707, "wall_clock_seconds": 0.3, "raw": 0.085364, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 20798625, "wall_clock_seconds": 0.3, "raw": 0.100098, "normalized": 0.000123, "discount": 1.0, "effective": 0.000123, "running_max": 0.000123, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 21486134, "wall_clock_seconds": 0.3, "raw": 0.100446, "normalized": 0.000558, "discount": 1.0, "effective": 0.000558, "running_max": 0.000558, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 22131247, "wall_clock_seconds": 0.2, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000558, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "5iZxVE3CJQ8ofh6YDMoqGg:ship_detect", "eval_id": "5iZxVE3CJQ8ofh6YDMoqGg", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "caefdb0", "source_log_sha256": "ad30c7fe7dc2c57bee726b42aebb9da9b7ac9a4ab475faf10bc50be6c79c22d2", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.152585, "area_term": 0.12826, "final_best": 0.249885, "best_raw": 0.249885, "best_effective": 0.249885, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 29, "max_submissions": null, "tokens_total": 58477041, "usage": {"input": 1476, "output": 270339, "cache_read": 55744278, "cache_write": 2460948}, "cost_usd": 100.0378, "cost_source": "billed", "wall_clock_seconds": 25462.6, "stop_reason": "cost_limit", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 18, "tokens": 392241, "cost_weighted": 21535.8, "wall_clock_seconds": 423.4, "raw": 0.058259, "normalized": 0.058259, "effective": 0.058259, "running_max": 0.058259, "scored": true, "final": false}, {"index": 2, "model_call": 63, "tokens": 3538978, "cost_weighted": 125297.5, "wall_clock_seconds": 1365.9, "raw": 0.088314, "normalized": 0.088314, "effective": 0.088314, "running_max": 0.088314, "scored": true, "final": false}, {"index": 3, "model_call": 104, "tokens": 9043486, "cost_weighted": 272058.4, "wall_clock_seconds": 2658.5, "raw": 0.153315, "normalized": 0.153315, "effective": 0.153315, "running_max": 0.153315, "scored": true, "final": false}, {"index": 4, "model_call": 122, "tokens": 12069947, "cost_weighted": 388209.5, "wall_clock_seconds": 3750.8, "raw": 0.162824, "normalized": 0.162824, "effective": 0.162824, "running_max": 0.162824, "scored": true, "final": false}, {"index": 5, "model_call": 127, "tokens": 12864077, "cost_weighted": 413551.9, "wall_clock_seconds": 4081.6, "raw": 0.179801, "normalized": 0.179801, "effective": 0.179801, "running_max": 0.179801, "scored": true, "final": false}, {"index": 6, "model_call": 128, "tokens": 13027243, "cost_weighted": 417168.0, "wall_clock_seconds": 7400.7, "raw": 0.170786, "normalized": 0.170786, "effective": 0.170786, "running_max": 0.179801, "scored": true, "final": false}, {"index": 7, "model_call": 130, "tokens": 13355685, "cost_weighted": 462547.5, "wall_clock_seconds": 7458.9, "raw": 0.196733, "normalized": 0.196733, "effective": 0.196733, "running_max": 0.196733, "scored": true, "final": false}, {"index": 8, "model_call": 142, "tokens": 15506858, "cost_weighted": 531645.3, "wall_clock_seconds": 8123.1, "raw": 0.212916, "normalized": 0.212916, "effective": 0.212916, "running_max": 0.212916, "scored": true, "final": false}, {"index": 9, "model_call": 235, "tokens": 23436161, "cost_weighted": 804397.0, "wall_clock_seconds": 9707.8, "raw": 0.220595, "normalized": 0.220595, "effective": 0.220595, "running_max": 0.220595, "scored": true, "final": false}, {"index": 10, "model_call": 262, "tokens": 28035438, "cost_weighted": 915676.6, "wall_clock_seconds": 10340.4, "raw": 0.218994, "normalized": 0.218994, "effective": 0.218994, "running_max": 0.220595, "scored": true, "final": false}, {"index": 11, "model_call": 266, "tokens": 28782752, "cost_weighted": 932885.7, "wall_clock_seconds": 12858.5, "raw": 0.23776, "normalized": 0.23776, "effective": 0.23776, "running_max": 0.23776, "scored": true, "final": false}, {"index": 12, "model_call": 292, "tokens": 33617783, "cost_weighted": 1179969.8, "wall_clock_seconds": 13781.0, "raw": 0.238159, "normalized": 0.238159, "effective": 0.238159, "running_max": 0.238159, "scored": true, "final": false}, {"index": 13, "model_call": 298, "tokens": 33765972, "cost_weighted": 1190279.2, "wall_clock_seconds": 13830.1, "raw": 0.242094, "normalized": 0.242094, "effective": 0.242094, "running_max": 0.242094, "scored": true, "final": false}, {"index": 14, "model_call": 327, "tokens": 35393033, "cost_weighted": 1240118.2, "wall_clock_seconds": 14194.2, "raw": 0.24335, "normalized": 0.24335, "effective": 0.24335, "running_max": 0.24335, "scored": true, "final": false}, {"index": 15, "model_call": 371, "tokens": 40088743, "cost_weighted": 1360156.1, "wall_clock_seconds": 14849.2, "raw": 0.243828, "normalized": 0.243828, "effective": 0.243828, "running_max": 0.243828, "scored": true, "final": false}, {"index": 16, "model_call": 385, "tokens": 42255523, "cost_weighted": 1415941.7, "wall_clock_seconds": 15146.4, "raw": 0.249554, "normalized": 0.249554, "effective": 0.249554, "running_max": 0.249554, "scored": true, "final": false}, {"index": 17, "model_call": 403, "tokens": 45356156, "cost_weighted": 1490141.3, "wall_clock_seconds": 15540.6, "raw": 0.244145, "normalized": 0.244145, "effective": 0.244145, "running_max": 0.249554, "scored": true, "final": false}, {"index": 18, "model_call": 405, "tokens": 45731500, "cost_weighted": 1501355.9, "wall_clock_seconds": 15827.0, "raw": 0.242939, "normalized": 0.242939, "effective": 0.242939, "running_max": 0.249554, "scored": true, "final": false}, {"index": 19, "model_call": 407, "tokens": 46108875, "cost_weighted": 1509683.3, "wall_clock_seconds": 18279.2, "raw": 0.249066, "normalized": 0.249066, "effective": 0.249066, "running_max": 0.249554, "scored": true, "final": false}, {"index": 20, "model_call": 413, "tokens": 47264455, "cost_weighted": 1580815.5, "wall_clock_seconds": 18449.6, "raw": 0.227471, "normalized": 0.227471, "effective": 0.227471, "running_max": 0.249554, "scored": true, "final": false}, {"index": 21, "model_call": 414, "tokens": 47460892, "cost_weighted": 1585332.8, "wall_clock_seconds": 18808.9, "raw": 0.24938, "normalized": 0.24938, "effective": 0.24938, "running_max": 0.249554, "scored": true, "final": false}, {"index": 22, "model_call": 421, "tokens": 48871546, "cost_weighted": 1620291.1, "wall_clock_seconds": 19627.2, "raw": 0.249885, "normalized": 0.249885, "effective": 0.249885, "running_max": 0.249885, "scored": true, "final": false}, {"index": 23, "model_call": 443, "tokens": 53689695, "cost_weighted": 1736118.4, "wall_clock_seconds": 20248.0, "raw": 0.228506, "normalized": 0.228506, "effective": 0.228506, "running_max": 0.249885, "scored": true, "final": false}, {"index": 24, "model_call": 444, "tokens": 53924027, "cost_weighted": 1741367.0, "wall_clock_seconds": 21206.2, "raw": 0.228037, "normalized": 0.228037, "effective": 0.228037, "running_max": 0.249885, "scored": true, "final": false}, {"index": 25, "model_call": 448, "tokens": 54878437, "cost_weighted": 1766774.5, "wall_clock_seconds": 23733.9, "raw": 0.229794, "normalized": 0.229794, "effective": 0.229794, "running_max": 0.249885, "scored": true, "final": false}, {"index": 26, "model_call": 480, "tokens": 57374765, "cost_weighted": 1963239.7, "wall_clock_seconds": 24450.4, "raw": 0.239771, "normalized": 0.239771, "effective": 0.239771, "running_max": 0.249885, "scored": true, "final": false}, {"index": 27, "model_call": 484, "tokens": 57683709, "cost_weighted": 1975136.2, "wall_clock_seconds": 24582.5, "raw": 0.240415, "normalized": 0.240415, "effective": 0.240415, "running_max": 0.249885, "scored": true, "final": false}, {"index": 28, "model_call": 490, "tokens": 58200883, "cost_weighted": 1991479.7, "wall_clock_seconds": 25331.8, "raw": 0.244108, "normalized": 0.244108, "effective": 0.244108, "running_max": 0.249885, "scored": true, "final": false}, {"index": 29, "model_call": 492, "tokens": 58383829, "cost_weighted": 1998320.1, "wall_clock_seconds": 25426.0, "raw": 0.244119, "normalized": 0.244119, "effective": 0.244119, "running_max": 0.249885, "scored": true, "final": false} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:mystery_box", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.009366, "area_term": 0.007596, "final_best": 0.016443, "best_raw": 0.096565, "best_effective": 0.016443, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 18047561, "usage": {"input": 582, "output": 156130, "cache_read": 17119816, "cache_write": 771033}, "cost_usd": 6.914, "cost_source": "billed", "wall_clock_seconds": 5395.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5636425, "wall_clock_seconds": 3.8, "raw": 0.092142, "normalized": 0.06063, "discount": 0.25, "effective": 0.015158, "running_max": 0.015158, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6273322, "wall_clock_seconds": 3.5, "raw": 0.092132, "normalized": 0.060619, "discount": 0.25, "effective": 0.015155, "running_max": 0.015158, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8809523, "wall_clock_seconds": 3.6, "raw": 0.093719, "normalized": 0.062464, "discount": 0.25, "effective": 0.015616, "running_max": 0.015616, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 10890123, "wall_clock_seconds": 3.4, "raw": 0.095288, "normalized": 0.064288, "discount": 0.25, "effective": 0.016072, "running_max": 0.016072, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 12334683, "wall_clock_seconds": 3.5, "raw": 0.082641, "normalized": 0.049582, "discount": 0.25, "effective": 0.012396, "running_max": 0.016072, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 13482291, "wall_clock_seconds": 3.8, "raw": 0.095315, "normalized": 0.06432, "discount": 0.25, "effective": 0.01608, "running_max": 0.01608, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 16355610, "wall_clock_seconds": 3.2, "raw": 0.071977, "normalized": 0.037182, "discount": 0.25, "effective": 0.009296, "running_max": 0.01608, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 16851657, "wall_clock_seconds": 3.4, "raw": 0.096565, "normalized": 0.065774, "discount": 0.25, "effective": 0.016443, "running_max": 0.016443, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 17701063, "wall_clock_seconds": 3.5, "raw": 0.09625, "normalized": 0.065407, "discount": 0.25, "effective": 0.016352, "running_max": 0.016443, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 18047561, "wall_clock_seconds": 3.6, "raw": 0.096146, "normalized": 0.065286, "discount": 0.25, "effective": 0.016321, "running_max": 0.016443, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:mystery_box--nohints", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.030405, "area_term": 0.028769, "final_best": 0.03695, "best_raw": 0.071777, "best_effective": 0.03695, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5293916, "usage": {"input": 207, "output": 59173, "cache_read": 5087846, "cache_write": 146690}, "cost_usd": 1.9764, "cost_source": "billed", "wall_clock_seconds": 3372.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 88908, "wall_clock_seconds": 2.6, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 326638, "wall_clock_seconds": 5.0, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 406278, "wall_clock_seconds": 3.0, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 700976, "wall_clock_seconds": 2.7, "raw": 0.044445, "normalized": 0.005169, "discount": 1.0, "effective": 0.005169, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 814518, "wall_clock_seconds": 3.9, "raw": 0.051104, "normalized": 0.012912, "discount": 1.0, "effective": 0.012912, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 993393, "wall_clock_seconds": 4.0, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1391138, "wall_clock_seconds": 1.6, "raw": 0.039647, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2195909, "wall_clock_seconds": 5.3, "raw": 0.05856, "normalized": 0.021582, "discount": 1.0, "effective": 0.021582, "running_max": 0.021582, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 3328482, "wall_clock_seconds": 2.7, "raw": 0.050258, "normalized": 0.011928, "discount": 1.0, "effective": 0.011928, "running_max": 0.021582, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5293916, "wall_clock_seconds": 2.8, "raw": 0.071777, "normalized": 0.03695, "discount": 1.0, "effective": 0.03695, "running_max": 0.03695, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:night_school", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.182537, "area_term": 0.157274, "final_best": 0.283589, "best_raw": 0.679781, "best_effective": 0.283589, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 12611864, "usage": {"input": 576, "output": 64569, "cache_read": 12180518, "cache_write": 366201}, "cost_usd": 3.9984, "cost_source": "billed", "wall_clock_seconds": 6876.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}, {"id": 1, "multiplier": 0.85, "before_submission": 2}, {"id": 3, "multiplier": 0.7, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 259800, "wall_clock_seconds": 3.3, "raw": 0.197, "normalized": 0.055294, "discount": 0.65, "effective": 0.035941, "running_max": 0.035941, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4256284, "wall_clock_seconds": 12.7, "raw": 0.246, "normalized": 0.112941, "discount": 0.65, "effective": 0.073412, "running_max": 0.073412, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4718299, "wall_clock_seconds": 15.0, "raw": 0.391563, "normalized": 0.284191, "discount": 0.455, "effective": 0.129307, "running_max": 0.129307, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 5075567, "wall_clock_seconds": 14.1, "raw": 0.569406, "normalized": 0.493419, "discount": 0.455, "effective": 0.224506, "running_max": 0.224506, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 5437263, "wall_clock_seconds": 14.2, "raw": 0.593656, "normalized": 0.521949, "discount": 0.455, "effective": 0.237487, "running_max": 0.237487, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 6563827, "wall_clock_seconds": 14.5, "raw": 0.634437, "normalized": 0.569926, "discount": 0.455, "effective": 0.259317, "running_max": 0.259317, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 7339863, "wall_clock_seconds": 15.8, "raw": 0.669531, "normalized": 0.611213, "discount": 0.455, "effective": 0.278102, "running_max": 0.278102, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 8129563, "wall_clock_seconds": 17.6, "raw": 0.66625, "normalized": 0.607353, "discount": 0.455, "effective": 0.276346, "running_max": 0.278102, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8805360, "wall_clock_seconds": 13.7, "raw": 0.667969, "normalized": 0.609375, "discount": 0.455, "effective": 0.277266, "running_max": 0.278102, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 12611864, "wall_clock_seconds": 13.1, "raw": 0.679781, "normalized": 0.623272, "discount": 0.455, "effective": 0.283589, "running_max": 0.283589, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:reaction_rates", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.071776, "area_term": 0.042462, "final_best": 0.189028, "best_raw": 0.756415, "best_effective": 0.189028, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 19686844, "usage": {"input": 711, "output": 142979, "cache_read": 18792473, "cache_write": 750681}, "cost_usd": 7.0664, "cost_source": "billed", "wall_clock_seconds": 8789.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.8, "before_submission": 1}, {"id": 1, "multiplier": 0.7, "before_submission": 2}, {"id": 3, "multiplier": 0.7, "before_submission": 2}, {"id": 5, "multiplier": 0.85, "before_submission": 2}, {"id": 2, "multiplier": 0.75, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 469260, "wall_clock_seconds": 22.8, "raw": 0.0, "normalized": 0.0, "discount": 0.8, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 22.25}, {"index": 2, "tokens": 2355784, "wall_clock_seconds": 170.6, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 169.95}, {"index": 3, "tokens": 4677687, "wall_clock_seconds": 171.8, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 171.22}, {"index": 4, "tokens": 5696697, "wall_clock_seconds": 126.0, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 125.41}, {"index": 5, "tokens": 7193607, "wall_clock_seconds": 127.1, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 126.53}, {"index": 6, "tokens": 10143992, "wall_clock_seconds": 68.1, "raw": 0.068363, "normalized": 0.068363, "discount": 0.2499, "effective": 0.017084, "running_max": 0.017084, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 67.14}, {"index": 7, "tokens": 14035468, "wall_clock_seconds": 175.1, "raw": 0.073847, "normalized": 0.073847, "discount": 0.2499, "effective": 0.018454, "running_max": 0.018454, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 174.49}, {"index": 8, "tokens": 15007858, "wall_clock_seconds": 175.2, "raw": 0.182496, "normalized": 0.182496, "discount": 0.2499, "effective": 0.045606, "running_max": 0.045606, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 174.66}, {"index": 9, "tokens": 17216401, "wall_clock_seconds": 202.9, "raw": 0.188441, "normalized": 0.188441, "discount": 0.2499, "effective": 0.047092, "running_max": 0.047092, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 202.29}, {"index": 10, "tokens": 19686844, "wall_clock_seconds": 223.8, "raw": 0.756415, "normalized": 0.756415, "discount": 0.2499, "effective": 0.189028, "running_max": 0.189028, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 223.21} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:reaction_rates--nohints", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.186598, "area_term": 0.167656, "final_best": 0.262367, "best_raw": 0.262367, "best_effective": 0.262367, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10737103, "usage": {"input": 210, "output": 86837, "cache_read": 10412588, "cache_write": 237468}, "cost_usd": 3.545, "cost_source": "billed", "wall_clock_seconds": 3371.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 541454, "wall_clock_seconds": 24.5, "raw": 0.030147, "normalized": 0.030147, "discount": 1.0, "effective": 0.030147, "running_max": 0.030147, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 24.05}, {"index": 2, "tokens": 1839761, "wall_clock_seconds": 29.4, "raw": 0.165253, "normalized": 0.165253, "discount": 1.0, "effective": 0.165253, "running_max": 0.165253, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 28.91}, {"index": 3, "tokens": 5160188, "wall_clock_seconds": 28.1, "raw": 0.195644, "normalized": 0.195644, "discount": 1.0, "effective": 0.195644, "running_max": 0.195644, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.75}, {"index": 4, "tokens": 5555517, "wall_clock_seconds": 27.6, "raw": 0.130908, "normalized": 0.130908, "discount": 1.0, "effective": 0.130908, "running_max": 0.195644, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.27}, {"index": 5, "tokens": 6809830, "wall_clock_seconds": 28.0, "raw": 0.198526, "normalized": 0.198526, "discount": 1.0, "effective": 0.198526, "running_max": 0.198526, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.55}, {"index": 6, "tokens": 8157023, "wall_clock_seconds": 27.6, "raw": 0.25575, "normalized": 0.25575, "discount": 1.0, "effective": 0.25575, "running_max": 0.25575, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.31}, {"index": 7, "tokens": 8620646, "wall_clock_seconds": 27.3, "raw": 0.255822, "normalized": 0.255822, "discount": 1.0, "effective": 0.255822, "running_max": 0.255822, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.78}, {"index": 8, "tokens": 9086215, "wall_clock_seconds": 26.4, "raw": 0.201265, "normalized": 0.201265, "discount": 1.0, "effective": 0.201265, "running_max": 0.255822, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.93}, {"index": 9, "tokens": 9554631, "wall_clock_seconds": 26.4, "raw": 0.262367, "normalized": 0.262367, "discount": 1.0, "effective": 0.262367, "running_max": 0.262367, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.99}, {"index": 10, "tokens": 10737103, "wall_clock_seconds": 27.4, "raw": 0.261949, "normalized": 0.261949, "discount": 1.0, "effective": 0.261949, "running_max": 0.262367, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 27.07} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:scan_stitch", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.169862, "area_term": 0.1493, "final_best": 0.252109, "best_raw": 0.676248, "best_effective": 0.252109, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11679496, "usage": {"input": 492, "output": 75264, "cache_read": 11280411, "cache_write": 323329}, "cost_usd": 3.818, "cost_source": "billed", "wall_clock_seconds": 4605.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 1}, {"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2453897, "wall_clock_seconds": 0.4, "raw": 0.535927, "normalized": 0.544909, "discount": 0.35, "effective": 0.190718, "running_max": 0.190718, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4148556, "wall_clock_seconds": 0.3, "raw": 0.555439, "normalized": 0.569299, "discount": 0.35, "effective": 0.199254, "running_max": 0.199254, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 5861646, "wall_clock_seconds": 0.3, "raw": 0.56287, "normalized": 0.578587, "discount": 0.35, "effective": 0.202505, "running_max": 0.202505, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 6187706, "wall_clock_seconds": 0.3, "raw": 0.56287, "normalized": 0.578587, "discount": 0.35, "effective": 0.202505, "running_max": 0.202505, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 7191319, "wall_clock_seconds": 0.3, "raw": 0.56393, "normalized": 0.579913, "discount": 0.35, "effective": 0.202969, "running_max": 0.202969, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8252806, "wall_clock_seconds": 0.4, "raw": 0.573993, "normalized": 0.592491, "discount": 0.35, "effective": 0.207372, "running_max": 0.207372, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 9169368, "wall_clock_seconds": 0.3, "raw": 0.672064, "normalized": 0.71508, "discount": 0.35, "effective": 0.250278, "running_max": 0.250278, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 9355939, "wall_clock_seconds": 0.3, "raw": 0.672744, "normalized": 0.715929, "discount": 0.35, "effective": 0.250575, "running_max": 0.250575, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 10883923, "wall_clock_seconds": 0.4, "raw": 0.676248, "normalized": 0.720311, "discount": 0.35, "effective": 0.252109, "running_max": 0.252109, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 11679496, "wall_clock_seconds": 0.3, "raw": 0.67564, "normalized": 0.71955, "discount": 0.35, "effective": 0.251843, "running_max": 0.252109, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:scan_stitch--nohints", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.021689, "area_term": 0.020301, "final_best": 0.02724, "best_raw": 0.121792, "best_effective": 0.02724, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8097796, "usage": {"input": 240, "output": 67066, "cache_read": 7831089, "cache_write": 199401}, "cost_usd": 2.7359, "cost_source": "billed", "wall_clock_seconds": 2080.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 325583, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 692841, "wall_clock_seconds": 0.3, "raw": 0.103576, "normalized": 0.004471, "discount": 1.0, "effective": 0.004471, "running_max": 0.004471, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 873247, "wall_clock_seconds": 0.3, "raw": 0.107081, "normalized": 0.008851, "discount": 1.0, "effective": 0.008851, "running_max": 0.008851, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 998648, "wall_clock_seconds": 0.3, "raw": 0.091783, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.008851, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1361369, "wall_clock_seconds": 0.3, "raw": 0.100066, "normalized": 8.3e-05, "discount": 1.0, "effective": 8.3e-05, "running_max": 0.008851, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1607842, "wall_clock_seconds": 0.3, "raw": 0.106339, "normalized": 0.007924, "discount": 1.0, "effective": 0.007924, "running_max": 0.008851, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1779737, "wall_clock_seconds": 0.3, "raw": 0.115511, "normalized": 0.019388, "discount": 1.0, "effective": 0.019388, "running_max": 0.019388, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2148253, "wall_clock_seconds": 0.3, "raw": 0.115511, "normalized": 0.019388, "discount": 1.0, "effective": 0.019388, "running_max": 0.019388, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 3241014, "wall_clock_seconds": 0.3, "raw": 0.121792, "normalized": 0.02724, "discount": 1.0, "effective": 0.02724, "running_max": 0.02724, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 8097796, "wall_clock_seconds": 0.3, "raw": 0.104553, "normalized": 0.005692, "discount": 1.0, "effective": 0.005692, "running_max": 0.02724, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:shapes_generalize", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.071369, "area_term": 0.069367, "final_best": 0.079375, "best_raw": 0.327, "best_effective": 0.079375, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5217040, "usage": {"input": 402, "output": 47584, "cache_read": 4806056, "cache_write": 362998}, "cost_usd": 2.3454, "cost_source": "billed", "wall_clock_seconds": 2593.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.5, "before_submission": 5}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 340348, "wall_clock_seconds": 17.1, "raw": 0.224, "normalized": 0.03, "discount": 1.0, "effective": 0.03, "running_max": 0.03, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 16.41}, {"index": 2, "tokens": 696836, "wall_clock_seconds": 26.3, "raw": 0.232, "normalized": 0.04, "discount": 1.0, "effective": 0.04, "running_max": 0.04, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.68}, {"index": 3, "tokens": 1006597, "wall_clock_seconds": 27.0, "raw": 0.038, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.39}, {"index": 4, "tokens": 1164560, "wall_clock_seconds": 27.9, "raw": 0.239, "normalized": 0.04875, "discount": 1.0, "effective": 0.04875, "running_max": 0.04875, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.8}, {"index": 5, "tokens": 1486561, "wall_clock_seconds": 31.4, "raw": 0.311, "normalized": 0.13875, "discount": 0.5, "effective": 0.069375, "running_max": 0.069375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 30.66}, {"index": 6, "tokens": 1816030, "wall_clock_seconds": 36.3, "raw": 0.327, "normalized": 0.15875, "discount": 0.5, "effective": 0.079375, "running_max": 0.079375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 35.42}, {"index": 7, "tokens": 2941293, "wall_clock_seconds": 61.6, "raw": 0.323, "normalized": 0.15375, "discount": 0.5, "effective": 0.076875, "running_max": 0.079375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 56.32}, {"index": 8, "tokens": 3871386, "wall_clock_seconds": 43.8, "raw": 0.311, "normalized": 0.13875, "discount": 0.5, "effective": 0.069375, "running_max": 0.079375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 43.12}, {"index": 9, "tokens": 4412968, "wall_clock_seconds": 48.1, "raw": 0.297, "normalized": 0.12125, "discount": 0.5, "effective": 0.060625, "running_max": 0.079375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 47.34}, {"index": 10, "tokens": 5217040, "wall_clock_seconds": 41.7, "raw": 0.31, "normalized": 0.1375, "discount": 0.5, "effective": 0.06875, "running_max": 0.079375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 40.87} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:shattered_prior", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.146639, "area_term": 0.081262, "final_best": 0.40815, "best_raw": 0.627923, "best_effective": 0.40815, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 45089377, "usage": {"input": 1251, "output": 293158, "cache_read": 43036103, "cache_write": 1758865}, "cost_usd": 15.9385, "cost_source": "billed", "wall_clock_seconds": 13978.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 2}, {"id": 3, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 434815, "wall_clock_seconds": 1.9, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 13968217, "wall_clock_seconds": 1.9, "raw": 0.0, "normalized": 0.0, "discount": 0.65, "effective": 0.0, "running_max": 0.04873, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 24073262, "wall_clock_seconds": 1.7, "raw": 0.182708, "normalized": 0.182708, "discount": 0.65, "effective": 0.11876, "running_max": 0.11876, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 28300966, "wall_clock_seconds": 2.1, "raw": 0.353545, "normalized": 0.353545, "discount": 0.65, "effective": 0.229804, "running_max": 0.229804, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 29981596, "wall_clock_seconds": 1.8, "raw": 0.366033, "normalized": 0.366033, "discount": 0.65, "effective": 0.237922, "running_max": 0.237922, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 32418561, "wall_clock_seconds": 1.6, "raw": 0.366033, "normalized": 0.366033, "discount": 0.65, "effective": 0.237922, "running_max": 0.237922, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 41544658, "wall_clock_seconds": 2.0, "raw": 0.61965, "normalized": 0.61965, "discount": 0.65, "effective": 0.402772, "running_max": 0.402772, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 42814804, "wall_clock_seconds": 2.1, "raw": 0.61393, "normalized": 0.61393, "discount": 0.65, "effective": 0.399055, "running_max": 0.402772, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 43581439, "wall_clock_seconds": 1.8, "raw": 0.627923, "normalized": 0.627923, "discount": 0.65, "effective": 0.40815, "running_max": 0.40815, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 45089377, "wall_clock_seconds": 2.3, "raw": 0.62476, "normalized": 0.62476, "discount": 0.65, "effective": 0.406094, "running_max": 0.40815, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:shattered_prior--nohints", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.054327, "area_term": 0.049632, "final_best": 0.07311, "best_raw": 0.07311, "best_effective": 0.07311, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 45143621, "usage": {"input": 933, "output": 326693, "cache_read": 41934491, "cache_write": 2881504}, "cost_usd": 18.8595, "cost_source": "billed", "wall_clock_seconds": 24832.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 121622, "wall_clock_seconds": 1.9, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5883455, "wall_clock_seconds": 1.8, "raw": 0.046411, "normalized": 0.046411, "discount": 1.0, "effective": 0.046411, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 22374962, "wall_clock_seconds": 1.6, "raw": 0.04841, "normalized": 0.04841, "discount": 1.0, "effective": 0.04841, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 26593782, "wall_clock_seconds": 1.8, "raw": 0.028681, "normalized": 0.028681, "discount": 1.0, "effective": 0.028681, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 39005349, "wall_clock_seconds": 1.9, "raw": 0.037383, "normalized": 0.037383, "discount": 1.0, "effective": 0.037383, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 41862266, "wall_clock_seconds": 1.6, "raw": 0.062489, "normalized": 0.062489, "discount": 1.0, "effective": 0.062489, "running_max": 0.062489, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 42278599, "wall_clock_seconds": 1.5, "raw": 0.069505, "normalized": 0.069505, "discount": 1.0, "effective": 0.069505, "running_max": 0.069505, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 42971364, "wall_clock_seconds": 1.5, "raw": 0.0728, "normalized": 0.0728, "discount": 1.0, "effective": 0.0728, "running_max": 0.0728, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 43639986, "wall_clock_seconds": 1.6, "raw": 0.072745, "normalized": 0.072745, "discount": 1.0, "effective": 0.072745, "running_max": 0.0728, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 45143621, "wall_clock_seconds": 1.6, "raw": 0.07311, "normalized": 0.07311, "discount": 1.0, "effective": 0.07311, "running_max": 0.07311, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:ship_detect", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.032469, "area_term": 0.029953, "final_best": 0.042537, "best_raw": 0.042537, "best_effective": 0.042537, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 86, "max_submissions": null, "tokens_total": 40157244, "usage": {"input": 1620, "output": 478281, "cache_read": 36603274, "cache_write": 3074069}, "cost_usd": 19.7919, "cost_source": "billed", "wall_clock_seconds": 12973.4, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 3, "tokens": 23442, "cost_weighted": 4533.7, "wall_clock_seconds": 126.9, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.0, "scored": true, "final": false}, {"index": 2, "model_call": 13, "tokens": 495256, "cost_weighted": 38072.8, "wall_clock_seconds": 1383.9, "raw": 0.033549, "normalized": 0.033549, "effective": 0.033549, "running_max": 0.033549, "scored": true, "final": false}, {"index": 3, "model_call": 18, "tokens": 874444, "cost_weighted": 56603.4, "wall_clock_seconds": 1545.3, "raw": 0.033234, "normalized": 0.033234, "effective": 0.033234, "running_max": 0.033549, "scored": true, "final": false}, {"index": 4, "model_call": 23, "tokens": 1319698, "cost_weighted": 71902.5, "wall_clock_seconds": 1645.9, "raw": 0.033211, "normalized": 0.033211, "effective": 0.033211, "running_max": 0.033549, "scored": true, "final": false}, {"index": 5, "model_call": 25, "tokens": 1515047, "cost_weighted": 80608.9, "wall_clock_seconds": 1705.5, "raw": 0.033218, "normalized": 0.033218, "effective": 0.033218, "running_max": 0.033549, "scored": true, "final": false}, {"index": 6, "model_call": 27, "tokens": 1721070, "cost_weighted": 88395.3, "wall_clock_seconds": 1750.7, "raw": 0.033213, "normalized": 0.033213, "effective": 0.033213, "running_max": 0.033549, "scored": true, "final": false}, {"index": 7, "model_call": 29, "tokens": 1936898, "cost_weighted": 96480.5, "wall_clock_seconds": 1801.9, "raw": 0.033194, "normalized": 0.033194, "effective": 0.033194, "running_max": 0.033549, "scored": true, "final": false}, {"index": 8, "model_call": 31, "tokens": 2162352, "cost_weighted": 104282.9, "wall_clock_seconds": 1850.8, "raw": 0.033217, "normalized": 0.033217, "effective": 0.033217, "running_max": 0.033549, "scored": true, "final": false}, {"index": 9, "model_call": 35, "tokens": 2646729, "cost_weighted": 124920.4, "wall_clock_seconds": 2036.4, "raw": 0.033209, "normalized": 0.033209, "effective": 0.033209, "running_max": 0.033549, "scored": true, "final": false}, {"index": 10, "model_call": 37, "tokens": 2904130, "cost_weighted": 133559.3, "wall_clock_seconds": 2083.4, "raw": 0.033218, "normalized": 0.033218, "effective": 0.033218, "running_max": 0.033549, "scored": true, "final": false}, {"index": 11, "model_call": 42, "tokens": 3581359, "cost_weighted": 154209.8, "wall_clock_seconds": 2461.8, "raw": 0.033243, "normalized": 0.033243, "effective": 0.033243, "running_max": 0.033549, "scored": true, "final": false}, {"index": 12, "model_call": 45, "tokens": 4010507, "cost_weighted": 166160.4, "wall_clock_seconds": 2534.6, "raw": 0.033247, "normalized": 0.033247, "effective": 0.033247, "running_max": 0.033549, "scored": true, "final": false}, {"index": 13, "model_call": 47, "tokens": 4307378, "cost_weighted": 175370.3, "wall_clock_seconds": 2586.0, "raw": 0.033333, "normalized": 0.033333, "effective": 0.033333, "running_max": 0.033549, "scored": true, "final": false}, {"index": 14, "model_call": 50, "tokens": 4765945, "cost_weighted": 188624.4, "wall_clock_seconds": 2670.0, "raw": 0.033358, "normalized": 0.033358, "effective": 0.033358, "running_max": 0.033549, "scored": true, "final": false}, {"index": 15, "model_call": 52, "tokens": 5083222, "cost_weighted": 198513.6, "wall_clock_seconds": 2719.2, "raw": 0.03336, "normalized": 0.03336, "effective": 0.03336, "running_max": 0.033549, "scored": true, "final": false}, {"index": 16, "model_call": 54, "tokens": 5410057, "cost_weighted": 208431.8, "wall_clock_seconds": 2770.9, "raw": 0.033404, "normalized": 0.033404, "effective": 0.033404, "running_max": 0.033549, "scored": true, "final": false}, {"index": 17, "model_call": 56, "tokens": 5746711, "cost_weighted": 218890.2, "wall_clock_seconds": 2824.6, "raw": 0.03336, "normalized": 0.03336, "effective": 0.03336, "running_max": 0.033549, "scored": true, "final": false}, {"index": 18, "model_call": 58, "tokens": 6093345, "cost_weighted": 229415.8, "wall_clock_seconds": 2880.6, "raw": 0.033374, "normalized": 0.033374, "effective": 0.033374, "running_max": 0.033549, "scored": true, "final": false}, {"index": 19, "model_call": 60, "tokens": 6449414, "cost_weighted": 239768.4, "wall_clock_seconds": 2930.6, "raw": 0.033435, "normalized": 0.033435, "effective": 0.033435, "running_max": 0.033549, "scored": true, "final": false}, {"index": 20, "model_call": 62, "tokens": 6814582, "cost_weighted": 250217.3, "wall_clock_seconds": 2989.0, "raw": 0.033427, "normalized": 0.033427, "effective": 0.033427, "running_max": 0.033549, "scored": true, "final": false}, {"index": 21, "model_call": 64, "tokens": 7189029, "cost_weighted": 261024.4, "wall_clock_seconds": 3043.0, "raw": 0.033482, "normalized": 0.033482, "effective": 0.033482, "running_max": 0.033549, "scored": true, "final": false}, {"index": 22, "model_call": 66, "tokens": 7573050, "cost_weighted": 272183.3, "wall_clock_seconds": 3099.4, "raw": 0.033446, "normalized": 0.033446, "effective": 0.033446, "running_max": 0.033549, "scored": true, "final": false}, {"index": 23, "model_call": 68, "tokens": 7966945, "cost_weighted": 283751.2, "wall_clock_seconds": 3156.1, "raw": 0.033462, "normalized": 0.033462, "effective": 0.033462, "running_max": 0.033549, "scored": true, "final": false}, {"index": 24, "model_call": 71, "tokens": 8573899, "cost_weighted": 300881.6, "wall_clock_seconds": 3280.9, "raw": 0.033466, "normalized": 0.033466, "effective": 0.033466, "running_max": 0.033549, "scored": true, "final": false}, {"index": 25, "model_call": 73, "tokens": 8990766, "cost_weighted": 312968.6, "wall_clock_seconds": 3337.0, "raw": 0.033456, "normalized": 0.033456, "effective": 0.033456, "running_max": 0.033549, "scored": true, "final": false}, {"index": 26, "model_call": 75, "tokens": 9359773, "cost_weighted": 357339.9, "wall_clock_seconds": 3393.5, "raw": 0.033451, "normalized": 0.033451, "effective": 0.033451, "running_max": 0.033549, "scored": true, "final": false}, {"index": 27, "model_call": 77, "tokens": 9680817, "cost_weighted": 367256.2, "wall_clock_seconds": 3448.6, "raw": 0.03345, "normalized": 0.03345, "effective": 0.03345, "running_max": 0.033549, "scored": true, "final": false}, {"index": 28, "model_call": 79, "tokens": 10011308, "cost_weighted": 377119.5, "wall_clock_seconds": 3494.8, "raw": 0.033439, "normalized": 0.033439, "effective": 0.033439, "running_max": 0.033549, "scored": true, "final": false}, {"index": 29, "model_call": 81, "tokens": 10351323, "cost_weighted": 387460.6, "wall_clock_seconds": 3547.1, "raw": 0.033355, "normalized": 0.033355, "effective": 0.033355, "running_max": 0.033549, "scored": true, "final": false}, {"index": 30, "model_call": 83, "tokens": 10701137, "cost_weighted": 397989.9, "wall_clock_seconds": 3597.9, "raw": 0.033309, "normalized": 0.033309, "effective": 0.033309, "running_max": 0.033549, "scored": true, "final": false}, {"index": 31, "model_call": 103, "tokens": 11634404, "cost_weighted": 480580.6, "wall_clock_seconds": 3897.3, "raw": 0.033484, "normalized": 0.033484, "effective": 0.033484, "running_max": 0.033549, "scored": true, "final": false}, {"index": 32, "model_call": 111, "tokens": 12203648, "cost_weighted": 500490.4, "wall_clock_seconds": 4040.1, "raw": 0.033596, "normalized": 0.033596, "effective": 0.033596, "running_max": 0.033596, "scored": true, "final": false}, {"index": 33, "model_call": 120, "tokens": 12805993, "cost_weighted": 555771.9, "wall_clock_seconds": 4222.2, "raw": 0.033722, "normalized": 0.033722, "effective": 0.033722, "running_max": 0.033722, "scored": true, "final": false}, {"index": 34, "model_call": 126, "tokens": 13016356, "cost_weighted": 566946.8, "wall_clock_seconds": 4305.9, "raw": 0.033812, "normalized": 0.033812, "effective": 0.033812, "running_max": 0.033812, "scored": true, "final": false}, {"index": 35, "model_call": 131, "tokens": 13261107, "cost_weighted": 580024.6, "wall_clock_seconds": 4429.0, "raw": 0.0339, "normalized": 0.0339, "effective": 0.0339, "running_max": 0.0339, "scored": true, "final": false}, {"index": 36, "model_call": 139, "tokens": 13815897, "cost_weighted": 604022.7, "wall_clock_seconds": 4611.7, "raw": 0.0341, "normalized": 0.0341, "effective": 0.0341, "running_max": 0.0341, "scored": true, "final": false}, {"index": 37, "model_call": 143, "tokens": 14159325, "cost_weighted": 616302.0, "wall_clock_seconds": 4712.5, "raw": 0.034154, "normalized": 0.034154, "effective": 0.034154, "running_max": 0.034154, "scored": true, "final": false}, {"index": 38, "model_call": 148, "tokens": 14588910, "cost_weighted": 647359.6, "wall_clock_seconds": 4831.9, "raw": 0.034303, "normalized": 0.034303, "effective": 0.034303, "running_max": 0.034303, "scored": true, "final": false}, {"index": 39, "model_call": 162, "tokens": 15243898, "cost_weighted": 706558.9, "wall_clock_seconds": 5145.7, "raw": 0.034442, "normalized": 0.034442, "effective": 0.034442, "running_max": 0.034442, "scored": true, "final": false}, {"index": 40, "model_call": 171, "tokens": 15863180, "cost_weighted": 736133.8, "wall_clock_seconds": 5440.7, "raw": 0.034534, "normalized": 0.034534, "effective": 0.034534, "running_max": 0.034534, "scored": true, "final": false}, {"index": 41, "model_call": 174, "tokens": 16123553, "cost_weighted": 757687.8, "wall_clock_seconds": 5539.4, "raw": 0.03454, "normalized": 0.03454, "effective": 0.03454, "running_max": 0.03454, "scored": true, "final": false}, {"index": 42, "model_call": 178, "tokens": 16465113, "cost_weighted": 770615.0, "wall_clock_seconds": 5666.9, "raw": 0.034788, "normalized": 0.034788, "effective": 0.034788, "running_max": 0.034788, "scored": true, "final": false}, {"index": 43, "model_call": 213, "tokens": 17824552, "cost_weighted": 836804.9, "wall_clock_seconds": 6046.5, "raw": 0.035233, "normalized": 0.035233, "effective": 0.035233, "running_max": 0.035233, "scored": true, "final": false}, {"index": 44, "model_call": 230, "tokens": 18829629, "cost_weighted": 866124.0, "wall_clock_seconds": 6314.2, "raw": 0.035757, "normalized": 0.035757, "effective": 0.035757, "running_max": 0.035757, "scored": true, "final": false}, {"index": 45, "model_call": 249, "tokens": 20219394, "cost_weighted": 916490.7, "wall_clock_seconds": 6648.3, "raw": 0.036278, "normalized": 0.036278, "effective": 0.036278, "running_max": 0.036278, "scored": true, "final": false}, {"index": 46, "model_call": 266, "tokens": 21168771, "cost_weighted": 975010.4, "wall_clock_seconds": 6924.7, "raw": 0.036821, "normalized": 0.036821, "effective": 0.036821, "running_max": 0.036821, "scored": true, "final": false}, {"index": 47, "model_call": 274, "tokens": 21668233, "cost_weighted": 998784.5, "wall_clock_seconds": 7108.6, "raw": 0.037219, "normalized": 0.037219, "effective": 0.037219, "running_max": 0.037219, "scored": true, "final": false}, {"index": 48, "model_call": 279, "tokens": 22071251, "cost_weighted": 1015711.4, "wall_clock_seconds": 7241.4, "raw": 0.03751, "normalized": 0.03751, "effective": 0.03751, "running_max": 0.03751, "scored": true, "final": false}, {"index": 49, "model_call": 284, "tokens": 22534257, "cost_weighted": 1033235.9, "wall_clock_seconds": 7379.8, "raw": 0.037851, "normalized": 0.037851, "effective": 0.037851, "running_max": 0.037851, "scored": true, "final": false}, {"index": 50, "model_call": 287, "tokens": 22843095, "cost_weighted": 1044778.2, "wall_clock_seconds": 7469.6, "raw": 0.037884, "normalized": 0.037884, "effective": 0.037884, "running_max": 0.037884, "scored": true, "final": false}, {"index": 51, "model_call": 290, "tokens": 23136672, "cost_weighted": 1073569.0, "wall_clock_seconds": 7539.4, "raw": 0.038065, "normalized": 0.038065, "effective": 0.038065, "running_max": 0.038065, "scored": true, "final": false}, {"index": 52, "model_call": 293, "tokens": 23432184, "cost_weighted": 1084991.9, "wall_clock_seconds": 7620.8, "raw": 0.038096, "normalized": 0.038096, "effective": 0.038096, "running_max": 0.038096, "scored": true, "final": false}, {"index": 53, "model_call": 310, "tokens": 24033906, "cost_weighted": 1138935.9, "wall_clock_seconds": 7897.1, "raw": 0.03821, "normalized": 0.03821, "effective": 0.03821, "running_max": 0.03821, "scored": true, "final": false}, {"index": 54, "model_call": 320, "tokens": 24666501, "cost_weighted": 1166868.3, "wall_clock_seconds": 8143.1, "raw": 0.038358, "normalized": 0.038358, "effective": 0.038358, "running_max": 0.038358, "scored": true, "final": false}, {"index": 55, "model_call": 346, "tokens": 25671762, "cost_weighted": 1236972.8, "wall_clock_seconds": 8488.8, "raw": 0.03857, "normalized": 0.03857, "effective": 0.03857, "running_max": 0.03857, "scored": true, "final": false}, {"index": 56, "model_call": 375, "tokens": 27427257, "cost_weighted": 1289128.1, "wall_clock_seconds": 8892.7, "raw": 0.038971, "normalized": 0.038971, "effective": 0.038971, "running_max": 0.038971, "scored": true, "final": false}, {"index": 57, "model_call": 406, "tokens": 30261388, "cost_weighted": 1379119.9, "wall_clock_seconds": 9370.8, "raw": 0.039343, "normalized": 0.039343, "effective": 0.039343, "running_max": 0.039343, "scored": true, "final": false}, {"index": 58, "model_call": 438, "tokens": 31587434, "cost_weighted": 1445862.9, "wall_clock_seconds": 9728.9, "raw": 0.039964, "normalized": 0.039964, "effective": 0.039964, "running_max": 0.039964, "scored": true, "final": false}, {"index": 59, "model_call": 443, "tokens": 31799722, "cost_weighted": 1458824.8, "wall_clock_seconds": 9855.6, "raw": 0.040187, "normalized": 0.040187, "effective": 0.040187, "running_max": 0.040187, "scored": true, "final": false}, {"index": 60, "model_call": 445, "tokens": 31911409, "cost_weighted": 1468666.8, "wall_clock_seconds": 9937.6, "raw": 0.040309, "normalized": 0.040309, "effective": 0.040309, "running_max": 0.040309, "scored": true, "final": false}, {"index": 61, "model_call": 447, "tokens": 32044079, "cost_weighted": 1479533.6, "wall_clock_seconds": 10028.8, "raw": 0.040407, "normalized": 0.040407, "effective": 0.040407, "running_max": 0.040407, "scored": true, "final": false}, {"index": 62, "model_call": 450, "tokens": 32271409, "cost_weighted": 1492494.9, "wall_clock_seconds": 10133.0, "raw": 0.040487, "normalized": 0.040487, "effective": 0.040487, "running_max": 0.040487, "scored": true, "final": false}, {"index": 63, "model_call": 452, "tokens": 32447740, "cost_weighted": 1503946.9, "wall_clock_seconds": 10217.0, "raw": 0.0405, "normalized": 0.0405, "effective": 0.0405, "running_max": 0.0405, "scored": true, "final": false}, {"index": 64, "model_call": 454, "tokens": 32645253, "cost_weighted": 1517353.5, "wall_clock_seconds": 10304.7, "raw": 0.040482, "normalized": 0.040482, "effective": 0.040482, "running_max": 0.0405, "scored": true, "final": false}, {"index": 65, "model_call": 457, "tokens": 32969189, "cost_weighted": 1532399.2, "wall_clock_seconds": 10412.8, "raw": 0.040547, "normalized": 0.040547, "effective": 0.040547, "running_max": 0.040547, "scored": true, "final": false}, {"index": 66, "model_call": 459, "tokens": 33209945, "cost_weighted": 1545072.9, "wall_clock_seconds": 10501.1, "raw": 0.040428, "normalized": 0.040428, "effective": 0.040428, "running_max": 0.040547, "scored": true, "final": false}, {"index": 67, "model_call": 461, "tokens": 33452796, "cost_weighted": 1580684.0, "wall_clock_seconds": 10586.5, "raw": 0.040474, "normalized": 0.040474, "effective": 0.040474, "running_max": 0.040547, "scored": true, "final": false}, {"index": 68, "model_call": 464, "tokens": 33816822, "cost_weighted": 1596515.6, "wall_clock_seconds": 10700.4, "raw": 0.040616, "normalized": 0.040616, "effective": 0.040616, "running_max": 0.040616, "scored": true, "final": false}, {"index": 69, "model_call": 466, "tokens": 34084339, "cost_weighted": 1609686.4, "wall_clock_seconds": 10779.4, "raw": 0.040569, "normalized": 0.040569, "effective": 0.040569, "running_max": 0.040616, "scored": true, "final": false}, {"index": 70, "model_call": 472, "tokens": 34339846, "cost_weighted": 1663762.1, "wall_clock_seconds": 10988.1, "raw": 0.040695, "normalized": 0.040695, "effective": 0.040695, "running_max": 0.040695, "scored": true, "final": false}, {"index": 71, "model_call": 476, "tokens": 34528547, "cost_weighted": 1678158.5, "wall_clock_seconds": 11100.0, "raw": 0.040784, "normalized": 0.040784, "effective": 0.040784, "running_max": 0.040784, "scored": true, "final": false}, {"index": 72, "model_call": 480, "tokens": 34774639, "cost_weighted": 1692210.8, "wall_clock_seconds": 11213.5, "raw": 0.040937, "normalized": 0.040937, "effective": 0.040937, "running_max": 0.040937, "scored": true, "final": false}, {"index": 73, "model_call": 484, "tokens": 35076615, "cost_weighted": 1707395.1, "wall_clock_seconds": 11323.1, "raw": 0.041154, "normalized": 0.041154, "effective": 0.041154, "running_max": 0.041154, "scored": true, "final": false}, {"index": 74, "model_call": 488, "tokens": 35434788, "cost_weighted": 1723465.3, "wall_clock_seconds": 11437.7, "raw": 0.04128, "normalized": 0.04128, "effective": 0.04128, "running_max": 0.04128, "scored": true, "final": false}, {"index": 75, "model_call": 492, "tokens": 35845391, "cost_weighted": 1741351.4, "wall_clock_seconds": 11559.0, "raw": 0.041361, "normalized": 0.041361, "effective": 0.041361, "running_max": 0.041361, "scored": true, "final": false}, {"index": 76, "model_call": 496, "tokens": 36313942, "cost_weighted": 1760809.2, "wall_clock_seconds": 11699.0, "raw": 0.041555, "normalized": 0.041555, "effective": 0.041555, "running_max": 0.041555, "scored": true, "final": false}, {"index": 77, "model_call": 500, "tokens": 36789091, "cost_weighted": 1799830.6, "wall_clock_seconds": 11811.1, "raw": 0.041781, "normalized": 0.041781, "effective": 0.041781, "running_max": 0.041781, "scored": true, "final": false}, {"index": 78, "model_call": 503, "tokens": 37142214, "cost_weighted": 1814024.1, "wall_clock_seconds": 11902.4, "raw": 0.041835, "normalized": 0.041835, "effective": 0.041835, "running_max": 0.041835, "scored": true, "final": false}, {"index": 79, "model_call": 510, "tokens": 37417043, "cost_weighted": 1863261.8, "wall_clock_seconds": 12091.3, "raw": 0.042018, "normalized": 0.042018, "effective": 0.042018, "running_max": 0.042018, "scored": true, "final": false}, {"index": 80, "model_call": 515, "tokens": 37646662, "cost_weighted": 1877971.2, "wall_clock_seconds": 12227.0, "raw": 0.042096, "normalized": 0.042096, "effective": 0.042096, "running_max": 0.042096, "scored": true, "final": false}, {"index": 81, "model_call": 519, "tokens": 37890422, "cost_weighted": 1892587.0, "wall_clock_seconds": 12354.9, "raw": 0.042106, "normalized": 0.042106, "effective": 0.042106, "running_max": 0.042106, "scored": true, "final": false}, {"index": 82, "model_call": 523, "tokens": 38187677, "cost_weighted": 1907821.2, "wall_clock_seconds": 12472.0, "raw": 0.042239, "normalized": 0.042239, "effective": 0.042239, "running_max": 0.042239, "scored": true, "final": false}, {"index": 83, "model_call": 527, "tokens": 38539424, "cost_weighted": 1924200.8, "wall_clock_seconds": 12584.9, "raw": 0.042376, "normalized": 0.042376, "effective": 0.042376, "running_max": 0.042376, "scored": true, "final": false}, {"index": 84, "model_call": 531, "tokens": 38947100, "cost_weighted": 1941504.8, "wall_clock_seconds": 12703.7, "raw": 0.042489, "normalized": 0.042489, "effective": 0.042489, "running_max": 0.042489, "scored": true, "final": false}, {"index": 85, "model_call": 534, "tokens": 39284060, "cost_weighted": 1953384.1, "wall_clock_seconds": 12783.9, "raw": 0.042537, "normalized": 0.042537, "effective": 0.042537, "running_max": 0.042537, "scored": true, "final": false}, {"index": 86, "model_call": 537, "tokens": 39640219, "cost_weighted": 1965224.3, "wall_clock_seconds": 12865.1, "raw": 0.042514, "normalized": 0.042514, "effective": 0.042514, "running_max": 0.042537, "scored": true, "final": false} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:ship_tune", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.084384, "area_term": 0.074425, "final_best": 0.124222, "best_raw": 0.124222, "best_effective": 0.124222, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 18156487, "usage": {"input": 522, "output": 123013, "cache_read": 17583768, "cache_write": 449184}, "cost_usd": 5.8709, "cost_source": "billed", "wall_clock_seconds": 18842.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 223602, "wall_clock_seconds": 1.4, "raw": 0.006143, "normalized": 0.006143, "discount": 1.0, "effective": 0.006143, "running_max": 0.006143, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.67}, {"index": 2, "tokens": 2526780, "wall_clock_seconds": 57.2, "raw": 0.081416, "normalized": 0.081416, "discount": 1.0, "effective": 0.081416, "running_max": 0.081416, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 56.48}, {"index": 3, "tokens": 5529553, "wall_clock_seconds": 56.8, "raw": 0.112079, "normalized": 0.112079, "discount": 1.0, "effective": 0.112079, "running_max": 0.112079, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 56.12}, {"index": 4, "tokens": 5719389, "wall_clock_seconds": 249.2, "raw": 0.116736, "normalized": 0.116736, "discount": 1.0, "effective": 0.116736, "running_max": 0.116736, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 248.49}, {"index": 5, "tokens": 6930545, "wall_clock_seconds": 387.9, "raw": 0.118826, "normalized": 0.118826, "discount": 1.0, "effective": 0.118826, "running_max": 0.118826, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 387.13}, {"index": 6, "tokens": 8711705, "wall_clock_seconds": 271.1, "raw": 0.119326, "normalized": 0.119326, "discount": 1.0, "effective": 0.119326, "running_max": 0.119326, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 270.4}, {"index": 7, "tokens": 11640017, "wall_clock_seconds": 146.8, "raw": 0.124222, "normalized": 0.124222, "discount": 1.0, "effective": 0.124222, "running_max": 0.124222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 146.06}, {"index": 8, "tokens": 15323388, "wall_clock_seconds": 100.6, "raw": 0.112296, "normalized": 0.112296, "discount": 1.0, "effective": 0.112296, "running_max": 0.124222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 99.86}, {"index": 9, "tokens": 16888236, "wall_clock_seconds": 156.2, "raw": 0.120493, "normalized": 0.120493, "discount": 1.0, "effective": 0.120493, "running_max": 0.124222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 155.48}, {"index": 10, "tokens": 18156487, "wall_clock_seconds": 115.5, "raw": 0.12309, "normalized": 0.12309, "discount": 1.0, "effective": 0.12309, "running_max": 0.124222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 114.71} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:splash_generalize", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.207167, "area_term": 0.18632, "final_best": 0.290556, "best_raw": 0.623, "best_effective": 0.290556, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11377814, "usage": {"input": 372, "output": 73419, "cache_read": 10993434, "cache_write": 310589}, "cost_usd": 3.7101, "cost_source": "billed", "wall_clock_seconds": 6878.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.5, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1443347, "wall_clock_seconds": 8.0, "raw": 0.423, "normalized": 0.358889, "discount": 0.5, "effective": 0.179444, "running_max": 0.179444, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 7.32}, {"index": 2, "tokens": 2269307, "wall_clock_seconds": 7.3, "raw": 0.371, "normalized": 0.301111, "discount": 0.5, "effective": 0.150556, "running_max": 0.179444, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 6.68}, {"index": 3, "tokens": 4092235, "wall_clock_seconds": 31.0, "raw": 0.457, "normalized": 0.396667, "discount": 0.5, "effective": 0.198333, "running_max": 0.198333, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 30.36}, {"index": 4, "tokens": 4351834, "wall_clock_seconds": 34.5, "raw": 0.457, "normalized": 0.396667, "discount": 0.5, "effective": 0.198333, "running_max": 0.198333, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 33.86}, {"index": 5, "tokens": 5906415, "wall_clock_seconds": 34.1, "raw": 0.524, "normalized": 0.471111, "discount": 0.5, "effective": 0.235556, "running_max": 0.235556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 33.59}, {"index": 6, "tokens": 6654199, "wall_clock_seconds": 34.2, "raw": 0.55, "normalized": 0.5, "discount": 0.5, "effective": 0.25, "running_max": 0.25, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 33.67}, {"index": 7, "tokens": 8220911, "wall_clock_seconds": 33.6, "raw": 0.552, "normalized": 0.502222, "discount": 0.5, "effective": 0.251111, "running_max": 0.251111, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 33.06}, {"index": 8, "tokens": 8710236, "wall_clock_seconds": 40.0, "raw": 0.576, "normalized": 0.528889, "discount": 0.5, "effective": 0.264444, "running_max": 0.264444, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 39.44}, {"index": 9, "tokens": 9568243, "wall_clock_seconds": 47.0, "raw": 0.597, "normalized": 0.552222, "discount": 0.5, "effective": 0.276111, "running_max": 0.276111, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 46.39}, {"index": 10, "tokens": 11377814, "wall_clock_seconds": 45.3, "raw": 0.623, "normalized": 0.581111, "discount": 0.5, "effective": 0.290556, "running_max": 0.290556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 44.79} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:tod_pipeline--nohints", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.08744, "area_term": 0.081892, "final_best": 0.109632, "best_raw": 0.109632, "best_effective": 0.109632, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9490421, "usage": {"input": 270, "output": 70465, "cache_read": 9245434, "cache_write": 174252}, "cost_usd": 2.9899, "cost_source": "billed", "wall_clock_seconds": 2543.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1182382, "wall_clock_seconds": 14.2, "raw": 0.072256, "normalized": 0.072256, "discount": 1.0, "effective": 0.072256, "running_max": 0.072256, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.54}, {"index": 2, "tokens": 1676542, "wall_clock_seconds": 13.2, "raw": 0.054385, "normalized": 0.054385, "discount": 1.0, "effective": 0.054385, "running_max": 0.072256, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.0}, {"index": 3, "tokens": 2858048, "wall_clock_seconds": 14.4, "raw": 0.108885, "normalized": 0.108885, "discount": 1.0, "effective": 0.108885, "running_max": 0.108885, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.17}, {"index": 4, "tokens": 4646886, "wall_clock_seconds": 14.9, "raw": 0.109569, "normalized": 0.109569, "discount": 1.0, "effective": 0.109569, "running_max": 0.109569, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.52}, {"index": 5, "tokens": 5170454, "wall_clock_seconds": 14.6, "raw": 0.097294, "normalized": 0.097294, "discount": 1.0, "effective": 0.097294, "running_max": 0.109569, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.04}, {"index": 6, "tokens": 6727889, "wall_clock_seconds": 16.3, "raw": 0.10944, "normalized": 0.10944, "discount": 1.0, "effective": 0.10944, "running_max": 0.109569, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.99}, {"index": 7, "tokens": 7179524, "wall_clock_seconds": 16.6, "raw": 0.10954, "normalized": 0.10954, "discount": 1.0, "effective": 0.10954, "running_max": 0.109569, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 16.35}, {"index": 8, "tokens": 7804884, "wall_clock_seconds": 21.7, "raw": 0.108221, "normalized": 0.108221, "discount": 1.0, "effective": 0.108221, "running_max": 0.109569, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.38}, {"index": 9, "tokens": 8295130, "wall_clock_seconds": 18.4, "raw": 0.109632, "normalized": 0.109632, "discount": 1.0, "effective": 0.109632, "running_max": 0.109632, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.1}, {"index": 10, "tokens": 9490421, "wall_clock_seconds": 17.8, "raw": 0.109416, "normalized": 0.109416, "discount": 1.0, "effective": 0.109416, "running_max": 0.109632, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 17.48} ]}, {"sample_key": "9885nfw6Q66EL4MassEo9x:weirdml_bonanza", "eval_id": "9885nfw6Q66EL4MassEo9x", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gpt-5.6-sol-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "5bd1a28", "source_log_sha256": "d4b3df366d7e75f6bae06cad32571b0dbe6f881af8dca519ead7e96f7f2a2745", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.57161, "area_term": 0.550203, "final_best": 0.657237, "best_raw": 0.657237, "best_effective": 0.657237, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 8120812, "usage": {"input": 228, "output": 73856, "cache_read": 7885331, "cache_write": 161397}, "cost_usd": 2.7196, "cost_source": "billed", "wall_clock_seconds": 4403.7, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 586054, "wall_clock_seconds": 120.6, "raw": 0.227759, "normalized": 0.227759, "discount": 1.0, "effective": 0.227759, "running_max": 0.227759, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.0}, {"index": 2, "tokens": 728896, "wall_clock_seconds": 19.1, "raw": 0.298534, "normalized": 0.298534, "discount": 1.0, "effective": 0.298534, "running_max": 0.298534, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.41}, {"index": 3, "tokens": 873821, "wall_clock_seconds": 111.8, "raw": 0.454365, "normalized": 0.454365, "discount": 1.0, "effective": 0.454365, "running_max": 0.454365, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 110.9}, {"index": 4, "tokens": 1716128, "wall_clock_seconds": 114.5, "raw": 0.581555, "normalized": 0.581555, "discount": 1.0, "effective": 0.581555, "running_max": 0.581555, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 113.96}, {"index": 5, "tokens": 3519669, "wall_clock_seconds": 120.8, "raw": 0.411843, "normalized": 0.411843, "discount": 1.0, "effective": 0.411843, "running_max": 0.581555, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.0}, {"index": 6, "tokens": 3754507, "wall_clock_seconds": 88.7, "raw": 0.59622, "normalized": 0.59622, "discount": 1.0, "effective": 0.59622, "running_max": 0.59622, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 88.18}, {"index": 7, "tokens": 5295068, "wall_clock_seconds": 101.8, "raw": 0.627649, "normalized": 0.627649, "discount": 1.0, "effective": 0.627649, "running_max": 0.627649, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 101.25}, {"index": 8, "tokens": 6001953, "wall_clock_seconds": 120.6, "raw": 0.345843, "normalized": 0.345843, "discount": 1.0, "effective": 0.345843, "running_max": 0.627649, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.01}, {"index": 9, "tokens": 6581720, "wall_clock_seconds": 120.8, "raw": 0.459819, "normalized": 0.459819, "discount": 1.0, "effective": 0.459819, "running_max": 0.627649, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 120.0}, {"index": 10, "tokens": 8120812, "wall_clock_seconds": 103.8, "raw": 0.657237, "normalized": 0.657237, "discount": 1.0, "effective": 0.657237, "running_max": 0.657237, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 103.13} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:mystery_box", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.015323, "area_term": 0.012111, "final_best": 0.028171, "best_raw": 0.064227, "best_effective": 0.028171, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 27581818, "usage": {"input": 238234, "output": 138958, "cache_read": 27204626, "cache_write": 26786674}, "cost_usd": 3.8571, "cost_source": "list_price", "wall_clock_seconds": 2374.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 4}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4078881, "wall_clock_seconds": 4.6, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7899143, "wall_clock_seconds": 4.7, "raw": 0.038963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8447969, "wall_clock_seconds": 3.4, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 16689252, "wall_clock_seconds": 5.9, "raw": 0.055074, "normalized": 0.017528, "discount": 0.25, "effective": 0.004382, "running_max": 0.028171, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 18899411, "wall_clock_seconds": 5.4, "raw": 0.054914, "normalized": 0.017342, "discount": 0.25, "effective": 0.004336, "running_max": 0.028171, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 23485452, "wall_clock_seconds": 11.2, "raw": 0.056087, "normalized": 0.018706, "discount": 0.25, "effective": 0.004676, "running_max": 0.028171, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 24315102, "wall_clock_seconds": 12.7, "raw": 0.050375, "normalized": 0.012064, "discount": 0.25, "effective": 0.003016, "running_max": 0.028171, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 25605611, "wall_clock_seconds": 7.7, "raw": 0.056261, "normalized": 0.018908, "discount": 0.25, "effective": 0.004727, "running_max": 0.028171, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 26258042, "wall_clock_seconds": 7.4, "raw": 0.057568, "normalized": 0.020427, "discount": 0.25, "effective": 0.005107, "running_max": 0.028171, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 27581818, "wall_clock_seconds": 10.1, "raw": 0.057214, "normalized": 0.020017, "discount": 0.25, "effective": 0.005004, "running_max": 0.028171, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:mystery_box--nohints", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.021155, "area_term": 0.017656, "final_best": 0.035153, "best_raw": 0.070232, "best_effective": 0.035153, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9128820, "usage": {"input": 132064, "output": 61646, "cache_read": 8935110, "cache_write": 8731791}, "cost_usd": 1.3645, "cost_source": "list_price", "wall_clock_seconds": 1197.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4575045, "wall_clock_seconds": 3.8, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4758454, "wall_clock_seconds": 4.3, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 5047977, "wall_clock_seconds": 3.8, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 5331664, "wall_clock_seconds": 6.5, "raw": 0.03853, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 5922817, "wall_clock_seconds": 4.8, "raw": 0.064623, "normalized": 0.028631, "discount": 1.0, "effective": 0.028631, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 6021827, "wall_clock_seconds": 3.7, "raw": 0.070135, "normalized": 0.03504, "discount": 1.0, "effective": 0.03504, "running_max": 0.03504, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 6530455, "wall_clock_seconds": 2.9, "raw": 0.068236, "normalized": 0.032833, "discount": 1.0, "effective": 0.032833, "running_max": 0.03504, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 7281847, "wall_clock_seconds": 2.8, "raw": 0.048241, "normalized": 0.009583, "discount": 1.0, "effective": 0.009583, "running_max": 0.03504, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8184423, "wall_clock_seconds": 2.8, "raw": 0.067743, "normalized": 0.03226, "discount": 1.0, "effective": 0.03226, "running_max": 0.03504, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9128820, "wall_clock_seconds": 2.8, "raw": 0.070232, "normalized": 0.035153, "discount": 1.0, "effective": 0.035153, "running_max": 0.035153, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:night_school", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.049002, "area_term": 0.04701, "final_best": 0.056971, "best_raw": 0.2245, "best_effective": 0.056971, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 35196228, "usage": {"input": 289834, "output": 194117, "cache_read": 34712277, "cache_write": 34488369}, "cost_usd": 4.9869, "cost_source": "list_price", "wall_clock_seconds": 3517.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1012084, "wall_clock_seconds": 3.1, "raw": 0.197, "normalized": 0.055294, "discount": 1.0, "effective": 0.055294, "running_max": 0.055294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 10162680, "wall_clock_seconds": 9.8, "raw": 0.2075, "normalized": 0.067647, "discount": 0.65, "effective": 0.043971, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 13577992, "wall_clock_seconds": 11.1, "raw": 0.2135, "normalized": 0.074706, "discount": 0.65, "effective": 0.048559, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 19858441, "wall_clock_seconds": 12.2, "raw": 0.2115, "normalized": 0.072353, "discount": 0.65, "effective": 0.047029, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 24110970, "wall_clock_seconds": 12.0, "raw": 0.197, "normalized": 0.055294, "discount": 0.65, "effective": 0.035941, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 27363027, "wall_clock_seconds": 14.8, "raw": 0.2045, "normalized": 0.064118, "discount": 0.65, "effective": 0.041676, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 30286229, "wall_clock_seconds": 12.2, "raw": 0.2245, "normalized": 0.087647, "discount": 0.65, "effective": 0.056971, "running_max": 0.056971, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 31286090, "wall_clock_seconds": 13.7, "raw": 0.2225, "normalized": 0.085294, "discount": 0.65, "effective": 0.055441, "running_max": 0.056971, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 33870199, "wall_clock_seconds": 13.3, "raw": 0.2205, "normalized": 0.082941, "discount": 0.65, "effective": 0.053912, "running_max": 0.056971, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 35196228, "wall_clock_seconds": 12.4, "raw": 0.2245, "normalized": 0.087647, "discount": 0.65, "effective": 0.056971, "running_max": 0.056971, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:reaction_rates", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.007769, "area_term": 0.001828, "final_best": 0.031532, "best_raw": 0.056307, "best_effective": 0.031532, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 7, "max_submissions": 10, "tokens_total": 47920627, "usage": {"input": 354069, "output": 194894, "cache_read": 47371664, "cache_write": 45259604}, "cost_usd": 6.4366, "cost_source": "list_price", "wall_clock_seconds": 3500.7, "stop_reason": "agent_finished", "hints": [{"id": 4, "multiplier": 0.8, "before_submission": 2}, {"id": 1, "multiplier": 0.7, "before_submission": 5}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2911914, "wall_clock_seconds": 1.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.96}, {"index": 2, "tokens": 26463599, "wall_clock_seconds": 8.8, "raw": 0.0, "normalized": 0.0, "discount": 0.8, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 8.2}, {"index": 3, "tokens": 30159270, "wall_clock_seconds": 26.7, "raw": 0.01448, "normalized": 0.01448, "discount": 0.8, "effective": 0.011584, "running_max": 0.011584, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 21.84}, {"index": 4, "tokens": 34368391, "wall_clock_seconds": 12.5, "raw": 0.011907, "normalized": 0.011907, "discount": 0.8, "effective": 0.009525, "running_max": 0.011584, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 11.84}, {"index": 5, "tokens": 41356735, "wall_clock_seconds": 45.8, "raw": 0.020526, "normalized": 0.020526, "discount": 0.56, "effective": 0.011494, "running_max": 0.011584, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 45.12}, {"index": 6, "tokens": 43676113, "wall_clock_seconds": 85.2, "raw": 0.051801, "normalized": 0.051801, "discount": 0.56, "effective": 0.029009, "running_max": 0.029009, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 84.5}, {"index": 7, "tokens": 46069292, "wall_clock_seconds": 87.5, "raw": 0.056307, "normalized": 0.056307, "discount": 0.56, "effective": 0.031532, "running_max": 0.031532, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 86.86} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:reaction_rates--nohints", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.040674, "area_term": 0.012127, "final_best": 0.154861, "best_raw": 0.154861, "best_effective": 0.154861, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 50073794, "usage": {"input": 328028, "output": 207210, "cache_read": 49538556, "cache_write": 48770474}, "cost_usd": 6.7722, "cost_source": "list_price", "wall_clock_seconds": 3562.3, "stop_reason": "token_limit", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1028121, "wall_clock_seconds": 4.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.25}, {"index": 2, "tokens": 22018144, "wall_clock_seconds": 15.3, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 14.95}, {"index": 3, "tokens": 22864843, "wall_clock_seconds": 21.9, "raw": 0.063703, "normalized": 0.063703, "discount": 1.0, "effective": 0.063703, "running_max": 0.063703, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.23}, {"index": 4, "tokens": 39093157, "wall_clock_seconds": 21.2, "raw": 0.069138, "normalized": 0.069138, "discount": 1.0, "effective": 0.069138, "running_max": 0.069138, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 20.53}, {"index": 5, "tokens": 42114492, "wall_clock_seconds": 25.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.069138, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 24.94}, {"index": 6, "tokens": 47350220, "wall_clock_seconds": 21.9, "raw": 0.154861, "normalized": 0.154861, "discount": 1.0, "effective": 0.154861, "running_max": 0.154861, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.18} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:scan_stitch", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.000185, "area_term": 1.2e-05, "final_best": 0.00088, "best_raw": 0.102012, "best_effective": 0.00088, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 49763157, "usage": {"input": 404466, "output": 218231, "cache_read": 49140460, "cache_write": 46223588}, "cost_usd": 6.7348, "cost_source": "list_price", "wall_clock_seconds": 3894.7, "stop_reason": "agent_finished", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3644437, "wall_clock_seconds": 0.3, "raw": 0.098746, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 14168426, "wall_clock_seconds": 0.3, "raw": 0.090248, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 36137504, "wall_clock_seconds": 0.3, "raw": 0.090588, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 42630155, "wall_clock_seconds": 0.4, "raw": 0.091655, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 46298127, "wall_clock_seconds": 0.3, "raw": 0.093869, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 46638409, "wall_clock_seconds": 0.4, "raw": 0.101687, "normalized": 0.002109, "discount": 0.35, "effective": 0.000738, "running_max": 0.000738, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 47667433, "wall_clock_seconds": 0.3, "raw": 0.09763, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.000738, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 48011734, "wall_clock_seconds": 0.4, "raw": 0.098538, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.000738, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 49065475, "wall_clock_seconds": 0.3, "raw": 0.102012, "normalized": 0.002515, "discount": 0.35, "effective": 0.00088, "running_max": 0.00088, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:scan_stitch--nohints", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.099305, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 38118494, "usage": {"input": 439007, "output": 242178, "cache_read": 37437309, "cache_write": 34218301}, "cost_usd": 5.4721, "cost_source": "list_price", "wall_clock_seconds": 7558.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2634015, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 17936119, "wall_clock_seconds": 0.3, "raw": 0.033872, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 19588156, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 22644414, "wall_clock_seconds": 0.2, "raw": 0.079027, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 25081318, "wall_clock_seconds": 0.3, "raw": 0.082749, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 26120327, "wall_clock_seconds": 0.3, "raw": 0.098388, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 28887745, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 30867811, "wall_clock_seconds": 0.3, "raw": 0.099305, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 34767202, "wall_clock_seconds": 0.3, "raw": 0.082371, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 38118494, "wall_clock_seconds": 0.3, "raw": 0.086429, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:shapes_generalize", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.023769, "area_term": 0.01932, "final_best": 0.041562, "best_raw": 0.295, "best_effective": 0.041562, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 25704980, "usage": {"input": 215957, "output": 120718, "cache_read": 25368305, "cache_write": 25357694}, "cost_usd": 3.5747, "cost_source": "list_price", "wall_clock_seconds": 2377.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.5, "before_submission": 8}, {"id": 2, "multiplier": 0.35, "before_submission": 8}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5813302, "wall_clock_seconds": 1.0, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.33}, {"index": 2, "tokens": 6111845, "wall_clock_seconds": 0.8, "raw": 0.173, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 3, "tokens": 6309158, "wall_clock_seconds": 1.1, "raw": 0.188, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 4, "tokens": 6508499, "wall_clock_seconds": 0.9, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.2}, {"index": 5, "tokens": 6709909, "wall_clock_seconds": 0.8, "raw": 0.206, "normalized": 0.0075, "discount": 1.0, "effective": 0.0075, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 6, "tokens": 10015931, "wall_clock_seconds": 5.9, "raw": 0.208, "normalized": 0.01, "discount": 1.0, "effective": 0.01, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.26}, {"index": 7, "tokens": 10787733, "wall_clock_seconds": 5.8, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.04}, {"index": 8, "tokens": 23395732, "wall_clock_seconds": 7.1, "raw": 0.245, "normalized": 0.05625, "discount": 0.35, "effective": 0.019687, "running_max": 0.04125, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 6.32}, {"index": 9, "tokens": 24152296, "wall_clock_seconds": 7.0, "raw": 0.266, "normalized": 0.0825, "discount": 0.35, "effective": 0.028875, "running_max": 0.04125, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 6.32}, {"index": 10, "tokens": 25704980, "wall_clock_seconds": 7.8, "raw": 0.295, "normalized": 0.11875, "discount": 0.35, "effective": 0.041562, "running_max": 0.041562, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 7.11} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:shattered_prior", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0335, "area_term": 0.029693, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 31209673, "usage": {"input": 252120, "output": 161977, "cache_read": 30795576, "cache_write": 30415539}, "cost_usd": 4.3745, "cost_source": "list_price", "wall_clock_seconds": 4491.9, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.7, "before_submission": 2}, {"id": 3, "multiplier": 0.65, "before_submission": 2}, {"id": 4, "multiplier": 0.55, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3022113, "wall_clock_seconds": 1.8, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 13118468, "wall_clock_seconds": 2.3, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 26745756, "wall_clock_seconds": 2.0, "raw": 0.04873, "normalized": 0.04873, "discount": 0.55, "effective": 0.026802, "running_max": 0.04873, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:shattered_prior--nohints", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.029287, "area_term": 0.023266, "final_best": 0.053374, "best_raw": 0.053374, "best_effective": 0.053374, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 27256123, "usage": {"input": 253906, "output": 129608, "cache_read": 26872609, "cache_write": 24694833}, "cost_usd": 3.7217, "cost_source": "list_price", "wall_clock_seconds": 2140.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5929714, "wall_clock_seconds": 2.1, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 15396470, "wall_clock_seconds": 1.7, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 18120145, "wall_clock_seconds": 2.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 23079224, "wall_clock_seconds": 2.4, "raw": 0.047588, "normalized": 0.047588, "discount": 1.0, "effective": 0.047588, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 24302824, "wall_clock_seconds": 2.1, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 24845661, "wall_clock_seconds": 1.8, "raw": 0.053374, "normalized": 0.053374, "discount": 1.0, "effective": 0.053374, "running_max": 0.053374, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 25389340, "wall_clock_seconds": 2.7, "raw": 0.052928, "normalized": 0.052928, "discount": 1.0, "effective": 0.052928, "running_max": 0.053374, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 25755530, "wall_clock_seconds": 2.7, "raw": 0.053274, "normalized": 0.053274, "discount": 1.0, "effective": 0.053274, "running_max": 0.053374, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 26124570, "wall_clock_seconds": 2.8, "raw": 0.053284, "normalized": 0.053284, "discount": 1.0, "effective": 0.053284, "running_max": 0.053374, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:ship_detect", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.02594, "area_term": 0.021964, "final_best": 0.041846, "best_raw": 0.041846, "best_effective": 0.041846, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": null, "tokens_total": 6140965, "usage": {"input": 115646, "output": 40287, "cache_read": 5985032, "cache_write": 5985032}, "cost_usd": 0.9363, "cost_source": "list_price", "wall_clock_seconds": 1572.4, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 36, "tokens": 1312236, "cost_weighted": 61954.2, "wall_clock_seconds": 404.6, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.0, "scored": true, "final": false}, {"index": 2, "model_call": 45, "tokens": 1785529, "cost_weighted": 80343.1, "wall_clock_seconds": 467.5, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.0, "scored": true, "final": false}, {"index": 3, "model_call": 79, "tokens": 4085050, "cost_weighted": 171966.7, "wall_clock_seconds": 998.0, "raw": 0.035079, "normalized": 0.035079, "effective": 0.035079, "running_max": 0.035079, "scored": true, "final": false}, {"index": 4, "model_call": 91, "tokens": 5179606, "cost_weighted": 215413.9, "wall_clock_seconds": 1438.2, "raw": 0.041846, "normalized": 0.041846, "effective": 0.041846, "running_max": 0.041846, "scored": true, "final": false} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:ship_tune", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.042465, "area_term": 0.029804, "final_best": 0.093108, "best_raw": 0.093108, "best_effective": 0.093108, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 34451402, "usage": {"input": 373394, "output": 166187, "cache_read": 33911821, "cache_write": 30784945}, "cost_usd": 4.7304, "cost_source": "list_price", "wall_clock_seconds": 9078.8, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7871743, "wall_clock_seconds": 120.6, "raw": 0.032098, "normalized": 0.032098, "discount": 1.0, "effective": 0.032098, "running_max": 0.032098, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 119.93}, {"index": 2, "tokens": 10469449, "wall_clock_seconds": 122.3, "raw": 0.034094, "normalized": 0.034094, "discount": 1.0, "effective": 0.034094, "running_max": 0.034094, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 121.63}, {"index": 3, "tokens": 13112542, "wall_clock_seconds": 125.1, "raw": 0.085014, "normalized": 0.085014, "discount": 1.0, "effective": 0.085014, "running_max": 0.085014, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 124.56}, {"index": 4, "tokens": 14765953, "wall_clock_seconds": 120.5, "raw": 0.083014, "normalized": 0.083014, "discount": 1.0, "effective": 0.083014, "running_max": 0.085014, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 119.76}, {"index": 5, "tokens": 17540091, "wall_clock_seconds": 118.3, "raw": 0.085281, "normalized": 0.085281, "discount": 1.0, "effective": 0.085281, "running_max": 0.085281, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 117.69}, {"index": 6, "tokens": 21460429, "wall_clock_seconds": 125.2, "raw": 0.092382, "normalized": 0.092382, "discount": 1.0, "effective": 0.092382, "running_max": 0.092382, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 124.61}, {"index": 7, "tokens": 23110387, "wall_clock_seconds": 123.5, "raw": 0.091315, "normalized": 0.091315, "discount": 1.0, "effective": 0.091315, "running_max": 0.092382, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 122.89}, {"index": 8, "tokens": 26087657, "wall_clock_seconds": 212.5, "raw": 0.092038, "normalized": 0.092038, "discount": 1.0, "effective": 0.092038, "running_max": 0.092382, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 211.85}, {"index": 9, "tokens": 30849350, "wall_clock_seconds": 126.1, "raw": 0.093108, "normalized": 0.093108, "discount": 1.0, "effective": 0.093108, "running_max": 0.093108, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 125.45} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:splash_generalize", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.272052, "area_term": 0.197564, "final_best": 0.57, "best_raw": 0.613, "best_effective": 0.57, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 23080407, "usage": {"input": 371211, "output": 240000, "cache_read": 22469196, "cache_write": 19904790}, "cost_usd": 3.6936, "cost_source": "list_price", "wall_clock_seconds": 3177.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3604577, "wall_clock_seconds": 1.1, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.51}, {"index": 2, "tokens": 10100954, "wall_clock_seconds": 34.0, "raw": 0.611, "normalized": 0.567778, "discount": 1.0, "effective": 0.567778, "running_max": 0.567778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 33.46}, {"index": 3, "tokens": 11386999, "wall_clock_seconds": 43.7, "raw": 0.609, "normalized": 0.565556, "discount": 1.0, "effective": 0.565556, "running_max": 0.567778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 43.02}, {"index": 4, "tokens": 12926131, "wall_clock_seconds": 18.8, "raw": 0.61, "normalized": 0.566667, "discount": 1.0, "effective": 0.566667, "running_max": 0.567778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.12}, {"index": 5, "tokens": 13458976, "wall_clock_seconds": 2.7, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.567778, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 2.04}, {"index": 6, "tokens": 15508147, "wall_clock_seconds": 39.8, "raw": 0.607, "normalized": 0.563333, "discount": 1.0, "effective": 0.563333, "running_max": 0.567778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 39.16}, {"index": 7, "tokens": 16866564, "wall_clock_seconds": 40.9, "raw": 0.597, "normalized": 0.552222, "discount": 1.0, "effective": 0.552222, "running_max": 0.567778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 40.33}, {"index": 8, "tokens": 17891050, "wall_clock_seconds": 33.6, "raw": 0.611, "normalized": 0.567778, "discount": 1.0, "effective": 0.567778, "running_max": 0.567778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 32.9}, {"index": 9, "tokens": 19784503, "wall_clock_seconds": 4.6, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.567778, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 3.86}, {"index": 10, "tokens": 23080407, "wall_clock_seconds": 39.3, "raw": 0.613, "normalized": 0.57, "discount": 1.0, "effective": 0.57, "running_max": 0.57, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 38.75} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:tod_pipeline--nohints", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.040859, "area_term": 0.024156, "final_best": 0.107671, "best_raw": 0.107671, "best_effective": 0.107671, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 27836250, "usage": {"input": 250625, "output": 134497, "cache_read": 27451128, "cache_write": 26853062}, "cost_usd": 3.8709, "cost_source": "list_price", "wall_clock_seconds": 2307.6, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 17130358, "wall_clock_seconds": 9.9, "raw": 0.095313, "normalized": 0.095313, "discount": 1.0, "effective": 0.095313, "running_max": 0.095313, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.59}, {"index": 2, "tokens": 22626176, "wall_clock_seconds": 14.7, "raw": 0.102742, "normalized": 0.102742, "discount": 1.0, "effective": 0.102742, "running_max": 0.102742, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.4}, {"index": 3, "tokens": 25820976, "wall_clock_seconds": 18.9, "raw": 0.107671, "normalized": 0.107671, "discount": 1.0, "effective": 0.107671, "running_max": 0.107671, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.62} ]}, {"sample_key": "QehvuESr3VNW7R26oQ87MN:weirdml_bonanza", "eval_id": "QehvuESr3VNW7R26oQ87MN", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "efefaf7", "source_log_sha256": "db71bfca9290a72f20068a6a5258b129fc4586e3b69056e874b36b4b2f738504", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.22539, "area_term": 0.153546, "final_best": 0.512765, "best_raw": 0.512765, "best_effective": 0.512765, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 34682467, "usage": {"input": 397353, "output": 165264, "cache_read": 34119850, "cache_write": 32404767}, "cost_usd": 4.828, "cost_source": "list_price", "wall_clock_seconds": 2559.4, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 10635459, "wall_clock_seconds": 22.4, "raw": 0.318696, "normalized": 0.318696, "discount": 1.0, "effective": 0.318696, "running_max": 0.318696, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 21.88}, {"index": 2, "tokens": 14695093, "wall_clock_seconds": 45.8, "raw": 0.459835, "normalized": 0.459835, "discount": 1.0, "effective": 0.459835, "running_max": 0.459835, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 44.83}, {"index": 3, "tokens": 18793636, "wall_clock_seconds": 58.3, "raw": 0.471659, "normalized": 0.471659, "discount": 1.0, "effective": 0.471659, "running_max": 0.471659, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 57.75}, {"index": 4, "tokens": 22214831, "wall_clock_seconds": 56.8, "raw": 0.487747, "normalized": 0.487747, "discount": 1.0, "effective": 0.487747, "running_max": 0.487747, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 56.23}, {"index": 5, "tokens": 25563953, "wall_clock_seconds": 73.5, "raw": 0.509782, "normalized": 0.509782, "discount": 1.0, "effective": 0.509782, "running_max": 0.509782, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 72.45}, {"index": 6, "tokens": 29347148, "wall_clock_seconds": 76.4, "raw": 0.512765, "normalized": 0.512765, "discount": 1.0, "effective": 0.512765, "running_max": 0.512765, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.85} ]}, {"sample_key": "R8ouu9BMkq55zsJuj53PaF:ship_detect", "eval_id": "R8ouu9BMkq55zsJuj53PaF", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "6331f2d", "source_log_sha256": "05af0624248120367f4c58c8aa22c2a690203a6d81a0774251b97b5e593245a9", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.017146, "area_term": 0.014365, "final_best": 0.028267, "best_raw": 0.028267, "best_effective": 0.028267, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 2, "max_submissions": null, "tokens_total": 5593376, "usage": {"input": 111519, "output": 37781, "cache_read": 5444076, "cache_write": 5444076}, "cost_usd": 0.8606, "cost_source": "list_price", "wall_clock_seconds": 2004.0, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 35, "tokens": 974724, "cost_weighted": 46581.8, "wall_clock_seconds": 392.4, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.0, "scored": true, "final": false}, {"index": 2, "model_call": 91, "tokens": 4560717, "cost_weighted": 192595.1, "wall_clock_seconds": 1854.2, "raw": 0.028267, "normalized": 0.028267, "effective": 0.028267, "running_max": 0.028267, "scored": true, "final": false} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:mystery_box", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.001754, "area_term": 0.001051, "final_best": 0.004567, "best_raw": 0.055711, "best_effective": 0.004567, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 33331016, "usage": {"input": 291168, "output": 164695, "cache_read": 32875153, "cache_write": 31754787}, "cost_usd": 4.6258, "cost_source": "list_price", "wall_clock_seconds": 2993.6, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 12571559, "wall_clock_seconds": 3.8, "raw": 0.043344, "normalized": 0.003889, "discount": 0.6, "effective": 0.002333, "running_max": 0.002333, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 21347694, "wall_clock_seconds": 4.0, "raw": 0.053154, "normalized": 0.015296, "discount": 0.25, "effective": 0.003824, "running_max": 0.003824, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 24874323, "wall_clock_seconds": 4.0, "raw": 0.046769, "normalized": 0.007871, "discount": 0.25, "effective": 0.001968, "running_max": 0.003824, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 27117343, "wall_clock_seconds": 7.0, "raw": 0.053842, "normalized": 0.016095, "discount": 0.25, "effective": 0.004024, "running_max": 0.004024, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 28269992, "wall_clock_seconds": 11.9, "raw": 0.053814, "normalized": 0.016063, "discount": 0.25, "effective": 0.004016, "running_max": 0.004024, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 28735084, "wall_clock_seconds": 8.3, "raw": 0.053833, "normalized": 0.016085, "discount": 0.25, "effective": 0.004021, "running_max": 0.004024, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 30876290, "wall_clock_seconds": 5.0, "raw": 0.048631, "normalized": 0.010036, "discount": 0.25, "effective": 0.002509, "running_max": 0.004024, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 31856678, "wall_clock_seconds": 3.8, "raw": 0.047311, "normalized": 0.008501, "discount": 0.25, "effective": 0.002125, "running_max": 0.004024, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 32591228, "wall_clock_seconds": 5.3, "raw": 0.055067, "normalized": 0.01752, "discount": 0.25, "effective": 0.00438, "running_max": 0.00438, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 33331016, "wall_clock_seconds": 4.4, "raw": 0.055711, "normalized": 0.018268, "discount": 0.25, "effective": 0.004567, "running_max": 0.004567, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:mystery_box--nohints", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.016781, "area_term": 0.013818, "final_best": 0.028631, "best_raw": 0.064623, "best_effective": 0.028631, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 25015684, "usage": {"input": 208764, "output": 96117, "cache_read": 24710803, "cache_write": 24433084}, "cost_usd": 3.3892, "cost_source": "list_price", "wall_clock_seconds": 5155.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5389004, "wall_clock_seconds": 3.0, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5712704, "wall_clock_seconds": 2.9, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 6045350, "wall_clock_seconds": 2.8, "raw": 0.038963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 6376485, "wall_clock_seconds": 5.6, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 7405526, "wall_clock_seconds": 2.9, "raw": 0.064623, "normalized": 0.028631, "discount": 1.0, "effective": 0.028631, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8415054, "wall_clock_seconds": 3.0, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 13647112, "wall_clock_seconds": 2.8, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 19273926, "wall_clock_seconds": 2.9, "raw": 0.044143, "normalized": 0.004817, "discount": 1.0, "effective": 0.004817, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 22464592, "wall_clock_seconds": 3.5, "raw": 0.061452, "normalized": 0.024944, "discount": 1.0, "effective": 0.024944, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 25015684, "wall_clock_seconds": 2.9, "raw": 0.049389, "normalized": 0.010918, "discount": 1.0, "effective": 0.010918, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:night_school", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.055294, "area_term": 0.055294, "final_best": 0.055294, "best_raw": 0.217, "best_effective": 0.055294, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 3, "max_submissions": 10, "tokens_total": 13880901, "usage": {"input": 179596, "output": 112955, "cache_read": 13588350, "cache_write": 13527349}, "cost_usd": 2.1415, "cost_source": "list_price", "wall_clock_seconds": 2648.4, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}, {"id": 1, "multiplier": 0.85, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 354097, "wall_clock_seconds": 3.3, "raw": 0.197, "normalized": 0.055294, "discount": 1.0, "effective": 0.055294, "running_max": 0.055294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 5392246, "wall_clock_seconds": 11.7, "raw": 0.217, "normalized": 0.078824, "discount": 0.65, "effective": 0.051235, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 10003109, "wall_clock_seconds": 12.8, "raw": 0.217, "normalized": 0.078824, "discount": 0.65, "effective": 0.051235, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:reaction_rates", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.030965, "area_term": 0.015046, "final_best": 0.094637, "best_raw": 0.168995, "best_effective": 0.094637, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 31534002, "usage": {"input": 284693, "output": 183715, "cache_read": 31065594, "cache_write": 30836660}, "cost_usd": 4.5183, "cost_source": "list_price", "wall_clock_seconds": 5899.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 2}, {"id": 4, "multiplier": 0.8, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2208416, "wall_clock_seconds": 5.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.24}, {"index": 2, "tokens": 7691405, "wall_clock_seconds": 180.7, "raw": 0.0, "normalized": 0.0, "discount": 0.56, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 180.18}, {"index": 3, "tokens": 13900414, "wall_clock_seconds": 51.8, "raw": 0.0, "normalized": 0.0, "discount": 0.56, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 51.23}, {"index": 4, "tokens": 17201158, "wall_clock_seconds": 32.1, "raw": 0.0, "normalized": 0.0, "discount": 0.56, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 31.66}, {"index": 5, "tokens": 18592126, "wall_clock_seconds": 5.4, "raw": 0.0, "normalized": 0.0, "discount": 0.56, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 5.07}, {"index": 6, "tokens": 21274210, "wall_clock_seconds": 7.6, "raw": 0.015589, "normalized": 0.015589, "discount": 0.56, "effective": 0.00873, "running_max": 0.00873, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 7.25}, {"index": 7, "tokens": 24118597, "wall_clock_seconds": 18.3, "raw": 0.161188, "normalized": 0.161188, "discount": 0.56, "effective": 0.090266, "running_max": 0.090266, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 17.85}, {"index": 8, "tokens": 27419486, "wall_clock_seconds": 53.6, "raw": 0.165954, "normalized": 0.165954, "discount": 0.56, "effective": 0.092934, "running_max": 0.092934, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 53.23}, {"index": 9, "tokens": 29421140, "wall_clock_seconds": 53.2, "raw": 0.165954, "normalized": 0.165954, "discount": 0.56, "effective": 0.092934, "running_max": 0.092934, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 52.6}, {"index": 10, "tokens": 31534002, "wall_clock_seconds": 51.9, "raw": 0.168995, "normalized": 0.168995, "discount": 0.56, "effective": 0.094637, "running_max": 0.094637, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 51.35} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:reaction_rates--nohints", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.024038, "area_term": 0.020966, "final_best": 0.036329, "best_raw": 0.036329, "best_effective": 0.036329, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 43946834, "usage": {"input": 335369, "output": 194728, "cache_read": 43416737, "cache_write": 41523808}, "cost_usd": 5.9696, "cost_source": "list_price", "wall_clock_seconds": 3100.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3493340, "wall_clock_seconds": 7.7, "raw": 0.035979, "normalized": 0.035979, "discount": 1.0, "effective": 0.035979, "running_max": 0.035979, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.04}, {"index": 2, "tokens": 5021077, "wall_clock_seconds": 10.1, "raw": 0.036329, "normalized": 0.036329, "discount": 1.0, "effective": 0.036329, "running_max": 0.036329, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.35}, {"index": 3, "tokens": 26599834, "wall_clock_seconds": 16.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.036329, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.84}, {"index": 4, "tokens": 32028209, "wall_clock_seconds": 16.7, "raw": 0.035636, "normalized": 0.035636, "discount": 1.0, "effective": 0.035636, "running_max": 0.036329, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 16.17}, {"index": 5, "tokens": 42196918, "wall_clock_seconds": 6.0, "raw": 0.034357, "normalized": 0.034357, "discount": 1.0, "effective": 0.034357, "running_max": 0.036329, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.23} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:scan_stitch", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.000638, "area_term": 8e-06, "final_best": 0.00316, "best_raw": 0.107222, "best_effective": 0.00316, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 7, "max_submissions": 10, "tokens_total": 50085096, "usage": {"input": 365707, "output": 168985, "cache_read": 49550404, "cache_write": 48679711}, "cost_usd": 6.6542, "cost_source": "list_price", "wall_clock_seconds": 2853.8, "stop_reason": "token_limit", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 14432451, "wall_clock_seconds": 0.3, "raw": 0.083277, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 29687923, "wall_clock_seconds": 0.3, "raw": 0.083999, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 42945013, "wall_clock_seconds": 0.3, "raw": 0.083279, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 44180435, "wall_clock_seconds": 0.3, "raw": 0.091296, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 46310111, "wall_clock_seconds": 0.4, "raw": 0.090508, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 47874605, "wall_clock_seconds": 0.3, "raw": 0.082707, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 49450826, "wall_clock_seconds": 0.4, "raw": 0.107222, "normalized": 0.009028, "discount": 0.35, "effective": 0.00316, "running_max": 0.00316, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:scan_stitch--nohints", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.099305, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 37836054, "usage": {"input": 292556, "output": 155114, "cache_read": 37388384, "cache_write": 35524622}, "cost_usd": 5.0866, "cost_source": "list_price", "wall_clock_seconds": 2804.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 17614236, "wall_clock_seconds": 0.3, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 21948168, "wall_clock_seconds": 0.3, "raw": 0.098335, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 30379366, "wall_clock_seconds": 0.2, "raw": 0.07485, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 31395492, "wall_clock_seconds": 0.3, "raw": 0.098962, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 31803679, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 33472519, "wall_clock_seconds": 0.2, "raw": 0.028947, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 35398381, "wall_clock_seconds": 0.3, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 35832399, "wall_clock_seconds": 0.4, "raw": 0.081362, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 36709728, "wall_clock_seconds": 0.3, "raw": 0.063992, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 37836054, "wall_clock_seconds": 0.3, "raw": 0.099305, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:shapes_generalize", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.035531, "area_term": 0.026758, "final_best": 0.070625, "best_raw": 0.313, "best_effective": 0.070625, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 23342127, "usage": {"input": 202399, "output": 114997, "cache_read": 23024731, "cache_write": 23024731}, "cost_usd": 3.27, "cost_source": "list_price", "wall_clock_seconds": 2302.6, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.5, "before_submission": 8}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5219130, "wall_clock_seconds": 1.1, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.28}, {"index": 2, "tokens": 7536241, "wall_clock_seconds": 0.8, "raw": 0.173, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.25}, {"index": 3, "tokens": 7870935, "wall_clock_seconds": 0.9, "raw": 0.188, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 4, "tokens": 8096562, "wall_clock_seconds": 0.7, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.18}, {"index": 5, "tokens": 8561575, "wall_clock_seconds": 0.7, "raw": 0.206, "normalized": 0.0075, "discount": 1.0, "effective": 0.0075, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 6, "tokens": 10276076, "wall_clock_seconds": 15.7, "raw": 0.176, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.11}, {"index": 7, "tokens": 11066527, "wall_clock_seconds": 1.2, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.56}, {"index": 8, "tokens": 17872125, "wall_clock_seconds": 15.2, "raw": 0.311, "normalized": 0.13875, "discount": 0.5, "effective": 0.069375, "running_max": 0.069375, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 14.65}, {"index": 9, "tokens": 21118515, "wall_clock_seconds": 16.7, "raw": 0.313, "normalized": 0.14125, "discount": 0.5, "effective": 0.070625, "running_max": 0.070625, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 15.91}, {"index": 10, "tokens": 23342127, "wall_clock_seconds": 19.7, "raw": 0.311, "normalized": 0.13875, "discount": 0.5, "effective": 0.069375, "running_max": 0.070625, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 19.0} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:shattered_prior", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.04873, "area_term": 0.04873, "final_best": 0.04873, "best_raw": 0.081131, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 38557108, "usage": {"input": 307230, "output": 171105, "cache_read": 38078773, "cache_write": 37590001}, "cost_usd": 5.2955, "cost_source": "list_price", "wall_clock_seconds": 4355.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.7, "before_submission": 2}, {"id": 3, "multiplier": 0.65, "before_submission": 2}, {"id": 4, "multiplier": 0.55, "before_submission": 2}, {"id": 1, "multiplier": 0.9, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 119057, "wall_clock_seconds": 2.8, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 17979297, "wall_clock_seconds": 1.7, "raw": 0.063822, "normalized": 0.063822, "discount": 0.55, "effective": 0.035102, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 19904018, "wall_clock_seconds": 2.1, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 24890868, "wall_clock_seconds": 1.9, "raw": 0.002081, "normalized": 0.002081, "discount": 0.55, "effective": 0.001145, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 27609156, "wall_clock_seconds": 1.7, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 33089622, "wall_clock_seconds": 2.0, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 36884525, "wall_clock_seconds": 2.3, "raw": 0.071736, "normalized": 0.071736, "discount": 0.55, "effective": 0.039455, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 37436979, "wall_clock_seconds": 2.3, "raw": 0.07755, "normalized": 0.07755, "discount": 0.55, "effective": 0.042653, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 37713826, "wall_clock_seconds": 2.1, "raw": 0.080849, "normalized": 0.080849, "discount": 0.55, "effective": 0.044467, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 38557108, "wall_clock_seconds": 2.1, "raw": 0.081131, "normalized": 0.081131, "discount": 0.55, "effective": 0.044622, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:shattered_prior--nohints", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.028091, "area_term": 0.022196, "final_best": 0.05167, "best_raw": 0.05167, "best_effective": 0.05167, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 25191040, "usage": {"input": 192150, "output": 151853, "cache_read": 24847037, "cache_write": 24847037}, "cost_usd": 3.6132, "cost_source": "list_price", "wall_clock_seconds": 2110.5, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 6582924, "wall_clock_seconds": 1.9, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7711186, "wall_clock_seconds": 1.7, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 14990928, "wall_clock_seconds": 1.7, "raw": 0.006761, "normalized": 0.006761, "discount": 1.0, "effective": 0.006761, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 15422190, "wall_clock_seconds": 1.8, "raw": 0.050588, "normalized": 0.050588, "discount": 1.0, "effective": 0.050588, "running_max": 0.050588, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 15712080, "wall_clock_seconds": 2.2, "raw": 0.040511, "normalized": 0.040511, "discount": 1.0, "effective": 0.040511, "running_max": 0.050588, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 16003698, "wall_clock_seconds": 1.8, "raw": 0.051533, "normalized": 0.051533, "discount": 1.0, "effective": 0.051533, "running_max": 0.051533, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 16303550, "wall_clock_seconds": 1.9, "raw": 0.05167, "normalized": 0.05167, "discount": 1.0, "effective": 0.05167, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 16598687, "wall_clock_seconds": 2.3, "raw": 0.051587, "normalized": 0.051587, "discount": 1.0, "effective": 0.051587, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 16895726, "wall_clock_seconds": 1.7, "raw": 0.050409, "normalized": 0.050409, "discount": 1.0, "effective": 0.050409, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:ship_detect", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.028283, "area_term": 0.021299, "final_best": 0.05622, "best_raw": 0.05622, "best_effective": 0.05622, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": null, "tokens_total": 12235323, "usage": {"input": 173606, "output": 61008, "cache_read": 12000709, "cache_write": 11719975}, "cost_usd": 1.7478, "cost_source": "list_price", "wall_clock_seconds": 1541.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 25, "tokens": 698953, "cost_weighted": 36507.8, "wall_clock_seconds": 298.7, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.0, "scored": true, "final": false}, {"index": 2, "model_call": 132, "tokens": 8953796, "cost_weighted": 349404.6, "wall_clock_seconds": 1145.0, "raw": 0.05622, "normalized": 0.05622, "effective": 0.05622, "running_max": 0.05622, "scored": true, "final": false}, {"index": 3, "model_call": 142, "tokens": 10199970, "cost_weighted": 396326.7, "wall_clock_seconds": 1333.7, "raw": 0.04853, "normalized": 0.04853, "effective": 0.04853, "running_max": 0.05622, "scored": true, "final": false}, {"index": 4, "model_call": 149, "tokens": 11210267, "cost_weighted": 430428.0, "wall_clock_seconds": 1429.4, "raw": 0.04853, "normalized": 0.04853, "effective": 0.04853, "running_max": 0.05622, "scored": true, "final": false}, {"index": 5, "model_call": 154, "tokens": 11939573, "cost_weighted": 456333.4, "wall_clock_seconds": 1496.3, "raw": 0.04853, "normalized": 0.04853, "effective": 0.04853, "running_max": 0.05622, "scored": true, "final": false} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:ship_tune", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.040711, "area_term": 0.035862, "final_best": 0.060104, "best_raw": 0.060104, "best_effective": 0.060104, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10656251, "usage": {"input": 309665, "output": 88224, "cache_read": 10258362, "cache_write": 9396445}, "cost_usd": 1.7243, "cost_source": "list_price", "wall_clock_seconds": 3155.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2620884, "wall_clock_seconds": 122.8, "raw": 0.042658, "normalized": 0.042658, "discount": 1.0, "effective": 0.042658, "running_max": 0.042658, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 122.17}, {"index": 2, "tokens": 4041036, "wall_clock_seconds": 150.7, "raw": 0.054936, "normalized": 0.054936, "discount": 1.0, "effective": 0.054936, "running_max": 0.054936, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 149.74}, {"index": 3, "tokens": 5697564, "wall_clock_seconds": 146.4, "raw": 0.052168, "normalized": 0.052168, "discount": 1.0, "effective": 0.052168, "running_max": 0.054936, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 145.81}, {"index": 4, "tokens": 6217034, "wall_clock_seconds": 153.6, "raw": 0.054558, "normalized": 0.054558, "discount": 1.0, "effective": 0.054558, "running_max": 0.054936, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 152.82}, {"index": 5, "tokens": 7233739, "wall_clock_seconds": 154.4, "raw": 0.054746, "normalized": 0.054746, "discount": 1.0, "effective": 0.054746, "running_max": 0.054936, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 153.63}, {"index": 6, "tokens": 7960364, "wall_clock_seconds": 158.1, "raw": 0.053144, "normalized": 0.053144, "discount": 1.0, "effective": 0.053144, "running_max": 0.054936, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 157.43}, {"index": 7, "tokens": 9114977, "wall_clock_seconds": 176.4, "raw": 0.04983, "normalized": 0.04983, "discount": 1.0, "effective": 0.04983, "running_max": 0.054936, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 175.84}, {"index": 8, "tokens": 9519111, "wall_clock_seconds": 145.8, "raw": 0.059019, "normalized": 0.059019, "discount": 1.0, "effective": 0.059019, "running_max": 0.059019, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 145.24}, {"index": 9, "tokens": 10209297, "wall_clock_seconds": 160.7, "raw": 0.059905, "normalized": 0.059905, "discount": 1.0, "effective": 0.059905, "running_max": 0.059905, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 160.0}, {"index": 10, "tokens": 10656251, "wall_clock_seconds": 144.0, "raw": 0.060104, "normalized": 0.060104, "discount": 1.0, "effective": 0.060104, "running_max": 0.060104, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 143.39} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:splash_generalize", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.289985, "area_term": 0.223037, "final_best": 0.557778, "best_raw": 0.602, "best_effective": 0.557778, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 16556092, "usage": {"input": 388171, "output": 229735, "cache_read": 15938186, "cache_write": 14517928}, "cost_usd": 2.9534, "cost_source": "list_price", "wall_clock_seconds": 2577.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1367649, "wall_clock_seconds": 1.9, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.27}, {"index": 2, "tokens": 7530851, "wall_clock_seconds": 32.9, "raw": 0.49, "normalized": 0.433333, "discount": 1.0, "effective": 0.433333, "running_max": 0.433333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 32.32}, {"index": 3, "tokens": 8475803, "wall_clock_seconds": 2.8, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.433333, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.16}, {"index": 4, "tokens": 9488724, "wall_clock_seconds": 35.5, "raw": 0.602, "normalized": 0.557778, "discount": 1.0, "effective": 0.557778, "running_max": 0.557778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 34.78}, {"index": 5, "tokens": 10903953, "wall_clock_seconds": 1.5, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.557778, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 0.81}, {"index": 6, "tokens": 11384821, "wall_clock_seconds": 2.2, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.557778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.69}, {"index": 7, "tokens": 12210004, "wall_clock_seconds": 1.5, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.557778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.84}, {"index": 8, "tokens": 13584586, "wall_clock_seconds": 31.7, "raw": 0.57, "normalized": 0.522222, "discount": 1.0, "effective": 0.522222, "running_max": 0.557778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 31.18}, {"index": 9, "tokens": 14304748, "wall_clock_seconds": 1.1, "raw": 0.094, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.557778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.57}, {"index": 10, "tokens": 16556092, "wall_clock_seconds": 30.7, "raw": 0.567, "normalized": 0.518889, "discount": 1.0, "effective": 0.518889, "running_max": 0.557778, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.09} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:tod_pipeline--nohints", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.0, "best_effective": 0.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 36020377, "usage": {"input": 305146, "output": 137157, "cache_read": 35578074, "cache_write": 33450519}, "cost_usd": 4.8064, "cost_source": "list_price", "wall_clock_seconds": 2169.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 19041936, "wall_clock_seconds": 10.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 10.05}, {"index": 2, "tokens": 28423409, "wall_clock_seconds": 10.3, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 9.94}, {"index": 3, "tokens": 32367617, "wall_clock_seconds": 12.1, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 11.78}, {"index": 4, "tokens": 34714672, "wall_clock_seconds": 8.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.5} ]}, {"sample_key": "hzRfxsNadgRZ2Ysp3apAUC:weirdml_bonanza", "eval_id": "hzRfxsNadgRZ2Ysp3apAUC", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "efefaf7", "source_log_sha256": "58206ece6667bf2eb1f5d353d8c2cebf21f0285cc5314d40ff1b890e2e316292", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.24526, "area_term": 0.187059, "final_best": 0.478063, "best_raw": 0.478063, "best_effective": 0.478063, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 31892996, "usage": {"input": 362196, "output": 173737, "cache_read": 31357063, "cache_write": 31084932}, "cost_usd": 4.5712, "cost_source": "list_price", "wall_clock_seconds": 3323.7, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3444276, "wall_clock_seconds": 4.7, "raw": 0.16162, "normalized": 0.16162, "discount": 1.0, "effective": 0.16162, "running_max": 0.16162, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.14}, {"index": 2, "tokens": 11381109, "wall_clock_seconds": 56.6, "raw": 0.405222, "normalized": 0.405222, "discount": 1.0, "effective": 0.405222, "running_max": 0.405222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 56.0}, {"index": 3, "tokens": 13993263, "wall_clock_seconds": 67.4, "raw": 0.405352, "normalized": 0.405352, "discount": 1.0, "effective": 0.405352, "running_max": 0.405352, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 66.78}, {"index": 4, "tokens": 17632083, "wall_clock_seconds": 71.7, "raw": 0.435157, "normalized": 0.435157, "discount": 1.0, "effective": 0.435157, "running_max": 0.435157, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 71.02}, {"index": 5, "tokens": 20972902, "wall_clock_seconds": 75.7, "raw": 0.478063, "normalized": 0.478063, "discount": 1.0, "effective": 0.478063, "running_max": 0.478063, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.03}, {"index": 6, "tokens": 24204256, "wall_clock_seconds": 82.2, "raw": 0.477811, "normalized": 0.477811, "discount": 1.0, "effective": 0.477811, "running_max": 0.478063, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 81.71}, {"index": 7, "tokens": 27054296, "wall_clock_seconds": 76.8, "raw": 0.47781, "normalized": 0.47781, "discount": 1.0, "effective": 0.47781, "running_max": 0.478063, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 76.16}, {"index": 8, "tokens": 31033488, "wall_clock_seconds": 76.4, "raw": 0.473227, "normalized": 0.473227, "discount": 1.0, "effective": 0.473227, "running_max": 0.478063, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 75.85} ]}, {"sample_key": "juTEmAMMBJW8eaweCcZ9kC:scan_stitch", "eval_id": "juTEmAMMBJW8eaweCcZ9kC", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "5bd1a28", "source_log_sha256": "ad10fb3910a98ccae967b0aae4c7b83429a751fe4a180aed79f4372a60de6667", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.345868, "area_term": 0.312378, "final_best": 0.479829, "best_raw": 0.852674, "best_effective": 0.479829, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7986140, "usage": {"input": 393, "output": 89836, "cache_read": 7627322, "cache_write": 268589}, "cost_usd": 15.4804, "cost_source": "billed", "wall_clock_seconds": 3885.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 687591, "wall_clock_seconds": 0.3, "raw": 0.095401, "normalized": 0.0, "discount": 0.6375, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1191240, "wall_clock_seconds": 0.3, "raw": 0.102413, "normalized": 0.003016, "discount": 0.6375, "effective": 0.001923, "running_max": 0.001923, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2099843, "wall_clock_seconds": 0.3, "raw": 0.690318, "normalized": 0.737897, "discount": 0.51, "effective": 0.376328, "running_max": 0.376328, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2818171, "wall_clock_seconds": 0.3, "raw": 0.517751, "normalized": 0.522189, "discount": 0.51, "effective": 0.266317, "running_max": 0.376328, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3274339, "wall_clock_seconds": 0.3, "raw": 0.738012, "normalized": 0.797515, "discount": 0.51, "effective": 0.406733, "running_max": 0.406733, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4591256, "wall_clock_seconds": 0.3, "raw": 0.705275, "normalized": 0.756593, "discount": 0.51, "effective": 0.385863, "running_max": 0.406733, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5328525, "wall_clock_seconds": 0.3, "raw": 0.840382, "normalized": 0.925478, "discount": 0.51, "effective": 0.471994, "running_max": 0.471994, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5793843, "wall_clock_seconds": 0.3, "raw": 0.844901, "normalized": 0.931126, "discount": 0.51, "effective": 0.474874, "running_max": 0.474874, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6435296, "wall_clock_seconds": 0.3, "raw": 0.845982, "normalized": 0.932477, "discount": 0.51, "effective": 0.475563, "running_max": 0.475563, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7986140, "wall_clock_seconds": 0.4, "raw": 0.852674, "normalized": 0.940842, "discount": 0.51, "effective": 0.479829, "running_max": 0.479829, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "juTEmAMMBJW8eaweCcZ9kC:ship_detect", "eval_id": "juTEmAMMBJW8eaweCcZ9kC", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-16", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "5bd1a28", "source_log_sha256": "ad10fb3910a98ccae967b0aae4c7b83429a751fe4a180aed79f4372a60de6667", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.158138, "area_term": 0.131237, "final_best": 0.265738, "best_raw": 0.265738, "best_effective": 0.265738, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 54, "max_submissions": null, "tokens_total": 46867769, "usage": {"input": 1110, "output": 479721, "cache_read": 43807833, "cache_write": 2579105}, "cost_usd": 100.0438, "cost_source": "billed", "wall_clock_seconds": 40098.4, "stop_reason": "cost_limit", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 26, "tokens": 792106, "cost_weighted": 39770.2, "wall_clock_seconds": 616.5, "raw": 0.057341, "normalized": 0.057341, "effective": 0.057341, "running_max": 0.057341, "scored": true, "final": false}, {"index": 2, "model_call": 50, "tokens": 2722346, "cost_weighted": 101719.3, "wall_clock_seconds": 1120.4, "raw": 0.075293, "normalized": 0.075293, "effective": 0.075293, "running_max": 0.075293, "scored": true, "final": false}, {"index": 3, "model_call": 53, "tokens": 3048286, "cost_weighted": 109287.4, "wall_clock_seconds": 2226.3, "raw": 0.121209, "normalized": 0.121209, "effective": 0.121209, "running_max": 0.121209, "scored": true, "final": false}, {"index": 4, "model_call": 68, "tokens": 5210013, "cost_weighted": 189164.1, "wall_clock_seconds": 4340.3, "raw": 0.141099, "normalized": 0.141099, "effective": 0.141099, "running_max": 0.141099, "scored": true, "final": false}, {"index": 5, "model_call": 72, "tokens": 5923107, "cost_weighted": 212062.3, "wall_clock_seconds": 4612.7, "raw": 0.148312, "normalized": 0.148312, "effective": 0.148312, "running_max": 0.148312, "scored": true, "final": false}, {"index": 6, "model_call": 84, "tokens": 8006297, "cost_weighted": 305915.4, "wall_clock_seconds": 5261.4, "raw": 0.185635, "normalized": 0.185635, "effective": 0.185635, "running_max": 0.185635, "scored": true, "final": false}, {"index": 7, "model_call": 105, "tokens": 9758095, "cost_weighted": 425298.0, "wall_clock_seconds": 5968.2, "raw": 0.15974, "normalized": 0.15974, "effective": 0.15974, "running_max": 0.185635, "scored": true, "final": false}, {"index": 8, "model_call": 130, "tokens": 12603653, "cost_weighted": 518646.9, "wall_clock_seconds": 7412.0, "raw": 0.191485, "normalized": 0.191485, "effective": 0.191485, "running_max": 0.191485, "scored": true, "final": false}, {"index": 9, "model_call": 134, "tokens": 13232375, "cost_weighted": 540446.8, "wall_clock_seconds": 8001.1, "raw": 0.187084, "normalized": 0.187084, "effective": 0.187084, "running_max": 0.191485, "scored": true, "final": false}, {"index": 10, "model_call": 137, "tokens": 13733166, "cost_weighted": 559157.8, "wall_clock_seconds": 8502.0, "raw": 0.193056, "normalized": 0.193056, "effective": 0.193056, "running_max": 0.193056, "scored": true, "final": false}, {"index": 11, "model_call": 141, "tokens": 14417501, "cost_weighted": 578503.3, "wall_clock_seconds": 8880.1, "raw": 0.21306, "normalized": 0.21306, "effective": 0.21306, "running_max": 0.21306, "scored": true, "final": false}, {"index": 12, "model_call": 158, "tokens": 17950967, "cost_weighted": 756862.5, "wall_clock_seconds": 10097.6, "raw": 0.217228, "normalized": 0.217228, "effective": 0.217228, "running_max": 0.217228, "scored": true, "final": false}, {"index": 13, "model_call": 168, "tokens": 18260166, "cost_weighted": 774350.5, "wall_clock_seconds": 10225.4, "raw": 0.217592, "normalized": 0.217592, "effective": 0.217592, "running_max": 0.217592, "scored": true, "final": false}, {"index": 14, "model_call": 182, "tokens": 19224817, "cost_weighted": 809733.8, "wall_clock_seconds": 10561.2, "raw": 0.217839, "normalized": 0.217839, "effective": 0.217839, "running_max": 0.217839, "scored": true, "final": false}, {"index": 15, "model_call": 185, "tokens": 19496569, "cost_weighted": 820182.6, "wall_clock_seconds": 10657.4, "raw": 0.217872, "normalized": 0.217872, "effective": 0.217872, "running_max": 0.217872, "scored": true, "final": false}, {"index": 16, "model_call": 188, "tokens": 19796058, "cost_weighted": 830274.8, "wall_clock_seconds": 10767.5, "raw": 0.21797, "normalized": 0.21797, "effective": 0.21797, "running_max": 0.21797, "scored": true, "final": false}, {"index": 17, "model_call": 191, "tokens": 20118603, "cost_weighted": 840800.4, "wall_clock_seconds": 10856.4, "raw": 0.217956, "normalized": 0.217956, "effective": 0.217956, "running_max": 0.21797, "scored": true, "final": false}, {"index": 18, "model_call": 193, "tokens": 20357025, "cost_weighted": 850696.9, "wall_clock_seconds": 10957.9, "raw": 0.218205, "normalized": 0.218205, "effective": 0.218205, "running_max": 0.218205, "scored": true, "final": false}, {"index": 19, "model_call": 196, "tokens": 20738738, "cost_weighted": 862549.5, "wall_clock_seconds": 11069.1, "raw": 0.196375, "normalized": 0.196375, "effective": 0.196375, "running_max": 0.218205, "scored": true, "final": false}, {"index": 20, "model_call": 200, "tokens": 21290141, "cost_weighted": 882418.0, "wall_clock_seconds": 11255.0, "raw": 0.191163, "normalized": 0.191163, "effective": 0.191163, "running_max": 0.218205, "scored": true, "final": false}, {"index": 21, "model_call": 202, "tokens": 21580161, "cost_weighted": 892654.8, "wall_clock_seconds": 11376.9, "raw": 0.206475, "normalized": 0.206475, "effective": 0.206475, "running_max": 0.218205, "scored": true, "final": false}, {"index": 22, "model_call": 208, "tokens": 22510809, "cost_weighted": 924693.0, "wall_clock_seconds": 11841.9, "raw": 0.193808, "normalized": 0.193808, "effective": 0.193808, "running_max": 0.218205, "scored": true, "final": false}, {"index": 23, "model_call": 209, "tokens": 22672636, "cost_weighted": 930159.2, "wall_clock_seconds": 11917.4, "raw": 0.216781, "normalized": 0.216781, "effective": 0.216781, "running_max": 0.218205, "scored": true, "final": false}, {"index": 24, "model_call": 217, "tokens": 24066900, "cost_weighted": 972164.3, "wall_clock_seconds": 12381.8, "raw": 0.214836, "normalized": 0.214836, "effective": 0.214836, "running_max": 0.218205, "scored": true, "final": false}, {"index": 25, "model_call": 221, "tokens": 24815108, "cost_weighted": 992176.5, "wall_clock_seconds": 12678.8, "raw": 0.219958, "normalized": 0.219958, "effective": 0.219958, "running_max": 0.219958, "scored": true, "final": false}, {"index": 26, "model_call": 224, "tokens": 25401643, "cost_weighted": 1010551.8, "wall_clock_seconds": 12985.0, "raw": 0.22189, "normalized": 0.22189, "effective": 0.22189, "running_max": 0.22189, "scored": true, "final": false}, {"index": 27, "model_call": 225, "tokens": 25602180, "cost_weighted": 1018554.2, "wall_clock_seconds": 13085.0, "raw": 0.234025, "normalized": 0.234025, "effective": 0.234025, "running_max": 0.234025, "scored": true, "final": false}, {"index": 28, "model_call": 226, "tokens": 25803193, "cost_weighted": 1023737.7, "wall_clock_seconds": 13114.5, "raw": 0.206047, "normalized": 0.206047, "effective": 0.206047, "running_max": 0.234025, "scored": true, "final": false}, {"index": 29, "model_call": 229, "tokens": 26430890, "cost_weighted": 1047991.1, "wall_clock_seconds": 13422.2, "raw": 0.206418, "normalized": 0.206418, "effective": 0.206418, "running_max": 0.234025, "scored": true, "final": false}, {"index": 30, "model_call": 238, "tokens": 28431875, "cost_weighted": 1105786.6, "wall_clock_seconds": 13901.4, "raw": 0.232867, "normalized": 0.232867, "effective": 0.232867, "running_max": 0.234025, "scored": true, "final": false}, {"index": 31, "model_call": 247, "tokens": 29543108, "cost_weighted": 1207485.2, "wall_clock_seconds": 14308.4, "raw": 0.237534, "normalized": 0.237534, "effective": 0.237534, "running_max": 0.237534, "scored": true, "final": false}, {"index": 32, "model_call": 266, "tokens": 30992733, "cost_weighted": 1273450.6, "wall_clock_seconds": 14934.7, "raw": 0.2376, "normalized": 0.2376, "effective": 0.2376, "running_max": 0.2376, "scored": true, "final": false}, {"index": 33, "model_call": 267, "tokens": 31106946, "cost_weighted": 1277518.6, "wall_clock_seconds": 15001.2, "raw": 0.241072, "normalized": 0.241072, "effective": 0.241072, "running_max": 0.241072, "scored": true, "final": false}, {"index": 34, "model_call": 277, "tokens": 32469764, "cost_weighted": 1336766.2, "wall_clock_seconds": 15774.4, "raw": 0.241276, "normalized": 0.241276, "effective": 0.241276, "running_max": 0.241276, "scored": true, "final": false}, {"index": 35, "model_call": 278, "tokens": 32625601, "cost_weighted": 1342954.4, "wall_clock_seconds": 15824.8, "raw": 0.240754, "normalized": 0.240754, "effective": 0.240754, "running_max": 0.241276, "scored": true, "final": false}, {"index": 36, "model_call": 279, "tokens": 32781645, "cost_weighted": 1346579.8, "wall_clock_seconds": 16426.0, "raw": 0.242394, "normalized": 0.242394, "effective": 0.242394, "running_max": 0.242394, "scored": true, "final": false}, {"index": 37, "model_call": 281, "tokens": 33118020, "cost_weighted": 1368967.8, "wall_clock_seconds": 16857.5, "raw": 0.241835, "normalized": 0.241835, "effective": 0.241835, "running_max": 0.242394, "scored": true, "final": false}, {"index": 38, "model_call": 289, "tokens": 34625960, "cost_weighted": 1419489.4, "wall_clock_seconds": 17391.3, "raw": 0.242085, "normalized": 0.242085, "effective": 0.242085, "running_max": 0.242394, "scored": true, "final": false}, {"index": 39, "model_call": 291, "tokens": 35047723, "cost_weighted": 1436989.6, "wall_clock_seconds": 17655.1, "raw": 0.242257, "normalized": 0.242257, "effective": 0.242257, "running_max": 0.242394, "scored": true, "final": false}, {"index": 40, "model_call": 293, "tokens": 35496555, "cost_weighted": 1454444.0, "wall_clock_seconds": 17890.4, "raw": 0.242339, "normalized": 0.242339, "effective": 0.242339, "running_max": 0.242394, "scored": true, "final": false}, {"index": 41, "model_call": 296, "tokens": 36203419, "cost_weighted": 1479092.9, "wall_clock_seconds": 18154.5, "raw": 0.242637, "normalized": 0.242637, "effective": 0.242637, "running_max": 0.242637, "scored": true, "final": false}, {"index": 42, "model_call": 304, "tokens": 36880953, "cost_weighted": 1574741.1, "wall_clock_seconds": 18648.1, "raw": 0.24284, "normalized": 0.24284, "effective": 0.24284, "running_max": 0.24284, "scored": true, "final": false}, {"index": 43, "model_call": 308, "tokens": 37111657, "cost_weighted": 1589961.6, "wall_clock_seconds": 18817.0, "raw": 0.242849, "normalized": 0.242849, "effective": 0.242849, "running_max": 0.242849, "scored": true, "final": false}, {"index": 44, "model_call": 309, "tokens": 37178715, "cost_weighted": 1592465.2, "wall_clock_seconds": 20315.2, "raw": 0.265738, "normalized": 0.265738, "effective": 0.265738, "running_max": 0.265738, "scored": true, "final": false}, {"index": 45, "model_call": 312, "tokens": 37391817, "cost_weighted": 1599986.6, "wall_clock_seconds": 22180.6, "raw": 0.262925, "normalized": 0.262925, "effective": 0.262925, "running_max": 0.265738, "scored": true, "final": false}, {"index": 46, "model_call": 316, "tokens": 37719438, "cost_weighted": 1612954.0, "wall_clock_seconds": 24088.2, "raw": 0.255059, "normalized": 0.255059, "effective": 0.255059, "running_max": 0.265738, "scored": true, "final": false}, {"index": 47, "model_call": 324, "tokens": 38517108, "cost_weighted": 1641491.0, "wall_clock_seconds": 26177.9, "raw": 0.255284, "normalized": 0.255284, "effective": 0.255284, "running_max": 0.265738, "scored": true, "final": false}, {"index": 48, "model_call": 326, "tokens": 38749366, "cost_weighted": 1676596.1, "wall_clock_seconds": 28077.0, "raw": 0.260176, "normalized": 0.260176, "effective": 0.260176, "running_max": 0.265738, "scored": true, "final": false}, {"index": 49, "model_call": 329, "tokens": 39116074, "cost_weighted": 1690309.1, "wall_clock_seconds": 29991.3, "raw": 0.260068, "normalized": 0.260068, "effective": 0.260068, "running_max": 0.265738, "scored": true, "final": false}, {"index": 50, "model_call": 346, "tokens": 41707126, "cost_weighted": 1774013.3, "wall_clock_seconds": 32793.2, "raw": 0.260117, "normalized": 0.260117, "effective": 0.260117, "running_max": 0.265738, "scored": true, "final": false}, {"index": 51, "model_call": 348, "tokens": 42056778, "cost_weighted": 1782909.5, "wall_clock_seconds": 33736.8, "raw": 0.260693, "normalized": 0.260693, "effective": 0.260693, "running_max": 0.265738, "scored": true, "final": false}, {"index": 52, "model_call": 351, "tokens": 42596832, "cost_weighted": 1800677.4, "wall_clock_seconds": 33925.5, "raw": 0.257441, "normalized": 0.257441, "effective": 0.257441, "running_max": 0.265738, "scored": true, "final": false}, {"index": 53, "model_call": 352, "tokens": 42779804, "cost_weighted": 1804875.1, "wall_clock_seconds": 35356.7, "raw": 0.259386, "normalized": 0.259386, "effective": 0.259386, "running_max": 0.265738, "scored": true, "final": false}, {"index": 54, "model_call": 355, "tokens": 43348831, "cost_weighted": 1825082.5, "wall_clock_seconds": 37276.4, "raw": 0.260823, "normalized": 0.260823, "effective": 0.260823, "running_max": 0.265738, "scored": true, "final": false}, {"index": 55, "model_call": 361, "tokens": 44556391, "cost_weighted": 1865820.1, "wall_clock_seconds": 38843.9, "raw": null, "normalized": null, "effective": null, "running_max": null, "scored": false, "final": false}, {"index": 56, "model_call": 363, "tokens": 44988657, "cost_weighted": 1878582.6, "wall_clock_seconds": 38946.8, "raw": null, "normalized": null, "effective": null, "running_max": null, "scored": false, "final": false}, {"index": 57, "model_call": 364, "tokens": 45209739, "cost_weighted": 1885070.6, "wall_clock_seconds": 39011.5, "raw": null, "normalized": null, "effective": null, "running_max": null, "scored": false, "final": false} ]}, {"sample_key": "2BNUz7C8FvV2fKP7F49w7G:night_school", "eval_id": "2BNUz7C8FvV2fKP7F49w7G", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "fable-5.1", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "d9d7f13", "source_log_sha256": "2090bac6687b41d4b919e695d3d703bec874fabca1b73a9e63229041befcf295", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.343953, "area_term": 0.294301, "final_best": 0.542559, "best_raw": 0.8595, "best_effective": 0.542559, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 28972660, "usage": {"input": 31765, "output": 257354, "cache_read": 28294363, "cache_write": 389178}, "cost_usd": 28.0425, "cost_source": "list_price", "wall_clock_seconds": 21543.6, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2551237, "wall_clock_seconds": 11.1, "raw": 0.499, "normalized": 0.410588, "discount": 0.65, "effective": 0.266882, "running_max": 0.266882, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 4814056, "wall_clock_seconds": 11.9, "raw": 0.6915, "normalized": 0.637059, "discount": 0.65, "effective": 0.414088, "running_max": 0.414088, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8258105, "wall_clock_seconds": 13.9, "raw": 0.8195, "normalized": 0.787647, "discount": 0.65, "effective": 0.511971, "running_max": 0.511971, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 10903357, "wall_clock_seconds": 15.2, "raw": 0.844, "normalized": 0.816471, "discount": 0.65, "effective": 0.530706, "running_max": 0.530706, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 17822669, "wall_clock_seconds": 15.4, "raw": 0.853, "normalized": 0.827059, "discount": 0.65, "effective": 0.537588, "running_max": 0.537588, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 19116887, "wall_clock_seconds": 12.9, "raw": 0.8595, "normalized": 0.834706, "discount": 0.65, "effective": 0.542559, "running_max": 0.542559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 23817041, "wall_clock_seconds": 13.2, "raw": 0.8465, "normalized": 0.819412, "discount": 0.65, "effective": 0.532618, "running_max": 0.542559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 24859952, "wall_clock_seconds": 13.4, "raw": 0.858, "normalized": 0.832941, "discount": 0.65, "effective": 0.541412, "running_max": 0.542559, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "2BNUz7C8FvV2fKP7F49w7G:reaction_rates--nohints", "eval_id": "2BNUz7C8FvV2fKP7F49w7G", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "fable-5.1", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "d9d7f13", "source_log_sha256": "2090bac6687b41d4b919e695d3d703bec874fabca1b73a9e63229041befcf295", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.13983, "area_term": 0.071161, "final_best": 0.414502, "best_raw": 0.414502, "best_effective": 0.414502, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 37987042, "usage": {"input": 19391, "output": 267164, "cache_read": 37241170, "cache_write": 459317}, "cost_usd": 32.0487, "cost_source": "list_price", "wall_clock_seconds": 40897.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 13798061, "wall_clock_seconds": 47.2, "raw": 0.098414, "normalized": 0.098414, "discount": 1.0, "effective": 0.098414, "running_max": 0.098414, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 46.71}, {"index": 2, "tokens": 16681852, "wall_clock_seconds": 174.3, "raw": 0.141243, "normalized": 0.141243, "discount": 1.0, "effective": 0.141243, "running_max": 0.141243, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 173.79}, {"index": 3, "tokens": 17717559, "wall_clock_seconds": 161.2, "raw": 0.146628, "normalized": 0.146628, "discount": 1.0, "effective": 0.146628, "running_max": 0.146628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 160.72}, {"index": 4, "tokens": 20635611, "wall_clock_seconds": 517.1, "raw": 0.215473, "normalized": 0.215473, "discount": 1.0, "effective": 0.215473, "running_max": 0.215473, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 516.75}, {"index": 5, "tokens": 27224321, "wall_clock_seconds": 196.1, "raw": 0.274618, "normalized": 0.274618, "discount": 1.0, "effective": 0.274618, "running_max": 0.274618, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 195.61}, {"index": 6, "tokens": 31800456, "wall_clock_seconds": 458.0, "raw": 0.322064, "normalized": 0.322064, "discount": 1.0, "effective": 0.322064, "running_max": 0.322064, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 457.38}, {"index": 7, "tokens": 33530631, "wall_clock_seconds": 457.8, "raw": 0.337943, "normalized": 0.337943, "discount": 1.0, "effective": 0.337943, "running_max": 0.337943, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 457.3}, {"index": 8, "tokens": 34846257, "wall_clock_seconds": 503.3, "raw": 0.346021, "normalized": 0.346021, "discount": 1.0, "effective": 0.346021, "running_max": 0.346021, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 502.57}, {"index": 9, "tokens": 36628958, "wall_clock_seconds": 448.5, "raw": 0.400113, "normalized": 0.400113, "discount": 1.0, "effective": 0.400113, "running_max": 0.400113, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 448.09}, {"index": 10, "tokens": 37987042, "wall_clock_seconds": 474.7, "raw": 0.414502, "normalized": 0.414502, "discount": 1.0, "effective": 0.414502, "running_max": 0.414502, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 472.83} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:mystery_box", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.007802, "area_term": 0.007802, "final_best": 0.007802, "best_raw": 0.055836, "best_effective": 0.007802, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3130596, "usage": {"input": 24181, "output": 64376, "cache_read": 2904474, "cache_write": 137565}, "cost_usd": 4.5582, "cost_source": "list_price", "wall_clock_seconds": 1730.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 4}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 198818, "wall_clock_seconds": 2.9, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 277910, "wall_clock_seconds": 3.0, "raw": 0.038724, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 403108, "wall_clock_seconds": 2.8, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 970926, "wall_clock_seconds": 3.8, "raw": 0.045905, "normalized": 0.006866, "discount": 0.25, "effective": 0.001717, "running_max": 0.007802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1158814, "wall_clock_seconds": 3.8, "raw": 0.054029, "normalized": 0.016312, "discount": 0.25, "effective": 0.004078, "running_max": 0.007802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1776547, "wall_clock_seconds": 3.9, "raw": 0.055836, "normalized": 0.018413, "discount": 0.25, "effective": 0.004603, "running_max": 0.007802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 2245467, "wall_clock_seconds": 4.2, "raw": 0.048437, "normalized": 0.00981, "discount": 0.25, "effective": 0.002453, "running_max": 0.007802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2451005, "wall_clock_seconds": 3.3, "raw": 0.048944, "normalized": 0.0104, "discount": 0.25, "effective": 0.0026, "running_max": 0.007802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2895273, "wall_clock_seconds": 3.8, "raw": 0.055836, "normalized": 0.018413, "discount": 0.25, "effective": 0.004603, "running_max": 0.007802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 3130596, "wall_clock_seconds": 3.5, "raw": 0.047981, "normalized": 0.00928, "discount": 0.25, "effective": 0.00232, "running_max": 0.007802, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:mystery_box--nohints", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.030976, "area_term": 0.030973, "final_best": 0.030989, "best_raw": 0.06665, "best_effective": 0.030989, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 1708112, "usage": {"input": 16462, "output": 24564, "cache_read": 1597988, "cache_write": 69098}, "cost_usd": 2.1864, "cost_source": "list_price", "wall_clock_seconds": 616.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 221308, "wall_clock_seconds": 2.5, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 335166, "wall_clock_seconds": 2.4, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 501572, "wall_clock_seconds": 2.4, "raw": 0.06665, "normalized": 0.030989, "discount": 1.0, "effective": 0.030989, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 687628, "wall_clock_seconds": 2.8, "raw": 0.062402, "normalized": 0.026049, "discount": 1.0, "effective": 0.026049, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 895117, "wall_clock_seconds": 2.5, "raw": 0.046307, "normalized": 0.007333, "discount": 1.0, "effective": 0.007333, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1120538, "wall_clock_seconds": 2.6, "raw": 0.06111, "normalized": 0.024546, "discount": 1.0, "effective": 0.024546, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1303345, "wall_clock_seconds": 2.7, "raw": 0.059404, "normalized": 0.022563, "discount": 1.0, "effective": 0.022563, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 1432916, "wall_clock_seconds": 2.9, "raw": 0.051793, "normalized": 0.013713, "discount": 1.0, "effective": 0.013713, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 1638232, "wall_clock_seconds": 2.7, "raw": 0.062378, "normalized": 0.026021, "discount": 1.0, "effective": 0.026021, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 1708112, "wall_clock_seconds": 2.4, "raw": 0.046607, "normalized": 0.007682, "discount": 1.0, "effective": 0.007682, "running_max": 0.030989, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:night_school", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.055294, "area_term": 0.055294, "final_best": 0.055294, "best_raw": 0.216, "best_effective": 0.055294, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3271365, "usage": {"input": 22510, "output": 47159, "cache_read": 3073645, "cache_write": 128051}, "cost_usd": 4.1089, "cost_source": "list_price", "wall_clock_seconds": 1288.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}, {"id": 3, "multiplier": 0.7, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 421812, "wall_clock_seconds": 2.6, "raw": 0.197, "normalized": 0.055294, "discount": 1.0, "effective": 0.055294, "running_max": 0.055294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1351205, "wall_clock_seconds": 2.6, "raw": 0.186, "normalized": 0.042353, "discount": 0.455, "effective": 0.019271, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1497142, "wall_clock_seconds": 10.8, "raw": 0.186281, "normalized": 0.042684, "discount": 0.455, "effective": 0.019421, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1647944, "wall_clock_seconds": 12.7, "raw": 0.216, "normalized": 0.077647, "discount": 0.455, "effective": 0.035329, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1884315, "wall_clock_seconds": 12.1, "raw": 0.2025, "normalized": 0.061765, "discount": 0.455, "effective": 0.028103, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 2134307, "wall_clock_seconds": 12.4, "raw": 0.191, "normalized": 0.048235, "discount": 0.455, "effective": 0.021947, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 2493772, "wall_clock_seconds": 8.9, "raw": 0.2065, "normalized": 0.066471, "discount": 0.455, "effective": 0.030244, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2680743, "wall_clock_seconds": 12.3, "raw": 0.213, "normalized": 0.074118, "discount": 0.455, "effective": 0.033724, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2971355, "wall_clock_seconds": 12.2, "raw": 0.202, "normalized": 0.061176, "discount": 0.455, "effective": 0.027835, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 3271365, "wall_clock_seconds": 12.0, "raw": 0.2, "normalized": 0.058824, "discount": 0.455, "effective": 0.026765, "running_max": 0.055294, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:reaction_rates", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.0, "best_effective": 0.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6620667, "usage": {"input": 40395, "output": 78443, "cache_read": 6186379, "cache_write": 315450}, "cost_usd": 8.4107, "cost_source": "list_price", "wall_clock_seconds": 2791.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 2}, {"id": 3, "multiplier": 0.7, "before_submission": 2}, {"id": 2, "multiplier": 0.75, "before_submission": 4}, {"id": 4, "multiplier": 0.8, "before_submission": 4}, {"id": 5, "multiplier": 0.85, "before_submission": 6}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 416433, "wall_clock_seconds": 3.1, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 2.56}, {"index": 2, "tokens": 958789, "wall_clock_seconds": 9.7, "raw": 0.0, "normalized": 0.0, "discount": 0.49, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 9.22}, {"index": 3, "tokens": 1989655, "wall_clock_seconds": 78.1, "raw": 0.0, "normalized": 0.0, "discount": 0.49, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 77.54}, {"index": 4, "tokens": 2732738, "wall_clock_seconds": 109.6, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 109.11}, {"index": 5, "tokens": 3144608, "wall_clock_seconds": 70.7, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 70.17}, {"index": 6, "tokens": 3914357, "wall_clock_seconds": 10.9, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 10.42}, {"index": 7, "tokens": 4873740, "wall_clock_seconds": 29.6, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 29.22}, {"index": 8, "tokens": 5513575, "wall_clock_seconds": 12.8, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 12.21}, {"index": 9, "tokens": 6193308, "wall_clock_seconds": 10.2, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 9.68}, {"index": 10, "tokens": 6620667, "wall_clock_seconds": 43.2, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 42.7} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:scan_stitch", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.003704, "area_term": 0.003341, "final_best": 0.005158, "best_raw": 0.11179, "best_effective": 0.005158, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3348257, "usage": {"input": 33085, "output": 40964, "cache_read": 3160251, "cache_write": 113957}, "cost_usd": 3.9092, "cost_source": "list_price", "wall_clock_seconds": 1246.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 233691, "wall_clock_seconds": 0.3, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 986492, "wall_clock_seconds": 0.3, "raw": 0.096645, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1704972, "wall_clock_seconds": 0.4, "raw": 0.10217, "normalized": 0.002713, "discount": 0.35, "effective": 0.00095, "running_max": 0.00095, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1853716, "wall_clock_seconds": 0.3, "raw": 0.076832, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.00095, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2007473, "wall_clock_seconds": 0.3, "raw": 0.10339, "normalized": 0.004238, "discount": 0.35, "effective": 0.001483, "running_max": 0.001483, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 2249755, "wall_clock_seconds": 0.3, "raw": 0.089994, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.001483, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 2503411, "wall_clock_seconds": 0.3, "raw": 0.098627, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.001483, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2770771, "wall_clock_seconds": 0.3, "raw": 0.109719, "normalized": 0.012149, "discount": 0.35, "effective": 0.004252, "running_max": 0.004252, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 3053107, "wall_clock_seconds": 0.3, "raw": 0.096327, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.004252, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 3348257, "wall_clock_seconds": 0.3, "raw": 0.11179, "normalized": 0.014738, "discount": 0.35, "effective": 0.005158, "running_max": 0.005158, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:scan_stitch--nohints", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.089115, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2319704, "usage": {"input": 17255, "output": 37387, "cache_read": 2182054, "cache_write": 83008}, "cost_usd": 2.9421, "cost_source": "list_price", "wall_clock_seconds": 1867.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 224622, "wall_clock_seconds": 0.3, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 299392, "wall_clock_seconds": 0.4, "raw": 0.088106, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 655132, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 866752, "wall_clock_seconds": 0.3, "raw": 0.078635, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 979683, "wall_clock_seconds": 0.2, "raw": 0.034614, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1286802, "wall_clock_seconds": 0.3, "raw": 0.084967, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1417264, "wall_clock_seconds": 0.3, "raw": 0.078135, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 1918720, "wall_clock_seconds": 0.3, "raw": 0.083991, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2073381, "wall_clock_seconds": 0.3, "raw": 0.089115, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 2319704, "wall_clock_seconds": 0.3, "raw": 0.067488, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:shapes_generalize", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.022099, "area_term": 0.020296, "final_best": 0.029312, "best_raw": 0.267, "best_effective": 0.029312, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3814444, "usage": {"input": 24090, "output": 55828, "cache_read": 3548951, "cache_write": 185575}, "cost_usd": 5.1464, "cost_source": "list_price", "wall_clock_seconds": 1472.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.35, "before_submission": 4}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 419881, "wall_clock_seconds": 4.9, "raw": 0.165, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.37}, {"index": 2, "tokens": 1028718, "wall_clock_seconds": 11.5, "raw": 0.085, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 10.9}, {"index": 3, "tokens": 1295164, "wall_clock_seconds": 11.5, "raw": 0.028, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 10.82}, {"index": 4, "tokens": 1882060, "wall_clock_seconds": 10.3, "raw": 0.249, "normalized": 0.06125, "discount": 0.35, "effective": 0.021437, "running_max": 0.021437, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 9.71}, {"index": 5, "tokens": 2210989, "wall_clock_seconds": 26.1, "raw": 0.256, "normalized": 0.07, "discount": 0.35, "effective": 0.0245, "running_max": 0.0245, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 25.55}, {"index": 6, "tokens": 2759974, "wall_clock_seconds": 57.5, "raw": 0.23, "normalized": 0.0375, "discount": 0.35, "effective": 0.013125, "running_max": 0.0245, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 56.89}, {"index": 7, "tokens": 2957886, "wall_clock_seconds": 56.1, "raw": 0.267, "normalized": 0.08375, "discount": 0.35, "effective": 0.029312, "running_max": 0.029312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 55.56}, {"index": 8, "tokens": 3266815, "wall_clock_seconds": 6.7, "raw": 0.219, "normalized": 0.02375, "discount": 0.35, "effective": 0.008312, "running_max": 0.029312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 6.11}, {"index": 9, "tokens": 3589549, "wall_clock_seconds": 10.0, "raw": 0.184, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.029312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 9.39}, {"index": 10, "tokens": 3814444, "wall_clock_seconds": 38.4, "raw": 0.21, "normalized": 0.0125, "discount": 0.35, "effective": 0.004375, "running_max": 0.029312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 37.91} ]}, {"sample_key": "6ckQumybaWZDds4RhcrpSb:shattered_prior--nohints", "eval_id": "6ckQumybaWZDds4RhcrpSb", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "a0957d7", "source_log_sha256": "349621e22505b09c1ff535c30cfccc69bd1f31b9c7ec1f46391d2fa043800b5f", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.003564, "area_term": 0.003108, "final_best": 0.005386, "best_raw": 0.005386, "best_effective": 0.005386, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7358123, "usage": {"input": 31717, "output": 89668, "cache_read": 7082949, "cache_write": 153789}, "cost_usd": 7.4796, "cost_source": "list_price", "wall_clock_seconds": 2137.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1153145, "wall_clock_seconds": 1.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1993602, "wall_clock_seconds": 1.8, "raw": 0.000406, "normalized": 0.000406, "discount": 1.0, "effective": 0.000406, "running_max": 0.000406, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2832100, "wall_clock_seconds": 1.5, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.002081, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3140770, "wall_clock_seconds": 1.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.002081, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3928822, "wall_clock_seconds": 1.6, "raw": 0.004327, "normalized": 0.004327, "discount": 1.0, "effective": 0.004327, "running_max": 0.004327, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4792679, "wall_clock_seconds": 1.6, "raw": 0.005386, "normalized": 0.005386, "discount": 1.0, "effective": 0.005386, "running_max": 0.005386, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 5460053, "wall_clock_seconds": 1.7, "raw": 0.00079, "normalized": 0.00079, "discount": 1.0, "effective": 0.00079, "running_max": 0.005386, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5736029, "wall_clock_seconds": 1.7, "raw": 0.000663, "normalized": 0.000663, "discount": 1.0, "effective": 0.000663, "running_max": 0.005386, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6449200, "wall_clock_seconds": 1.6, "raw": 0.003527, "normalized": 0.003527, "discount": 1.0, "effective": 0.003527, "running_max": 0.005386, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7358123, "wall_clock_seconds": 2.2, "raw": 0.001865, "normalized": 0.001865, "discount": 1.0, "effective": 0.001865, "running_max": 0.005386, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:mystery_box", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.032911, "area_term": 0.032911, "final_best": 0.032911, "best_raw": 0.068304, "best_effective": 0.032911, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 4177526, "usage": {"input": 34654, "output": 51627, "cache_read": 3975728, "cache_write": 115517}, "cost_usd": 4.607, "cost_source": "list_price", "wall_clock_seconds": 1669.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 4}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 318699, "wall_clock_seconds": 2.7, "raw": 0.068304, "normalized": 0.032911, "discount": 1.0, "effective": 0.032911, "running_max": 0.032911, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 498957, "wall_clock_seconds": 2.7, "raw": 0.038963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.032911, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 546289, "wall_clock_seconds": 1.8, "raw": 0.03707, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.032911, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1461526, "wall_clock_seconds": 3.2, "raw": 0.058893, "normalized": 0.021969, "discount": 0.25, "effective": 0.005492, "running_max": 0.032911, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2287642, "wall_clock_seconds": 3.6, "raw": 0.048736, "normalized": 0.010158, "discount": 0.25, "effective": 0.00254, "running_max": 0.032911, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 2765771, "wall_clock_seconds": 3.2, "raw": 0.047493, "normalized": 0.008713, "discount": 0.25, "effective": 0.002178, "running_max": 0.032911, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 3173021, "wall_clock_seconds": 3.6, "raw": 0.03421, "normalized": 0.0, "discount": 0.25, "effective": 0.0, "running_max": 0.032911, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 3605307, "wall_clock_seconds": 3.5, "raw": 0.052242, "normalized": 0.014235, "discount": 0.25, "effective": 0.003559, "running_max": 0.032911, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 3944623, "wall_clock_seconds": 3.5, "raw": 0.04398, "normalized": 0.004628, "discount": 0.25, "effective": 0.001157, "running_max": 0.032911, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 4177526, "wall_clock_seconds": 3.6, "raw": 0.059727, "normalized": 0.022938, "discount": 0.25, "effective": 0.005735, "running_max": 0.032911, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:mystery_box--nohints", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.029536, "area_term": 0.029033, "final_best": 0.031544, "best_raw": 0.067128, "best_effective": 0.031544, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2033807, "usage": {"input": 17332, "output": 23880, "cache_read": 1946008, "cache_write": 46587}, "cost_usd": 2.1225, "cost_source": "list_price", "wall_clock_seconds": 652.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 293744, "wall_clock_seconds": 2.4, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 410984, "wall_clock_seconds": 2.6, "raw": 0.038724, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 536035, "wall_clock_seconds": 2.4, "raw": 0.04343, "normalized": 0.003989, "discount": 1.0, "effective": 0.003989, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 813733, "wall_clock_seconds": 2.7, "raw": 0.067128, "normalized": 0.031544, "discount": 1.0, "effective": 0.031544, "running_max": 0.031544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1076577, "wall_clock_seconds": 2.4, "raw": 0.050154, "normalized": 0.011806, "discount": 1.0, "effective": 0.011806, "running_max": 0.031544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1189329, "wall_clock_seconds": 2.9, "raw": 0.046607, "normalized": 0.007682, "discount": 1.0, "effective": 0.007682, "running_max": 0.031544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1306232, "wall_clock_seconds": 2.8, "raw": 0.064563, "normalized": 0.028562, "discount": 1.0, "effective": 0.028562, "running_max": 0.031544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 1553263, "wall_clock_seconds": 2.8, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.031544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 1683976, "wall_clock_seconds": 2.6, "raw": 0.051793, "normalized": 0.013713, "discount": 1.0, "effective": 0.013713, "running_max": 0.031544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 2033807, "wall_clock_seconds": 2.7, "raw": 0.057767, "normalized": 0.02066, "discount": 1.0, "effective": 0.02066, "running_max": 0.031544, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:night_school", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.055294, "area_term": 0.055294, "final_best": 0.055294, "best_raw": 0.2195, "best_effective": 0.055294, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10739797, "usage": {"input": 35876, "output": 104450, "cache_read": 10345221, "cache_write": 254250}, "cost_usd": 10.5057, "cost_source": "list_price", "wall_clock_seconds": 5297.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 338287, "wall_clock_seconds": 2.7, "raw": 0.197, "normalized": 0.055294, "discount": 1.0, "effective": 0.055294, "running_max": 0.055294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1314294, "wall_clock_seconds": 10.7, "raw": 0.2135, "normalized": 0.074706, "discount": 0.65, "effective": 0.048559, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3169803, "wall_clock_seconds": 10.9, "raw": 0.204, "normalized": 0.063529, "discount": 0.65, "effective": 0.041294, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 7243760, "wall_clock_seconds": 8.6, "raw": 0.2075, "normalized": 0.067647, "discount": 0.65, "effective": 0.043971, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 7540896, "wall_clock_seconds": 11.1, "raw": 0.2135, "normalized": 0.074706, "discount": 0.65, "effective": 0.048559, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 8304967, "wall_clock_seconds": 11.3, "raw": 0.2195, "normalized": 0.081765, "discount": 0.65, "effective": 0.053147, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 8616648, "wall_clock_seconds": 10.8, "raw": 0.2125, "normalized": 0.073529, "discount": 0.65, "effective": 0.047794, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 9252557, "wall_clock_seconds": 10.5, "raw": 0.216, "normalized": 0.077647, "discount": 0.65, "effective": 0.050471, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 9575615, "wall_clock_seconds": 11.1, "raw": 0.21, "normalized": 0.070588, "discount": 0.65, "effective": 0.045882, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 10739797, "wall_clock_seconds": 10.7, "raw": 0.2155, "normalized": 0.077059, "discount": 0.65, "effective": 0.050088, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:reaction_rates", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.00687, "area_term": 0.00574, "final_best": 0.011392, "best_raw": 0.038747, "best_effective": 0.011392, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6938691, "usage": {"input": 45523, "output": 78630, "cache_read": 6571744, "cache_write": 242794}, "cost_usd": 7.9072, "cost_source": "list_price", "wall_clock_seconds": 4982.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.7, "before_submission": 3}, {"id": 4, "multiplier": 0.8, "before_submission": 3}, {"id": 1, "multiplier": 0.7, "before_submission": 3}, {"id": 2, "multiplier": 0.75, "before_submission": 6}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 326352, "wall_clock_seconds": 1.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.91}, {"index": 2, "tokens": 1101752, "wall_clock_seconds": 40.2, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 39.77}, {"index": 3, "tokens": 2459392, "wall_clock_seconds": 42.6, "raw": 0.0, "normalized": 0.0, "discount": 0.392, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 42.12}, {"index": 4, "tokens": 2623640, "wall_clock_seconds": 4.7, "raw": 0.0, "normalized": 0.0, "discount": 0.392, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 4.08}, {"index": 5, "tokens": 3176070, "wall_clock_seconds": 40.5, "raw": 0.0, "normalized": 0.0, "discount": 0.392, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 39.98}, {"index": 6, "tokens": 4911437, "wall_clock_seconds": 10.7, "raw": 0.038747, "normalized": 0.038747, "discount": 0.294, "effective": 0.011392, "running_max": 0.011392, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 10.09}, {"index": 7, "tokens": 5158497, "wall_clock_seconds": 11.3, "raw": 0.003388, "normalized": 0.003388, "discount": 0.294, "effective": 0.000996, "running_max": 0.011392, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 10.69}, {"index": 8, "tokens": 5673332, "wall_clock_seconds": 12.0, "raw": 0.0, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.011392, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 11.42}, {"index": 9, "tokens": 6360169, "wall_clock_seconds": 12.2, "raw": 0.035834, "normalized": 0.035834, "discount": 0.294, "effective": 0.010535, "running_max": 0.011392, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 11.61}, {"index": 10, "tokens": 6938691, "wall_clock_seconds": 3.5, "raw": null, "normalized": 0.0, "discount": 0.294, "effective": 0.0, "running_max": 0.011392, "hints_active": 4, "valid": false, "invalid_kind": "missing", "runtime_seconds": 3.22} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:reaction_rates--nohints", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.025707, "area_term": 0.025078, "final_best": 0.028222, "best_raw": 0.028222, "best_effective": 0.028222, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3679978, "usage": {"input": 25069, "output": 47331, "cache_read": 3504710, "cache_write": 102868}, "cost_usd": 4.0897, "cost_source": "list_price", "wall_clock_seconds": 4900.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 606704, "wall_clock_seconds": 4.1, "raw": 0.019913, "normalized": 0.019913, "discount": 1.0, "effective": 0.019913, "running_max": 0.019913, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.56}, {"index": 2, "tokens": 1118555, "wall_clock_seconds": 4.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.019913, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.08}, {"index": 3, "tokens": 1525651, "wall_clock_seconds": 4.2, "raw": 0.025899, "normalized": 0.025899, "discount": 1.0, "effective": 0.025899, "running_max": 0.025899, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.48}, {"index": 4, "tokens": 2071622, "wall_clock_seconds": 4.0, "raw": 0.026942, "normalized": 0.026942, "discount": 1.0, "effective": 0.026942, "running_max": 0.026942, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.54}, {"index": 5, "tokens": 2601726, "wall_clock_seconds": 4.6, "raw": 0.021122, "normalized": 0.021122, "discount": 1.0, "effective": 0.021122, "running_max": 0.026942, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.05}, {"index": 6, "tokens": 2788541, "wall_clock_seconds": 4.3, "raw": 0.00065, "normalized": 0.00065, "discount": 1.0, "effective": 0.00065, "running_max": 0.026942, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.73}, {"index": 7, "tokens": 2979153, "wall_clock_seconds": 4.9, "raw": 0.027364, "normalized": 0.027364, "discount": 1.0, "effective": 0.027364, "running_max": 0.027364, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.17}, {"index": 8, "tokens": 3173039, "wall_clock_seconds": 5.1, "raw": 0.024207, "normalized": 0.024207, "discount": 1.0, "effective": 0.024207, "running_max": 0.027364, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.53}, {"index": 9, "tokens": 3474078, "wall_clock_seconds": 4.6, "raw": 0.027247, "normalized": 0.027247, "discount": 1.0, "effective": 0.027247, "running_max": 0.027364, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.0}, {"index": 10, "tokens": 3679978, "wall_clock_seconds": 4.5, "raw": 0.028222, "normalized": 0.028222, "discount": 1.0, "effective": 0.028222, "running_max": 0.028222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.89} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:scan_stitch", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.097314, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5455771, "usage": {"input": 45993, "output": 53916, "cache_read": 5206350, "cache_write": 149512}, "cost_usd": 5.6762, "cost_source": "list_price", "wall_clock_seconds": 1337.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 260686, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1401424, "wall_clock_seconds": 0.3, "raw": 0.091635, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1978606, "wall_clock_seconds": 0.3, "raw": 0.077201, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2136832, "wall_clock_seconds": 0.3, "raw": 0.076475, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2824718, "wall_clock_seconds": 0.3, "raw": 0.091869, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4038543, "wall_clock_seconds": 0.3, "raw": 0.089735, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4375322, "wall_clock_seconds": 0.3, "raw": 0.087754, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4606836, "wall_clock_seconds": 0.3, "raw": 0.083251, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5205777, "wall_clock_seconds": 0.3, "raw": 0.097314, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5455771, "wall_clock_seconds": 0.3, "raw": 0.089246, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:scan_stitch--nohints", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.091357, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2048076, "usage": {"input": 18458, "output": 32469, "cache_read": 1917464, "cache_write": 79685}, "cost_usd": 2.6596, "cost_source": "list_price", "wall_clock_seconds": 1502.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 227332, "wall_clock_seconds": 0.4, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 379977, "wall_clock_seconds": 0.3, "raw": 0.062212, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 612883, "wall_clock_seconds": 0.3, "raw": 0.060486, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 843487, "wall_clock_seconds": 0.3, "raw": 0.085652, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1095578, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1439359, "wall_clock_seconds": 0.3, "raw": 0.089205, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1584172, "wall_clock_seconds": 0.3, "raw": 0.082163, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 1733688, "wall_clock_seconds": 0.3, "raw": 0.082464, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 1888258, "wall_clock_seconds": 0.3, "raw": 0.091357, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 2048076, "wall_clock_seconds": 0.3, "raw": 0.069994, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:shapes_generalize", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.011356, "area_term": 0.010258, "final_best": 0.01575, "best_raw": 0.236, "best_effective": 0.01575, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 3755995, "usage": {"input": 26199, "output": 52109, "cache_read": 3538658, "cache_write": 139029}, "cost_usd": 4.5933, "cost_source": "list_price", "wall_clock_seconds": 1202.6, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.35, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 361317, "wall_clock_seconds": 1.1, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.35}, {"index": 2, "tokens": 701701, "wall_clock_seconds": 2.1, "raw": 0.144, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.6}, {"index": 3, "tokens": 1744658, "wall_clock_seconds": 14.4, "raw": 0.213, "normalized": 0.01625, "discount": 0.35, "effective": 0.005687, "running_max": 0.005687, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 13.62}, {"index": 4, "tokens": 2074695, "wall_clock_seconds": 23.8, "raw": 0.177, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.005687, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 23.18}, {"index": 5, "tokens": 2527336, "wall_clock_seconds": 1.8, "raw": 0.191, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.005687, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 1.29}, {"index": 6, "tokens": 2718168, "wall_clock_seconds": 11.4, "raw": 0.223, "normalized": 0.02875, "discount": 0.35, "effective": 0.010062, "running_max": 0.010062, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 10.82}, {"index": 7, "tokens": 3015835, "wall_clock_seconds": 10.8, "raw": 0.209, "normalized": 0.01125, "discount": 0.35, "effective": 0.003937, "running_max": 0.010062, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 10.19}, {"index": 8, "tokens": 3325435, "wall_clock_seconds": 10.9, "raw": 0.236, "normalized": 0.045, "discount": 0.35, "effective": 0.01575, "running_max": 0.01575, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 10.15}, {"index": 9, "tokens": 3538056, "wall_clock_seconds": 10.3, "raw": 0.212, "normalized": 0.015, "discount": 0.35, "effective": 0.00525, "running_max": 0.01575, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 9.6}, {"index": 10, "tokens": 3755995, "wall_clock_seconds": 9.8, "raw": 0.195, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.01575, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 9.25} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:shattered_prior", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.045957, "area_term": 0.045264, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6968922, "usage": {"input": 28519, "output": 109470, "cache_read": 6625312, "cache_write": 205621}, "cost_usd": 8.2482, "cost_source": "list_price", "wall_clock_seconds": 4798.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.55, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 693810, "wall_clock_seconds": 1.6, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1650118, "wall_clock_seconds": 1.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3160728, "wall_clock_seconds": 1.9, "raw": 0.001521, "normalized": 0.001521, "discount": 0.55, "effective": 0.000837, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3642130, "wall_clock_seconds": 1.6, "raw": 0.000494, "normalized": 0.000494, "discount": 0.55, "effective": 0.000272, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4153018, "wall_clock_seconds": 1.6, "raw": 0.002518, "normalized": 0.002518, "discount": 0.55, "effective": 0.001385, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4554704, "wall_clock_seconds": 1.5, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4969589, "wall_clock_seconds": 1.6, "raw": 0.002018, "normalized": 0.002018, "discount": 0.55, "effective": 0.00111, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5401042, "wall_clock_seconds": 1.9, "raw": 0.0032, "normalized": 0.0032, "discount": 0.55, "effective": 0.00176, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6637858, "wall_clock_seconds": 1.9, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 6968922, "wall_clock_seconds": 1.6, "raw": 0.04873, "normalized": 0.04873, "discount": 0.55, "effective": 0.026802, "running_max": 0.04873, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "FN9bFR4s59se5EMJKWrmRR:shattered_prior--nohints", "eval_id": "FN9bFR4s59se5EMJKWrmRR", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "opus-4.5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "a0957d7", "source_log_sha256": "63bf5cc0f4b3fb608818276be4605a314da62dc0137148a7bd9206ea73036ade", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.04873, "area_term": 0.04873, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 4650694, "usage": {"input": 20785, "output": 86977, "cache_read": 4427997, "cache_write": 114935}, "cost_usd": 5.6417, "cost_source": "list_price", "wall_clock_seconds": 1630.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 416483, "wall_clock_seconds": 1.7, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1382799, "wall_clock_seconds": 1.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1947760, "wall_clock_seconds": 1.7, "raw": 0.002007, "normalized": 0.002007, "discount": 1.0, "effective": 0.002007, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2713877, "wall_clock_seconds": 1.7, "raw": 0.019857, "normalized": 0.019857, "discount": 1.0, "effective": 0.019857, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2949487, "wall_clock_seconds": 1.6, "raw": 0.004132, "normalized": 0.004132, "discount": 1.0, "effective": 0.004132, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3705718, "wall_clock_seconds": 1.8, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 3970711, "wall_clock_seconds": 1.6, "raw": 0.006698, "normalized": 0.006698, "discount": 1.0, "effective": 0.006698, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4104141, "wall_clock_seconds": 1.6, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 4374659, "wall_clock_seconds": 1.6, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 4650694, "wall_clock_seconds": 1.6, "raw": 0.00119, "normalized": 0.00119, "discount": 1.0, "effective": 0.00119, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "HDDogiqYLdtfeFXNkic6kf:mystery_box", "eval_id": "HDDogiqYLdtfeFXNkic6kf", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "opus-5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "d451af8", "source_log_sha256": "ad42c5946b7fec676034637d71cefd2bb8afe033d642250fa8a6a390d6ea485e", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.048053, "area_term": 0.038197, "final_best": 0.087479, "best_raw": 0.340928, "best_effective": 0.087479, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 29226477, "usage": {"input": 12123, "output": 307605, "cache_read": 28511591, "cache_write": 395158}, "cost_usd": 25.9581, "cost_source": "list_price", "wall_clock_seconds": 9381.6, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 4}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2710643, "wall_clock_seconds": 3.8, "raw": 0.080605, "normalized": 0.047215, "discount": 0.6, "effective": 0.028329, "running_max": 0.028329, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6027293, "wall_clock_seconds": 4.1, "raw": 0.127152, "normalized": 0.10134, "discount": 0.6, "effective": 0.060804, "running_max": 0.060804, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 7839526, "wall_clock_seconds": 3.8, "raw": 0.115217, "normalized": 0.087462, "discount": 0.6, "effective": 0.052477, "running_max": 0.060804, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 12351605, "wall_clock_seconds": 3.4, "raw": 0.256158, "normalized": 0.251347, "discount": 0.25, "effective": 0.062837, "running_max": 0.062837, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 14101959, "wall_clock_seconds": 3.3, "raw": 0.25254, "normalized": 0.247139, "discount": 0.25, "effective": 0.061785, "running_max": 0.062837, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 19359924, "wall_clock_seconds": 3.5, "raw": 0.316935, "normalized": 0.322017, "discount": 0.25, "effective": 0.080504, "running_max": 0.080504, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 20389205, "wall_clock_seconds": 3.2, "raw": 0.315548, "normalized": 0.320405, "discount": 0.25, "effective": 0.080101, "running_max": 0.080504, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 23214494, "wall_clock_seconds": 3.5, "raw": 0.332346, "normalized": 0.339938, "discount": 0.25, "effective": 0.084984, "running_max": 0.084984, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 26554805, "wall_clock_seconds": 3.4, "raw": 0.340928, "normalized": 0.349916, "discount": 0.25, "effective": 0.087479, "running_max": 0.087479, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 29226477, "wall_clock_seconds": 3.5, "raw": 0.324039, "normalized": 0.330278, "discount": 0.25, "effective": 0.08257, "running_max": 0.087479, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "HDDogiqYLdtfeFXNkic6kf:mystery_box--nohints", "eval_id": "HDDogiqYLdtfeFXNkic6kf", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "opus-5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "d451af8", "source_log_sha256": "ad42c5946b7fec676034637d71cefd2bb8afe033d642250fa8a6a390d6ea485e", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.024846, "area_term": 0.024014, "final_best": 0.028171, "best_raw": 0.064227, "best_effective": 0.028171, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 5531277, "usage": {"input": 3895, "output": 184530, "cache_read": 5137782, "cache_write": 205070}, "cost_usd": 9.2523, "cost_source": "list_price", "wall_clock_seconds": 2884.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 572731, "wall_clock_seconds": 3.6, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1015973, "wall_clock_seconds": 2.6, "raw": 0.048971, "normalized": 0.010431, "discount": 1.0, "effective": 0.010431, "running_max": 0.010431, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1198723, "wall_clock_seconds": 2.8, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1618794, "wall_clock_seconds": 2.8, "raw": 0.056621, "normalized": 0.019326, "discount": 1.0, "effective": 0.019326, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2310079, "wall_clock_seconds": 2.8, "raw": 0.052713, "normalized": 0.014782, "discount": 1.0, "effective": 0.014782, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3352246, "wall_clock_seconds": 2.1, "raw": 0.036509, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 3947210, "wall_clock_seconds": 2.2, "raw": 0.04487, "normalized": 0.005662, "discount": 1.0, "effective": 0.005662, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4389918, "wall_clock_seconds": 2.5, "raw": 0.031784, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 4836642, "wall_clock_seconds": 2.2, "raw": 0.036881, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 5531277, "wall_clock_seconds": 2.4, "raw": 0.054727, "normalized": 0.017124, "discount": 1.0, "effective": 0.017124, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "HDDogiqYLdtfeFXNkic6kf:night_school", "eval_id": "HDDogiqYLdtfeFXNkic6kf", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "opus-5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "d451af8", "source_log_sha256": "ad42c5946b7fec676034637d71cefd2bb8afe033d642250fa8a6a390d6ea485e", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.243559, "area_term": 0.196911, "final_best": 0.430147, "best_raw": 0.7125, "best_effective": 0.430147, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 8, "max_submissions": 10, "tokens_total": 37638059, "usage": {"input": 14359, "output": 349835, "cache_read": 36849147, "cache_write": 424718}, "cost_usd": 31.4894, "cost_source": "list_price", "wall_clock_seconds": 28400.5, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}, {"id": 1, "multiplier": 0.85, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4030389, "wall_clock_seconds": 10.8, "raw": 0.521, "normalized": 0.436471, "discount": 0.65, "effective": 0.283706, "running_max": 0.283706, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 11628006, "wall_clock_seconds": 10.2, "raw": 0.5745, "normalized": 0.499412, "discount": 0.65, "effective": 0.324618, "running_max": 0.324618, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 13365794, "wall_clock_seconds": 12.2, "raw": 0.698, "normalized": 0.644706, "discount": 0.65, "effective": 0.419059, "running_max": 0.419059, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 21664255, "wall_clock_seconds": 9.8, "raw": 0.708, "normalized": 0.656471, "discount": 0.65, "effective": 0.426706, "running_max": 0.426706, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 28228051, "wall_clock_seconds": 12.4, "raw": 0.6905, "normalized": 0.635882, "discount": 0.65, "effective": 0.413324, "running_max": 0.426706, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 30284935, "wall_clock_seconds": 9.9, "raw": 0.7125, "normalized": 0.661765, "discount": 0.65, "effective": 0.430147, "running_max": 0.430147, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 32811910, "wall_clock_seconds": 11.5, "raw": 0.696, "normalized": 0.642353, "discount": 0.65, "effective": 0.417529, "running_max": 0.430147, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 33668235, "wall_clock_seconds": 10.3, "raw": 0.693, "normalized": 0.638824, "discount": 0.65, "effective": 0.415235, "running_max": 0.430147, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:mystery_box", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.003564, "area_term": 0.00246, "final_best": 0.007979, "best_raw": 0.067448, "best_effective": 0.007979, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 21774550, "usage": {"input": 618, "output": 85567, "cache_read": 21410392, "cache_write": 277973}, "cost_usd": 0.6005, "cost_source": "billed", "wall_clock_seconds": 2666.4, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.6, "before_submission": 1}, {"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 11741232, "wall_clock_seconds": 4.2, "raw": 0.065179, "normalized": 0.029278, "discount": 0.25, "effective": 0.00732, "running_max": 0.00732, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 12751934, "wall_clock_seconds": 3.5, "raw": 0.065986, "normalized": 0.030216, "discount": 0.25, "effective": 0.007554, "running_max": 0.007554, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 15503338, "wall_clock_seconds": 3.5, "raw": 0.050504, "normalized": 0.012214, "discount": 0.25, "effective": 0.003053, "running_max": 0.007554, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 16840527, "wall_clock_seconds": 3.4, "raw": 0.047643, "normalized": 0.008887, "discount": 0.25, "effective": 0.002222, "running_max": 0.007554, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 17524637, "wall_clock_seconds": 3.4, "raw": 0.05228, "normalized": 0.014279, "discount": 0.25, "effective": 0.00357, "running_max": 0.007554, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 17985728, "wall_clock_seconds": 3.5, "raw": 0.064977, "normalized": 0.029043, "discount": 0.25, "effective": 0.007261, "running_max": 0.007554, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 18449927, "wall_clock_seconds": 3.5, "raw": 0.063728, "normalized": 0.027591, "discount": 0.25, "effective": 0.006898, "running_max": 0.007554, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 18915513, "wall_clock_seconds": 4.6, "raw": 0.065978, "normalized": 0.030206, "discount": 0.25, "effective": 0.007552, "running_max": 0.007554, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 19382931, "wall_clock_seconds": 6.5, "raw": 0.067448, "normalized": 0.031917, "discount": 0.25, "effective": 0.007979, "running_max": 0.007979, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 21774550, "wall_clock_seconds": 3.3, "raw": 0.066834, "normalized": 0.031202, "discount": 0.25, "effective": 0.0078, "running_max": 0.007979, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:mystery_box--nohints", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.027408, "area_term": 0.027218, "final_best": 0.028171, "best_raw": 0.064227, "best_effective": 0.028171, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 2570859, "usage": {"input": 156, "output": 32850, "cache_read": 2460421, "cache_write": 77432}, "cost_usd": 0.108, "cost_source": "billed", "wall_clock_seconds": 798.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 186066, "wall_clock_seconds": 2.7, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 405222, "wall_clock_seconds": 2.7, "raw": 0.051326, "normalized": 0.013169, "discount": 1.0, "effective": 0.013169, "running_max": 0.013169, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 582721, "wall_clock_seconds": 2.6, "raw": 0.05774, "normalized": 0.020628, "discount": 1.0, "effective": 0.020628, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 674369, "wall_clock_seconds": 2.7, "raw": 0.045111, "normalized": 0.005943, "discount": 1.0, "effective": 0.005943, "running_max": 0.020628, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 769081, "wall_clock_seconds": 3.1, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 916860, "wall_clock_seconds": 3.1, "raw": 0.046607, "normalized": 0.007682, "discount": 1.0, "effective": 0.007682, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1018978, "wall_clock_seconds": 3.5, "raw": 0.038963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 1122222, "wall_clock_seconds": 2.6, "raw": 0.05158, "normalized": 0.013465, "discount": 1.0, "effective": 0.013465, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2050544, "wall_clock_seconds": 2.6, "raw": 0.044831, "normalized": 0.005618, "discount": 1.0, "effective": 0.005618, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 2570859, "wall_clock_seconds": 2.8, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:night_school", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.07539, "area_term": 0.060313, "final_best": 0.135697, "best_raw": 0.4035, "best_effective": 0.135697, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 23039740, "usage": {"input": 741, "output": 65094, "cache_read": 22619162, "cache_write": 354743}, "cost_usd": 0.6193, "cost_source": "billed", "wall_clock_seconds": 4472.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}, {"id": 3, "multiplier": 0.7, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 176564, "wall_clock_seconds": 2.7, "raw": 0.197, "normalized": 0.055294, "discount": 0.455, "effective": 0.025159, "running_max": 0.025159, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6811586, "wall_clock_seconds": 12.5, "raw": 0.285625, "normalized": 0.159559, "discount": 0.455, "effective": 0.072599, "running_max": 0.072599, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 10760865, "wall_clock_seconds": 12.8, "raw": 0.220313, "normalized": 0.082721, "discount": 0.455, "effective": 0.037638, "running_max": 0.072599, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 11730321, "wall_clock_seconds": 12.6, "raw": 0.2255, "normalized": 0.088824, "discount": 0.455, "effective": 0.040415, "running_max": 0.072599, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 12397482, "wall_clock_seconds": 12.3, "raw": 0.305312, "normalized": 0.182721, "discount": 0.455, "effective": 0.083138, "running_max": 0.083138, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 12904036, "wall_clock_seconds": 11.8, "raw": 0.259906, "normalized": 0.129301, "discount": 0.455, "effective": 0.058832, "running_max": 0.083138, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 14629707, "wall_clock_seconds": 12.0, "raw": 0.298969, "normalized": 0.175257, "discount": 0.455, "effective": 0.079742, "running_max": 0.083138, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 16567194, "wall_clock_seconds": 12.5, "raw": 0.271062, "normalized": 0.142426, "discount": 0.455, "effective": 0.064804, "running_max": 0.083138, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 18370273, "wall_clock_seconds": 12.3, "raw": 0.4035, "normalized": 0.298235, "discount": 0.455, "effective": 0.135697, "running_max": 0.135697, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 23039740, "wall_clock_seconds": 13.6, "raw": 0.391, "normalized": 0.283529, "discount": 0.455, "effective": 0.129006, "running_max": 0.135697, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:reaction_rates", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.0, "best_effective": 0.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 29704313, "usage": {"input": 933, "output": 91385, "cache_read": 29241891, "cache_write": 370104}, "cost_usd": 0.7872, "cost_source": "billed", "wall_clock_seconds": 7372.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 3}, {"id": 3, "multiplier": 0.7, "before_submission": 3}, {"id": 4, "multiplier": 0.8, "before_submission": 3}, {"id": 5, "multiplier": 0.85, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 458756, "wall_clock_seconds": 1.9, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.38}, {"index": 2, "tokens": 588043, "wall_clock_seconds": 2.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.48}, {"index": 3, "tokens": 20108024, "wall_clock_seconds": 36.0, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 35.44}, {"index": 4, "tokens": 22476126, "wall_clock_seconds": 36.8, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 36.15}, {"index": 5, "tokens": 26825774, "wall_clock_seconds": 157.8, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 157.16}, {"index": 6, "tokens": 27823875, "wall_clock_seconds": 32.5, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 31.91}, {"index": 7, "tokens": 28234817, "wall_clock_seconds": 85.3, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 84.63}, {"index": 8, "tokens": 28856558, "wall_clock_seconds": 115.3, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 114.68}, {"index": 9, "tokens": 29276660, "wall_clock_seconds": 115.3, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 114.64}, {"index": 10, "tokens": 29704313, "wall_clock_seconds": 30.6, "raw": 0.0, "normalized": 0.0, "discount": 0.3332, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": 30.01} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:reaction_rates--nohints", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.003583, "area_term": 0.002865, "final_best": 0.006456, "best_raw": 0.006456, "best_effective": 0.006456, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 10524043, "usage": {"input": 249, "output": 71927, "cache_read": 10262905, "cache_write": 188962}, "cost_usd": 0.3389, "cost_source": "billed", "wall_clock_seconds": 2054.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 626271, "wall_clock_seconds": 5.3, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 5.09}, {"index": 2, "tokens": 800208, "wall_clock_seconds": 1.3, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.83}, {"index": 3, "tokens": 2275513, "wall_clock_seconds": 18.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 17.65}, {"index": 4, "tokens": 4325746, "wall_clock_seconds": 1.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.34}, {"index": 5, "tokens": 4617063, "wall_clock_seconds": 8.0, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 7.78}, {"index": 6, "tokens": 4910197, "wall_clock_seconds": 8.3, "raw": null, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": false, "invalid_kind": "missing", "runtime_seconds": 8.06}, {"index": 7, "tokens": 5354368, "wall_clock_seconds": 30.0, "raw": 0.00311, "normalized": 0.00311, "discount": 1.0, "effective": 0.00311, "running_max": 0.00311, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 29.5}, {"index": 8, "tokens": 6937439, "wall_clock_seconds": 35.9, "raw": 0.003792, "normalized": 0.003792, "discount": 1.0, "effective": 0.003792, "running_max": 0.003792, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 35.44}, {"index": 9, "tokens": 7953380, "wall_clock_seconds": 5.9, "raw": 0.006456, "normalized": 0.006456, "discount": 1.0, "effective": 0.006456, "running_max": 0.006456, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.43}, {"index": 10, "tokens": 10524043, "wall_clock_seconds": 5.3, "raw": 0.002504, "normalized": 0.002504, "discount": 1.0, "effective": 0.002504, "running_max": 0.006456, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 4.67} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:scan_stitch", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.008908, "area_term": 0.007599, "final_best": 0.014143, "best_raw": 0.132326, "best_effective": 0.014143, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 27500170, "usage": {"input": 783, "output": 122620, "cache_read": 26620318, "cache_write": 756449}, "cost_usd": 0.8688, "cost_source": "billed", "wall_clock_seconds": 3954.2, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.8, "before_submission": 1}, {"id": 3, "multiplier": 0.75, "before_submission": 1}, {"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3877522, "wall_clock_seconds": 0.3, "raw": 0.130965, "normalized": 0.038706, "discount": 0.35, "effective": 0.013547, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 6747122, "wall_clock_seconds": 0.3, "raw": 0.087278, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 9554920, "wall_clock_seconds": 0.4, "raw": 0.078113, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 18009415, "wall_clock_seconds": 0.3, "raw": 0.087612, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 21089524, "wall_clock_seconds": 0.3, "raw": 0.117177, "normalized": 0.021471, "discount": 0.35, "effective": 0.007515, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 21375466, "wall_clock_seconds": 0.3, "raw": 0.081523, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 22170772, "wall_clock_seconds": 0.2, "raw": 0.110995, "normalized": 0.013743, "discount": 0.35, "effective": 0.00481, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 23603689, "wall_clock_seconds": 0.3, "raw": 0.124366, "normalized": 0.030458, "discount": 0.35, "effective": 0.01066, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 26078534, "wall_clock_seconds": 0.3, "raw": 0.097939, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.013547, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 27500170, "wall_clock_seconds": 0.3, "raw": 0.132326, "normalized": 0.040407, "discount": 0.35, "effective": 0.014143, "running_max": 0.014143, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:scan_stitch--nohints", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.09917, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 4795411, "usage": {"input": 204, "output": 41578, "cache_read": 4641574, "cache_write": 112055}, "cost_usd": 0.1708, "cost_source": "billed", "wall_clock_seconds": 1031.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 679996, "wall_clock_seconds": 0.3, "raw": 0.068023, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 789575, "wall_clock_seconds": 0.3, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 902147, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 1265360, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 1458664, "wall_clock_seconds": 0.2, "raw": 0.085652, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 1589647, "wall_clock_seconds": 0.4, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 1791509, "wall_clock_seconds": 0.3, "raw": 0.066535, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 2000305, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 2215431, "wall_clock_seconds": 0.3, "raw": 0.091755, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 4795411, "wall_clock_seconds": 0.3, "raw": 0.06813, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:shapes_generalize", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.054807, "area_term": 0.041274, "final_best": 0.108937, "best_raw": 0.449, "best_effective": 0.108937, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 11929979, "usage": {"input": 459, "output": 71351, "cache_read": 11676445, "cache_write": 181724}, "cost_usd": 0.3647, "cost_source": "billed", "wall_clock_seconds": 2955.3, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3828735, "wall_clock_seconds": 0.8, "raw": 0.233, "normalized": 0.04125, "discount": 0.35, "effective": 0.014437, "running_max": 0.014437, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 0.23}, {"index": 2, "tokens": 6189195, "wall_clock_seconds": 24.7, "raw": 0.178, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.014437, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 23.95}, {"index": 3, "tokens": 6747349, "wall_clock_seconds": 0.9, "raw": 0.233, "normalized": 0.04125, "discount": 0.35, "effective": 0.014437, "running_max": 0.014437, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 0.41}, {"index": 4, "tokens": 7320564, "wall_clock_seconds": 0.9, "raw": 0.233, "normalized": 0.04125, "discount": 0.35, "effective": 0.014437, "running_max": 0.014437, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 0.22}, {"index": 5, "tokens": 7611372, "wall_clock_seconds": 3.0, "raw": 0.233, "normalized": 0.04125, "discount": 0.35, "effective": 0.014437, "running_max": 0.014437, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 2.39}, {"index": 6, "tokens": 9244642, "wall_clock_seconds": 31.4, "raw": 0.299, "normalized": 0.12375, "discount": 0.35, "effective": 0.043312, "running_max": 0.043312, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 30.83}, {"index": 7, "tokens": 10163761, "wall_clock_seconds": 61.3, "raw": 0.433, "normalized": 0.29125, "discount": 0.35, "effective": 0.101937, "running_max": 0.101937, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 60.57}, {"index": 8, "tokens": 10633598, "wall_clock_seconds": 80.5, "raw": 0.449, "normalized": 0.31125, "discount": 0.35, "effective": 0.108937, "running_max": 0.108937, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 79.83}, {"index": 9, "tokens": 11111023, "wall_clock_seconds": 0.8, "raw": 0.173, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.108937, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 0.2}, {"index": 10, "tokens": 11929979, "wall_clock_seconds": 82.8, "raw": 0.437, "normalized": 0.29625, "discount": 0.35, "effective": 0.103687, "running_max": 0.108937, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 82.24} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:shattered_prior", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.001178, "area_term": 1.2e-05, "final_best": 0.005843, "best_raw": 0.010624, "best_effective": 0.005843, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 50065151, "usage": {"input": 1302, "output": 179460, "cache_read": 48493508, "cache_write": 1390881}, "cost_usd": 1.5332, "cost_source": "billed", "wall_clock_seconds": 6395.2, "stop_reason": "token_limit", "hints": [{"id": 4, "multiplier": 0.55, "before_submission": 1}, {"id": 2, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.65, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 8051825, "wall_clock_seconds": 2.0, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 30789375, "wall_clock_seconds": 1.6, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 45587167, "wall_clock_seconds": 1.6, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 49418199, "wall_clock_seconds": 1.7, "raw": 0.007399, "normalized": 0.007399, "discount": 0.55, "effective": 0.004069, "running_max": 0.004069, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 49633486, "wall_clock_seconds": 1.6, "raw": 0.004993, "normalized": 0.004993, "discount": 0.55, "effective": 0.002746, "running_max": 0.004069, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 49849156, "wall_clock_seconds": 1.5, "raw": 0.010624, "normalized": 0.010624, "discount": 0.55, "effective": 0.005843, "running_max": 0.005843, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "Jm8xYSCt9yPVQrkA7p4VAZ:shattered_prior--nohints", "eval_id": "Jm8xYSCt9yPVQrkA7p4VAZ", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gpt-5.6-luna-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "a0957d7", "source_log_sha256": "54074b484a1361bd6c843fa62e8e61228bda030df6b4832ad9abbcf468716f07", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.050914, "area_term": 0.049556, "final_best": 0.056346, "best_raw": 0.056346, "best_effective": 0.056346, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 31648186, "usage": {"input": 714, "output": 207953, "cache_read": 30776011, "cache_write": 663508}, "cost_usd": 1.0311, "cost_source": "billed", "wall_clock_seconds": 4494.0, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 162095, "wall_clock_seconds": 1.7, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 16838278, "wall_clock_seconds": 1.8, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 28421452, "wall_clock_seconds": 1.7, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 29299024, "wall_clock_seconds": 1.6, "raw": 0.019857, "normalized": 0.019857, "discount": 1.0, "effective": 0.019857, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 29476796, "wall_clock_seconds": 1.7, "raw": 0.05167, "normalized": 0.05167, "discount": 1.0, "effective": 0.05167, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 29833466, "wall_clock_seconds": 1.7, "raw": 0.050588, "normalized": 0.050588, "discount": 1.0, "effective": 0.050588, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 30012091, "wall_clock_seconds": 1.7, "raw": 0.051253, "normalized": 0.051253, "discount": 1.0, "effective": 0.051253, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 30190967, "wall_clock_seconds": 1.6, "raw": 0.047692, "normalized": 0.047692, "discount": 1.0, "effective": 0.047692, "running_max": 0.05167, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 30914105, "wall_clock_seconds": 1.7, "raw": 0.056346, "normalized": 0.056346, "discount": 1.0, "effective": 0.056346, "running_max": 0.056346, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 31648186, "wall_clock_seconds": 1.6, "raw": 0.054985, "normalized": 0.054985, "discount": 1.0, "effective": 0.054985, "running_max": 0.056346, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "KRiR8Dj8eTqMpvx69PUjsE:mystery_box", "eval_id": "KRiR8Dj8eTqMpvx69PUjsE", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "opus-5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "d451af8", "source_log_sha256": "7e5c972b7335833c49c8bec613dd53a7fdee6383a1c6f982b883ff3c92f563cd", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.05061, "area_term": 0.036123, "final_best": 0.108557, "best_raw": 0.413438, "best_effective": 0.108557, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 28013681, "usage": {"input": 5845, "output": 379826, "cache_read": 27204138, "cache_write": 423872}, "cost_usd": 27.3657, "cost_source": "list_price", "wall_clock_seconds": 6939.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 3}, {"id": 3, "multiplier": 0.25, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 956469, "wall_clock_seconds": 4.5, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1028809, "wall_clock_seconds": 2.8, "raw": 0.04343, "normalized": 0.003989, "discount": 1.0, "effective": 0.003989, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8963207, "wall_clock_seconds": 4.6, "raw": 0.045752, "normalized": 0.006689, "discount": 0.25, "effective": 0.001672, "running_max": 0.00657, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 11714309, "wall_clock_seconds": 3.8, "raw": 0.362416, "normalized": 0.374903, "discount": 0.25, "effective": 0.093726, "running_max": 0.093726, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 15770166, "wall_clock_seconds": 3.6, "raw": 0.394056, "normalized": 0.411693, "discount": 0.25, "effective": 0.102923, "running_max": 0.102923, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 19357180, "wall_clock_seconds": 4.2, "raw": 0.374878, "normalized": 0.389394, "discount": 0.25, "effective": 0.097348, "running_max": 0.102923, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 22905388, "wall_clock_seconds": 3.8, "raw": 0.392549, "normalized": 0.409941, "discount": 0.25, "effective": 0.102485, "running_max": 0.102923, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 23722006, "wall_clock_seconds": 4.0, "raw": 0.366298, "normalized": 0.379416, "discount": 0.25, "effective": 0.094854, "running_max": 0.102923, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 25829930, "wall_clock_seconds": 3.6, "raw": 0.379072, "normalized": 0.394269, "discount": 0.25, "effective": 0.098567, "running_max": 0.102923, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 28013681, "wall_clock_seconds": 3.9, "raw": 0.413438, "normalized": 0.43423, "discount": 0.25, "effective": 0.108557, "running_max": 0.108557, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "KRiR8Dj8eTqMpvx69PUjsE:mystery_box--nohints", "eval_id": "KRiR8Dj8eTqMpvx69PUjsE", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "opus-5", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "d451af8", "source_log_sha256": "7e5c972b7335833c49c8bec613dd53a7fdee6383a1c6f982b883ff3c92f563cd", "agent": "claude_code", "agent_version": "2.1.270", "task_score": 0.020923, "area_term": 0.019111, "final_best": 0.028171, "best_raw": 0.064227, "best_effective": 0.028171, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 7037596, "usage": {"input": 4155, "output": 189100, "cache_read": 6598224, "cache_write": 246117}, "cost_usd": 10.5086, "cost_source": "list_price", "wall_clock_seconds": 2977.3, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1495051, "wall_clock_seconds": 3.9, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1699942, "wall_clock_seconds": 2.3, "raw": 0.04705, "normalized": 0.008198, "discount": 1.0, "effective": 0.008198, "running_max": 0.008198, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 2032317, "wall_clock_seconds": 2.9, "raw": 0.044751, "normalized": 0.005525, "discount": 1.0, "effective": 0.005525, "running_max": 0.008198, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2549140, "wall_clock_seconds": 2.7, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2988848, "wall_clock_seconds": 2.7, "raw": 0.051848, "normalized": 0.013777, "discount": 1.0, "effective": 0.013777, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3851439, "wall_clock_seconds": 2.5, "raw": 0.041816, "normalized": 0.002112, "discount": 1.0, "effective": 0.002112, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4223289, "wall_clock_seconds": 4.2, "raw": 0.044603, "normalized": 0.005352, "discount": 1.0, "effective": 0.005352, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5856905, "wall_clock_seconds": 3.6, "raw": 0.051678, "normalized": 0.01358, "discount": 1.0, "effective": 0.01358, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 6311534, "wall_clock_seconds": 2.2, "raw": 0.041864, "normalized": 0.002167, "discount": 1.0, "effective": 0.002167, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 7037596, "wall_clock_seconds": 2.1, "raw": 0.03853, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:mystery_box", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.002348, "area_term": 0.001562, "final_best": 0.005491, "best_raw": 0.058888, "best_effective": 0.005491, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 29879392, "usage": {"input": 519520, "output": 390912, "cache_read": 28968960, "cache_write": 0}, "cost_usd": 0.7988, "cost_source": "list_price", "wall_clock_seconds": 5940.0, "stop_reason": "submissions_exhausted", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 8632565, "wall_clock_seconds": 5.0, "raw": 0.051228, "normalized": 0.013056, "discount": 0.25, "effective": 0.003264, "running_max": 0.003264, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 12218286, "wall_clock_seconds": 9.4, "raw": 0.049189, "normalized": 0.010685, "discount": 0.25, "effective": 0.002671, "running_max": 0.003264, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 20185472, "wall_clock_seconds": 13.6, "raw": 0.052312, "normalized": 0.014317, "discount": 0.25, "effective": 0.003579, "running_max": 0.003579, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 22124855, "wall_clock_seconds": 8.4, "raw": 0.052674, "normalized": 0.014737, "discount": 0.25, "effective": 0.003684, "running_max": 0.003684, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 26206967, "wall_clock_seconds": 4.3, "raw": 0.042628, "normalized": 0.003056, "discount": 0.25, "effective": 0.000764, "running_max": 0.003684, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 26804774, "wall_clock_seconds": 4.8, "raw": 0.052938, "normalized": 0.015044, "discount": 0.25, "effective": 0.003761, "running_max": 0.003761, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 27404680, "wall_clock_seconds": 7.7, "raw": 0.058888, "normalized": 0.021963, "discount": 0.25, "effective": 0.005491, "running_max": 0.005491, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 28010102, "wall_clock_seconds": 10.8, "raw": 0.033809, "normalized": 0.0, "discount": 0.25, "effective": 0.0, "running_max": 0.005491, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 28621691, "wall_clock_seconds": 7.8, "raw": 0.053412, "normalized": 0.015596, "discount": 0.25, "effective": 0.003899, "running_max": 0.005491, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 29879392, "wall_clock_seconds": 5.1, "raw": 0.050639, "normalized": 0.012371, "discount": 0.25, "effective": 0.003093, "running_max": 0.005491, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:mystery_box--nohints", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.021186, "area_term": 0.019157, "final_best": 0.029302, "best_raw": 0.0652, "best_effective": 0.029302, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9810830, "usage": {"input": 316547, "output": 658315, "cache_read": 8835968, "cache_write": 0}, "cost_usd": 0.938, "cost_source": "list_price", "wall_clock_seconds": 2934.4, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1220045, "wall_clock_seconds": 6.6, "raw": 0.052697, "normalized": 0.014764, "discount": 1.0, "effective": 0.014764, "running_max": 0.014764, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2426361, "wall_clock_seconds": 2.9, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.014764, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 3247330, "wall_clock_seconds": 2.7, "raw": 0.057452, "normalized": 0.020293, "discount": 1.0, "effective": 0.020293, "running_max": 0.020293, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 3431074, "wall_clock_seconds": 2.7, "raw": 0.045122, "normalized": 0.005956, "discount": 1.0, "effective": 0.005956, "running_max": 0.020293, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 4218793, "wall_clock_seconds": 2.6, "raw": 0.039979, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.020293, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4905410, "wall_clock_seconds": 2.7, "raw": 0.038963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.020293, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 6429244, "wall_clock_seconds": 2.8, "raw": 0.064534, "normalized": 0.028527, "discount": 1.0, "effective": 0.028527, "running_max": 0.028527, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 6983370, "wall_clock_seconds": 2.7, "raw": 0.043905, "normalized": 0.00454, "discount": 1.0, "effective": 0.00454, "running_max": 0.028527, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 8295341, "wall_clock_seconds": 2.6, "raw": 0.0652, "normalized": 0.029302, "discount": 1.0, "effective": 0.029302, "running_max": 0.029302, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 9810830, "wall_clock_seconds": 2.5, "raw": 0.05519, "normalized": 0.017663, "discount": 1.0, "effective": 0.017663, "running_max": 0.029302, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:night_school", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.035663, "area_term": 0.019554, "final_best": 0.1001, "best_raw": 0.337, "best_effective": 0.1001, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 41127700, "usage": {"input": 568675, "output": 337457, "cache_read": 40221568, "cache_write": 0}, "cost_usd": 0.8169, "cost_source": "list_price", "wall_clock_seconds": 10341.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 1}, {"id": 3, "multiplier": 0.7, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 14087066, "wall_clock_seconds": 10.9, "raw": 0.2125, "normalized": 0.073529, "discount": 0.455, "effective": 0.033456, "running_max": 0.033456, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 17599453, "wall_clock_seconds": 10.1, "raw": 0.238, "normalized": 0.103529, "discount": 0.455, "effective": 0.047106, "running_max": 0.047106, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 25394661, "wall_clock_seconds": 11.6, "raw": 0.288, "normalized": 0.162353, "discount": 0.455, "effective": 0.073871, "running_max": 0.073871, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 27922020, "wall_clock_seconds": 11.4, "raw": 0.337, "normalized": 0.22, "discount": 0.455, "effective": 0.1001, "running_max": 0.1001, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 29218373, "wall_clock_seconds": 11.7, "raw": 0.309, "normalized": 0.187059, "discount": 0.455, "effective": 0.085112, "running_max": 0.1001, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 34250654, "wall_clock_seconds": 11.8, "raw": 0.3035, "normalized": 0.180588, "discount": 0.455, "effective": 0.082168, "running_max": 0.1001, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 38538916, "wall_clock_seconds": 10.0, "raw": 0.217531, "normalized": 0.079449, "discount": 0.455, "effective": 0.036149, "running_max": 0.1001, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 39640671, "wall_clock_seconds": 11.2, "raw": 0.323, "normalized": 0.203529, "discount": 0.455, "effective": 0.092606, "running_max": 0.1001, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 40380361, "wall_clock_seconds": 12.5, "raw": 0.3195, "normalized": 0.199412, "discount": 0.455, "effective": 0.090732, "running_max": 0.1001, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 41127700, "wall_clock_seconds": 12.5, "raw": 0.321, "normalized": 0.201176, "discount": 0.455, "effective": 0.091535, "running_max": 0.1001, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:reaction_rates", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.0, "best_effective": 0.0, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 7, "max_submissions": 10, "tokens_total": 31183237, "usage": {"input": 588820, "output": 326257, "cache_read": 30268160, "cache_write": 0}, "cost_usd": 0.7498, "cost_source": "list_price", "wall_clock_seconds": 11574.6, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 1}, {"id": 4, "multiplier": 0.8, "before_submission": 1}, {"id": 5, "multiplier": 0.85, "before_submission": 1}, {"id": 2, "multiplier": 0.75, "before_submission": 3}, {"id": 3, "multiplier": 0.7, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 4161465, "wall_clock_seconds": 41.1, "raw": 0.0, "normalized": 0.0, "discount": 0.476, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 40.47}, {"index": 2, "tokens": 11134124, "wall_clock_seconds": 42.4, "raw": 0.0, "normalized": 0.0, "discount": 0.476, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 41.97}, {"index": 3, "tokens": 17040602, "wall_clock_seconds": 42.2, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 41.75}, {"index": 4, "tokens": 20199141, "wall_clock_seconds": 2.1, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 1.68}, {"index": 5, "tokens": 23860068, "wall_clock_seconds": 11.3, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 10.7}, {"index": 6, "tokens": 25261298, "wall_clock_seconds": 32.6, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 32.1}, {"index": 7, "tokens": 27767739, "wall_clock_seconds": 30.4, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 29.9} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:reaction_rates--nohints", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.001785, "area_term": 0.000973, "final_best": 0.005031, "best_raw": 0.005031, "best_effective": 0.005031, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 27994770, "usage": {"input": 374890, "output": 322344, "cache_read": 27297536, "cache_write": 0}, "cost_usd": 0.6631, "cost_source": "list_price", "wall_clock_seconds": 6778.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1105471, "wall_clock_seconds": 2.3, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.86}, {"index": 2, "tokens": 6172504, "wall_clock_seconds": 16.0, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.47}, {"index": 3, "tokens": 7283010, "wall_clock_seconds": 15.2, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.71}, {"index": 4, "tokens": 10789504, "wall_clock_seconds": 1.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 1.02}, {"index": 5, "tokens": 16620767, "wall_clock_seconds": 14.8, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.39}, {"index": 6, "tokens": 20521401, "wall_clock_seconds": 12.8, "raw": 0.005031, "normalized": 0.005031, "discount": 1.0, "effective": 0.005031, "running_max": 0.005031, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.44}, {"index": 7, "tokens": 22785089, "wall_clock_seconds": 26.2, "raw": 0.005031, "normalized": 0.005031, "discount": 1.0, "effective": 0.005031, "running_max": 0.005031, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.83}, {"index": 8, "tokens": 24743478, "wall_clock_seconds": 15.3, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.005031, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 14.86}, {"index": 9, "tokens": 26346746, "wall_clock_seconds": 25.6, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.005031, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.14}, {"index": 10, "tokens": 27994770, "wall_clock_seconds": 12.5, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.005031, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 11.95} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:scan_stitch", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.095736, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 36809539, "usage": {"input": 569231, "output": 312884, "cache_read": 35927424, "cache_write": 0}, "cost_usd": 0.7618, "cost_source": "list_price", "wall_clock_seconds": 6091.9, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}, {"id": 1, "multiplier": 0.85, "before_submission": 9}, {"id": 2, "multiplier": 0.8, "before_submission": 9}, {"id": 3, "multiplier": 0.75, "before_submission": 9}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 13612159, "wall_clock_seconds": 0.3, "raw": 0.086435, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 19488269, "wall_clock_seconds": 0.3, "raw": 0.091288, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 24978817, "wall_clock_seconds": 0.3, "raw": 0.080889, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 27178244, "wall_clock_seconds": 0.2, "raw": 0.093597, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 29482241, "wall_clock_seconds": 0.3, "raw": 0.078857, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 30153918, "wall_clock_seconds": 0.3, "raw": 0.085743, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 32556967, "wall_clock_seconds": 0.3, "raw": 0.093267, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 33604389, "wall_clock_seconds": 0.3, "raw": 0.094266, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 36087909, "wall_clock_seconds": 0.3, "raw": 0.087521, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 36809539, "wall_clock_seconds": 0.3, "raw": 0.095736, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:scan_stitch--nohints", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.098388, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6339907, "usage": {"input": 195519, "output": 166020, "cache_read": 5978368, "cache_write": 0}, "cost_usd": 0.2937, "cost_source": "list_price", "wall_clock_seconds": 1306.8, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1039342, "wall_clock_seconds": 0.2, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 1260991, "wall_clock_seconds": 0.3, "raw": 0.098388, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1633289, "wall_clock_seconds": 0.3, "raw": 0.068023, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2427560, "wall_clock_seconds": 0.3, "raw": 0.066548, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 3751117, "wall_clock_seconds": 0.2, "raw": 0.080224, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 3917093, "wall_clock_seconds": 0.6, "raw": 0.064514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4441720, "wall_clock_seconds": 0.6, "raw": 0.054494, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 4987470, "wall_clock_seconds": 0.3, "raw": 0.082464, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5547888, "wall_clock_seconds": 0.2, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 6339907, "wall_clock_seconds": 0.3, "raw": 0.097215, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:shapes_generalize", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.075352, "area_term": 0.06619, "final_best": 0.112, "best_raw": 0.456, "best_effective": 0.112, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 9195518, "usage": {"input": 198802, "output": 286316, "cache_read": 8710400, "cache_write": 0}, "cost_usd": 0.4555, "cost_source": "list_price", "wall_clock_seconds": 3135.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.35, "before_submission": 1}, {"id": 1, "multiplier": 0.5, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2335500, "wall_clock_seconds": 37.7, "raw": null, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 2, "valid": false, "invalid_kind": "missing", "runtime_seconds": 37.12}, {"index": 2, "tokens": 2613434, "wall_clock_seconds": 37.8, "raw": 0.403, "normalized": 0.25375, "discount": 0.35, "effective": 0.088812, "running_max": 0.088812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 37.24}, {"index": 3, "tokens": 3409483, "wall_clock_seconds": 16.9, "raw": 0.305, "normalized": 0.13125, "discount": 0.35, "effective": 0.045937, "running_max": 0.088812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 16.46}, {"index": 4, "tokens": 3739889, "wall_clock_seconds": 15.2, "raw": 0.232, "normalized": 0.04, "discount": 0.35, "effective": 0.014, "running_max": 0.088812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 14.72}, {"index": 5, "tokens": 4214810, "wall_clock_seconds": 11.2, "raw": 0.301, "normalized": 0.12625, "discount": 0.35, "effective": 0.044187, "running_max": 0.088812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 10.65}, {"index": 6, "tokens": 5903368, "wall_clock_seconds": 60.2, "raw": 0.407, "normalized": 0.25875, "discount": 0.35, "effective": 0.090562, "running_max": 0.090562, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 59.62}, {"index": 7, "tokens": 6380678, "wall_clock_seconds": 66.2, "raw": 0.395, "normalized": 0.24375, "discount": 0.35, "effective": 0.085312, "running_max": 0.090562, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 65.34}, {"index": 8, "tokens": 7046997, "wall_clock_seconds": 57.6, "raw": 0.405, "normalized": 0.25625, "discount": 0.35, "effective": 0.089687, "running_max": 0.090562, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 57.06}, {"index": 9, "tokens": 8121808, "wall_clock_seconds": 43.4, "raw": 0.456, "normalized": 0.32, "discount": 0.35, "effective": 0.112, "running_max": 0.112, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 42.87}, {"index": 10, "tokens": 9195518, "wall_clock_seconds": 71.0, "raw": 0.426, "normalized": 0.2825, "discount": 0.35, "effective": 0.098875, "running_max": 0.112, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 70.46} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:shattered_prior", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.036837, "area_term": 0.033864, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 44297533, "usage": {"input": 830997, "output": 427560, "cache_read": 43038976, "cache_write": 0}, "cost_usd": 1.0206, "cost_source": "list_price", "wall_clock_seconds": 4573.8, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.9, "before_submission": 2}, {"id": 2, "multiplier": 0.7, "before_submission": 2}, {"id": 3, "multiplier": 0.65, "before_submission": 2}, {"id": 4, "multiplier": 0.55, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2037565, "wall_clock_seconds": 1.6, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 31065218, "wall_clock_seconds": 1.6, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 38937658, "wall_clock_seconds": 1.6, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 41841529, "wall_clock_seconds": 1.9, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.04873, "hints_active": 4, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:shattered_prior--nohints", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.044566, "area_term": 0.043525, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 15049482, "usage": {"input": 671870, "output": 401036, "cache_read": 13976576, "cache_write": 0}, "cost_usd": 0.7667, "cost_source": "list_price", "wall_clock_seconds": 3407.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 817673, "wall_clock_seconds": 1.6, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 9819130, "wall_clock_seconds": 1.8, "raw": 0.019857, "normalized": 0.019857, "discount": 1.0, "effective": 0.019857, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 11450806, "wall_clock_seconds": 1.6, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 13204996, "wall_clock_seconds": 1.9, "raw": 0.047841, "normalized": 0.047841, "discount": 1.0, "effective": 0.047841, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:ship_detect", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.039283, "area_term": 0.034375, "final_best": 0.058913, "best_raw": 0.058913, "best_effective": 0.058913, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 53, "max_submissions": null, "tokens_total": 50891793, "usage": {"input": 1052828, "output": 366709, "cache_read": 49472256, "cache_write": 0}, "cost_usd": 1.0527, "cost_source": "list_price", "wall_clock_seconds": 18419.9, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 29, "tokens": 884543, "cost_weighted": 48020.8, "wall_clock_seconds": 682.3, "raw": 0.0, "normalized": 0.0, "effective": 0.0, "running_max": 0.0, "scored": true, "final": false}, {"index": 2, "model_call": 60, "tokens": 3629492, "cost_weighted": 127976.7, "wall_clock_seconds": 1881.6, "raw": 0.056166, "normalized": 0.056166, "effective": 0.056166, "running_max": 0.056166, "scored": true, "final": false}, {"index": 3, "model_call": 86, "tokens": 7087163, "cost_weighted": 206231.8, "wall_clock_seconds": 3453.8, "raw": 0.028834, "normalized": 0.028834, "effective": 0.028834, "running_max": 0.056166, "scored": true, "final": false}, {"index": 4, "model_call": 120, "tokens": 8618612, "cost_weighted": 305274.6, "wall_clock_seconds": 4984.6, "raw": 0.028219, "normalized": 0.028219, "effective": 0.028219, "running_max": 0.056166, "scored": true, "final": false}, {"index": 5, "model_call": 134, "tokens": 9725067, "cost_weighted": 364552.4, "wall_clock_seconds": 6307.2, "raw": 0.025901, "normalized": 0.025901, "effective": 0.025901, "running_max": 0.056166, "scored": true, "final": false}, {"index": 6, "model_call": 148, "tokens": 11086437, "cost_weighted": 436359.4, "wall_clock_seconds": 6666.0, "raw": 0.057271, "normalized": 0.057271, "effective": 0.057271, "running_max": 0.057271, "scored": true, "final": false}, {"index": 7, "model_call": 149, "tokens": 11197122, "cost_weighted": 442546.1, "wall_clock_seconds": 6740.5, "raw": 0.057805, "normalized": 0.057805, "effective": 0.057805, "running_max": 0.057805, "scored": true, "final": false}, {"index": 8, "model_call": 160, "tokens": 12571154, "cost_weighted": 475691.3, "wall_clock_seconds": 7318.4, "raw": 0.057624, "normalized": 0.057624, "effective": 0.057624, "running_max": 0.057805, "scored": true, "final": false}, {"index": 9, "model_call": 188, "tokens": 16766957, "cost_weighted": 550179.4, "wall_clock_seconds": 11335.7, "raw": 0.057867, "normalized": 0.057867, "effective": 0.057867, "running_max": 0.057867, "scored": true, "final": false}, {"index": 10, "model_call": 191, "tokens": 17286025, "cost_weighted": 558909.8, "wall_clock_seconds": 11424.6, "raw": 0.058913, "normalized": 0.058913, "effective": 0.058913, "running_max": 0.058913, "scored": true, "final": false}, {"index": 11, "model_call": 194, "tokens": 17817221, "cost_weighted": 566129.6, "wall_clock_seconds": 11587.8, "raw": 0.041364, "normalized": 0.041364, "effective": 0.041364, "running_max": 0.058913, "scored": true, "final": false}, {"index": 12, "model_call": 195, "tokens": 17996025, "cost_weighted": 567867.6, "wall_clock_seconds": 11608.3, "raw": 0.053714, "normalized": 0.053714, "effective": 0.053714, "running_max": 0.058913, "scored": true, "final": false}, {"index": 13, "model_call": 196, "tokens": 18175223, "cost_weighted": 569319.1, "wall_clock_seconds": 11641.7, "raw": 0.050153, "normalized": 0.050153, "effective": 0.050153, "running_max": 0.058913, "scored": true, "final": false}, {"index": 14, "model_call": 200, "tokens": 18900392, "cost_weighted": 576447.6, "wall_clock_seconds": 11836.0, "raw": 0.057677, "normalized": 0.057677, "effective": 0.057677, "running_max": 0.058913, "scored": true, "final": false}, {"index": 15, "model_call": 212, "tokens": 21231595, "cost_weighted": 608331.4, "wall_clock_seconds": 12600.5, "raw": 0.026629, "normalized": 0.026629, "effective": 0.026629, "running_max": 0.058913, "scored": true, "final": false}, {"index": 16, "model_call": 213, "tokens": 21433390, "cost_weighted": 609583.3, "wall_clock_seconds": 12622.2, "raw": 0.050944, "normalized": 0.050944, "effective": 0.050944, "running_max": 0.058913, "scored": true, "final": false}, {"index": 17, "model_call": 216, "tokens": 22043843, "cost_weighted": 615842.9, "wall_clock_seconds": 12821.8, "raw": 0.031374, "normalized": 0.031374, "effective": 0.031374, "running_max": 0.058913, "scored": true, "final": false}, {"index": 18, "model_call": 220, "tokens": 22871644, "cost_weighted": 623508.1, "wall_clock_seconds": 12917.7, "raw": 0.017006, "normalized": 0.017006, "effective": 0.017006, "running_max": 0.058913, "scored": true, "final": false}, {"index": 19, "model_call": 222, "tokens": 23290816, "cost_weighted": 626969.8, "wall_clock_seconds": 13035.3, "raw": 0.050028, "normalized": 0.050028, "effective": 0.050028, "running_max": 0.058913, "scored": true, "final": false}, {"index": 20, "model_call": 224, "tokens": 23714311, "cost_weighted": 632753.9, "wall_clock_seconds": 13103.9, "raw": 0.052113, "normalized": 0.052113, "effective": 0.052113, "running_max": 0.058913, "scored": true, "final": false}, {"index": 21, "model_call": 228, "tokens": 24576373, "cost_weighted": 642584.2, "wall_clock_seconds": 13225.0, "raw": 0.026533, "normalized": 0.026533, "effective": 0.026533, "running_max": 0.058913, "scored": true, "final": false}, {"index": 22, "model_call": 230, "tokens": 25015049, "cost_weighted": 647712.2, "wall_clock_seconds": 13344.7, "raw": 0.026533, "normalized": 0.026533, "effective": 0.026533, "running_max": 0.058913, "scored": true, "final": false}, {"index": 23, "model_call": 235, "tokens": 26147083, "cost_weighted": 666592.4, "wall_clock_seconds": 13573.0, "raw": 0.010259, "normalized": 0.010259, "effective": 0.010259, "running_max": 0.058913, "scored": true, "final": false}, {"index": 24, "model_call": 236, "tokens": 26381942, "cost_weighted": 669431.4, "wall_clock_seconds": 13598.8, "raw": 0.020345, "normalized": 0.020345, "effective": 0.020345, "running_max": 0.058913, "scored": true, "final": false}, {"index": 25, "model_call": 237, "tokens": 26616917, "cost_weighted": 670762.8, "wall_clock_seconds": 13617.5, "raw": 0.02599, "normalized": 0.02599, "effective": 0.02599, "running_max": 0.058913, "scored": true, "final": false}, {"index": 26, "model_call": 238, "tokens": 26852007, "cost_weighted": 672090.8, "wall_clock_seconds": 13639.5, "raw": 0.025642, "normalized": 0.025642, "effective": 0.025642, "running_max": 0.058913, "scored": true, "final": false}, {"index": 27, "model_call": 239, "tokens": 27087214, "cost_weighted": 673417.4, "wall_clock_seconds": 13652.8, "raw": 0.022564, "normalized": 0.022564, "effective": 0.022564, "running_max": 0.058913, "scored": true, "final": false}, {"index": 28, "model_call": 240, "tokens": 27322537, "cost_weighted": 674741.7, "wall_clock_seconds": 13676.5, "raw": 0.018676, "normalized": 0.018676, "effective": 0.018676, "running_max": 0.058913, "scored": true, "final": false}, {"index": 29, "model_call": 243, "tokens": 28033024, "cost_weighted": 682141.4, "wall_clock_seconds": 13770.9, "raw": 0.028271, "normalized": 0.028271, "effective": 0.028271, "running_max": 0.058913, "scored": true, "final": false}, {"index": 30, "model_call": 264, "tokens": 33264215, "cost_weighted": 731590.1, "wall_clock_seconds": 14899.7, "raw": 0.026187, "normalized": 0.026187, "effective": 0.026187, "running_max": 0.058913, "scored": true, "final": false}, {"index": 31, "model_call": 265, "tokens": 33524093, "cost_weighted": 733721.2, "wall_clock_seconds": 14923.2, "raw": 0.026895, "normalized": 0.026895, "effective": 0.026895, "running_max": 0.058913, "scored": true, "final": false}, {"index": 32, "model_call": 266, "tokens": 33784091, "cost_weighted": 735164.8, "wall_clock_seconds": 14943.3, "raw": 0.026955, "normalized": 0.026955, "effective": 0.026955, "running_max": 0.058913, "scored": true, "final": false}, {"index": 33, "model_call": 269, "tokens": 34568560, "cost_weighted": 742226.1, "wall_clock_seconds": 15050.4, "raw": 0.025553, "normalized": 0.025553, "effective": 0.025553, "running_max": 0.058913, "scored": true, "final": false}, {"index": 34, "model_call": 270, "tokens": 34831485, "cost_weighted": 743962.3, "wall_clock_seconds": 15073.7, "raw": 0.024133, "normalized": 0.024133, "effective": 0.024133, "running_max": 0.058913, "scored": true, "final": false}, {"index": 35, "model_call": 273, "tokens": 35622969, "cost_weighted": 749796.8, "wall_clock_seconds": 15155.3, "raw": 0.027217, "normalized": 0.027217, "effective": 0.027217, "running_max": 0.058913, "scored": true, "final": false}, {"index": 36, "model_call": 277, "tokens": 36689534, "cost_weighted": 759849.2, "wall_clock_seconds": 15267.3, "raw": 0.024077, "normalized": 0.024077, "effective": 0.024077, "running_max": 0.058913, "scored": true, "final": false}, {"index": 37, "model_call": 283, "tokens": 38332127, "cost_weighted": 776022.6, "wall_clock_seconds": 15646.5, "raw": 0.041937, "normalized": 0.041937, "effective": 0.041937, "running_max": 0.058913, "scored": true, "final": false}, {"index": 38, "model_call": 286, "tokens": 39163469, "cost_weighted": 784138.2, "wall_clock_seconds": 15737.6, "raw": 0.036174, "normalized": 0.036174, "effective": 0.036174, "running_max": 0.058913, "scored": true, "final": false}, {"index": 39, "model_call": 290, "tokens": 40289280, "cost_weighted": 794049.9, "wall_clock_seconds": 15949.2, "raw": 0.026443, "normalized": 0.026443, "effective": 0.026443, "running_max": 0.058913, "scored": true, "final": false}, {"index": 40, "model_call": 291, "tokens": 40571981, "cost_weighted": 795740.9, "wall_clock_seconds": 15970.7, "raw": 0.026428, "normalized": 0.026428, "effective": 0.026428, "running_max": 0.058913, "scored": true, "final": false}, {"index": 41, "model_call": 292, "tokens": 40854797, "cost_weighted": 797303.2, "wall_clock_seconds": 15989.7, "raw": 0.02641, "normalized": 0.02641, "effective": 0.02641, "running_max": 0.058913, "scored": true, "final": false}, {"index": 42, "model_call": 293, "tokens": 41137730, "cost_weighted": 798864.1, "wall_clock_seconds": 16007.2, "raw": 0.02621, "normalized": 0.02621, "effective": 0.02621, "running_max": 0.058913, "scored": true, "final": false}, {"index": 43, "model_call": 294, "tokens": 41420778, "cost_weighted": 800421.6, "wall_clock_seconds": 16025.7, "raw": 0.026383, "normalized": 0.026383, "effective": 0.026383, "running_max": 0.058913, "scored": true, "final": false}, {"index": 44, "model_call": 295, "tokens": 41703943, "cost_weighted": 801977.7, "wall_clock_seconds": 16042.8, "raw": 0.026409, "normalized": 0.026409, "effective": 0.026409, "running_max": 0.058913, "scored": true, "final": false}, {"index": 45, "model_call": 296, "tokens": 41987224, "cost_weighted": 803531.5, "wall_clock_seconds": 16059.0, "raw": 0.026385, "normalized": 0.026385, "effective": 0.026385, "running_max": 0.058913, "scored": true, "final": false}, {"index": 46, "model_call": 297, "tokens": 42270620, "cost_weighted": 805081.9, "wall_clock_seconds": 16082.1, "raw": 0.026411, "normalized": 0.026411, "effective": 0.026411, "running_max": 0.058913, "scored": true, "final": false}, {"index": 47, "model_call": 299, "tokens": 42839686, "cost_weighted": 810104.7, "wall_clock_seconds": 16206.4, "raw": 0.044693, "normalized": 0.044693, "effective": 0.044693, "running_max": 0.058913, "scored": true, "final": false}, {"index": 48, "model_call": 304, "tokens": 44278210, "cost_weighted": 821343.3, "wall_clock_seconds": 16335.1, "raw": 0.026382, "normalized": 0.026382, "effective": 0.026382, "running_max": 0.058913, "scored": true, "final": false}, {"index": 49, "model_call": 310, "tokens": 46032047, "cost_weighted": 834317.5, "wall_clock_seconds": 17570.6, "raw": 0.025774, "normalized": 0.025774, "effective": 0.025774, "running_max": 0.058913, "scored": true, "final": false}, {"index": 50, "model_call": 313, "tokens": 46918860, "cost_weighted": 842589.7, "wall_clock_seconds": 17722.8, "raw": 0.026549, "normalized": 0.026549, "effective": 0.026549, "running_max": 0.058913, "scored": true, "final": false}, {"index": 51, "model_call": 321, "tokens": 49346149, "cost_weighted": 865127.9, "wall_clock_seconds": 18226.6, "raw": 0.024349, "normalized": 0.024349, "effective": 0.024349, "running_max": 0.058913, "scored": true, "final": false}, {"index": 52, "model_call": 323, "tokens": 49962261, "cost_weighted": 871294.6, "wall_clock_seconds": 18309.8, "raw": 0.026513, "normalized": 0.026513, "effective": 0.026513, "running_max": 0.058913, "scored": true, "final": false}, {"index": 53, "model_call": 324, "tokens": 50271896, "cost_weighted": 873665.8, "wall_clock_seconds": 18334.0, "raw": 0.02645, "normalized": 0.02645, "effective": 0.02645, "running_max": 0.058913, "scored": true, "final": false} ]}, {"sample_key": "UBZmokaFQURemJepXtEk7k:ship_tune", "eval_id": "UBZmokaFQURemJepXtEk7k", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "2176de0", "source_log_sha256": "f2d50b17e9a0f9818d27311cebb66b93d95c0d00b583af18aae13329dc3379ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.042065, "area_term": 0.034651, "final_best": 0.071722, "best_raw": 0.071722, "best_effective": 0.071722, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 28586725, "usage": {"input": 637959, "output": 308062, "cache_read": 27640704, "cache_write": 0}, "cost_usd": 0.7269, "cost_source": "list_price", "wall_clock_seconds": 9771.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1852523, "wall_clock_seconds": 330.1, "raw": 0.040076, "normalized": 0.040076, "discount": 1.0, "effective": 0.040076, "running_max": 0.040076, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 329.57}, {"index": 2, "tokens": 13382761, "wall_clock_seconds": 308.5, "raw": 0.041415, "normalized": 0.041415, "discount": 1.0, "effective": 0.041415, "running_max": 0.041415, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 307.97}, {"index": 3, "tokens": 19687095, "wall_clock_seconds": 101.1, "raw": 0.016425, "normalized": 0.016425, "discount": 1.0, "effective": 0.016425, "running_max": 0.041415, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 100.5}, {"index": 4, "tokens": 20427079, "wall_clock_seconds": 357.3, "raw": 0.062795, "normalized": 0.062795, "discount": 1.0, "effective": 0.062795, "running_max": 0.062795, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 356.81}, {"index": 5, "tokens": 22723751, "wall_clock_seconds": 355.4, "raw": 0.069228, "normalized": 0.069228, "discount": 1.0, "effective": 0.069228, "running_max": 0.069228, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 354.84}, {"index": 6, "tokens": 25418041, "wall_clock_seconds": 364.2, "raw": 0.066152, "normalized": 0.066152, "discount": 1.0, "effective": 0.066152, "running_max": 0.069228, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 363.65}, {"index": 7, "tokens": 25976389, "wall_clock_seconds": 371.6, "raw": 0.065012, "normalized": 0.065012, "discount": 1.0, "effective": 0.065012, "running_max": 0.069228, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 371.01}, {"index": 8, "tokens": 27117366, "wall_clock_seconds": 360.6, "raw": 0.071722, "normalized": 0.071722, "discount": 1.0, "effective": 0.071722, "running_max": 0.071722, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 359.27}, {"index": 9, "tokens": 27697127, "wall_clock_seconds": 363.9, "raw": 0.000207, "normalized": 0.000207, "discount": 1.0, "effective": 0.000207, "running_max": 0.071722, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 363.27}, {"index": 10, "tokens": 28586725, "wall_clock_seconds": 356.3, "raw": 0.000207, "normalized": 0.000207, "discount": 1.0, "effective": 0.000207, "running_max": 0.071722, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 355.7} ]}, {"sample_key": "XW9JqiiFfZQvc4fn2B43z9:scan_stitch--nohints", "eval_id": "XW9JqiiFfZQvc4fn2B43z9", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gpt-6-astra-xhigh-openrouter", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "2176de0", "source_log_sha256": "a45fb55412170ff5550f7f42b360af9d1b5ddcf6c9a9ad61913e733a3f9c58fd", "agent": "codex_cli", "agent_version": "0.154.0", "task_score": 0.000301, "area_term": 0.000237, "final_best": 0.000557, "best_raw": 0.100445, "best_effective": 0.000557, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 13730173, "usage": {"input": 390, "output": 187366, "cache_read": 12810547, "cache_write": 731870}, "cost_usd": 31.3311, "cost_source": "billed", "wall_clock_seconds": 6853.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1689129, "wall_clock_seconds": 0.4, "raw": 0.085305, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 3211508, "wall_clock_seconds": 0.4, "raw": 0.085353, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 4859607, "wall_clock_seconds": 0.4, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 7026984, "wall_clock_seconds": 0.3, "raw": 0.100445, "normalized": 0.000557, "discount": 1.0, "effective": 0.000557, "running_max": 0.000557, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 8944264, "wall_clock_seconds": 0.3, "raw": 0.076516, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000557, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 9309316, "wall_clock_seconds": 0.3, "raw": 0.076516, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000557, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 10281720, "wall_clock_seconds": 0.4, "raw": 0.064939, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000557, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 12257882, "wall_clock_seconds": 0.3, "raw": 0.086918, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000557, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 13330672, "wall_clock_seconds": 0.3, "raw": 0.077987, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000557, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 13730173, "wall_clock_seconds": 0.4, "raw": 0.090533, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.000557, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "YXiUmNKiYZkbtwAvYSpAvS:mystery_box--nohints", "eval_id": "YXiUmNKiYZkbtwAvYSpAvS", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "glm-5.3-flash-novita-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "d451af8", "source_log_sha256": "19962c79e95f5e32a5d0c65fef5baeb4b1e39b0613bc3bf581d6d81d9ba3e9ce", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.017454, "area_term": 0.016751, "final_best": 0.020267, "best_raw": 0.05743, "best_effective": 0.020267, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 6330202, "usage": {"input": 237251, "output": 796567, "cache_read": 5296384, "cache_write": 0}, "cost_usd": 0.2964, "cost_source": "list_price", "wall_clock_seconds": 4832.5, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 407350, "wall_clock_seconds": 12.4, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 468565, "wall_clock_seconds": 5.2, "raw": 0.038963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 1630856, "wall_clock_seconds": 3.1, "raw": 0.05743, "normalized": 0.020267, "discount": 1.0, "effective": 0.020267, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 2209881, "wall_clock_seconds": 1.6, "raw": 0.044388, "normalized": 0.005102, "discount": 1.0, "effective": 0.005102, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 2877588, "wall_clock_seconds": 2.7, "raw": 0.053739, "normalized": 0.015975, "discount": 1.0, "effective": 0.015975, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 4165336, "wall_clock_seconds": 3.9, "raw": 0.047903, "normalized": 0.009189, "discount": 1.0, "effective": 0.009189, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 4784369, "wall_clock_seconds": 4.2, "raw": 0.040619, "normalized": 0.00072, "discount": 1.0, "effective": 0.00072, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 5108423, "wall_clock_seconds": 5.4, "raw": 0.04923, "normalized": 0.010733, "discount": 1.0, "effective": 0.010733, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 5606499, "wall_clock_seconds": 2.8, "raw": 0.044411, "normalized": 0.005129, "discount": 1.0, "effective": 0.005129, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 6330202, "wall_clock_seconds": 3.3, "raw": 0.042522, "normalized": 0.002933, "discount": 1.0, "effective": 0.002933, "running_max": 0.020267, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "ggjyU6K9mSoAUVPNPaQkkv:shapes_generalize", "eval_id": "ggjyU6K9mSoAUVPNPaQkkv", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "2176de0", "source_log_sha256": "3dcdac639a0c321aa1ea996fa7a7ba91c2406ab993d51de42d5112e6d04b859e", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.072736, "area_term": 0.038529, "final_best": 0.209562, "best_raw": 0.679, "best_effective": 0.209562, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 34753860, "usage": {"input": 549447, "output": 226941, "cache_read": 33977472, "cache_write": 0}, "cost_usd": 0.641, "cost_source": "list_price", "wall_clock_seconds": 8126.5, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.5, "before_submission": 1}, {"id": 2, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 16162760, "wall_clock_seconds": 6.8, "raw": 0.435, "normalized": 0.29375, "discount": 0.35, "effective": 0.102812, "running_max": 0.102812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 6.05}, {"index": 2, "tokens": 16832889, "wall_clock_seconds": 6.4, "raw": 0.408, "normalized": 0.26, "discount": 0.35, "effective": 0.091, "running_max": 0.102812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 5.76}, {"index": 3, "tokens": 18680602, "wall_clock_seconds": 8.9, "raw": 0.435, "normalized": 0.29375, "discount": 0.35, "effective": 0.102812, "running_max": 0.102812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 8.15}, {"index": 4, "tokens": 21109537, "wall_clock_seconds": 8.4, "raw": 0.435, "normalized": 0.29375, "discount": 0.35, "effective": 0.102812, "running_max": 0.102812, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 7.48}, {"index": 5, "tokens": 24989668, "wall_clock_seconds": 7.4, "raw": 0.556, "normalized": 0.445, "discount": 0.35, "effective": 0.15575, "running_max": 0.15575, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 6.67}, {"index": 6, "tokens": 25523421, "wall_clock_seconds": 8.4, "raw": 0.556, "normalized": 0.445, "discount": 0.35, "effective": 0.15575, "running_max": 0.15575, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 7.58}, {"index": 7, "tokens": 28963061, "wall_clock_seconds": 6.6, "raw": 0.615, "normalized": 0.51875, "discount": 0.35, "effective": 0.181562, "running_max": 0.181562, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 5.83}, {"index": 8, "tokens": 30582291, "wall_clock_seconds": 6.1, "raw": 0.573, "normalized": 0.46625, "discount": 0.35, "effective": 0.163187, "running_max": 0.181562, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 5.37}, {"index": 9, "tokens": 31949227, "wall_clock_seconds": 6.7, "raw": 0.624, "normalized": 0.53, "discount": 0.35, "effective": 0.1855, "running_max": 0.1855, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 5.75}, {"index": 10, "tokens": 34753860, "wall_clock_seconds": 6.9, "raw": 0.679, "normalized": 0.59875, "discount": 0.35, "effective": 0.209562, "running_max": 0.209562, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 6.25} ]}, {"sample_key": "ggjyU6K9mSoAUVPNPaQkkv:ship_detect", "eval_id": "ggjyU6K9mSoAUVPNPaQkkv", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "2176de0", "source_log_sha256": "3dcdac639a0c321aa1ea996fa7a7ba91c2406ab993d51de42d5112e6d04b859e", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.052042, "area_term": 0.049708, "final_best": 0.061378, "best_raw": 0.061378, "best_effective": 0.061378, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 13, "max_submissions": null, "tokens_total": 18521112, "usage": {"input": 1179608, "output": 265152, "cache_read": 17076352, "cache_write": 0}, "cost_usd": 0.7745, "cost_source": "list_price", "wall_clock_seconds": 24843.4, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 32, "tokens": 1095173, "cost_weighted": 48005.9, "wall_clock_seconds": 1916.3, "raw": 0.061378, "normalized": 0.061378, "effective": 0.061378, "running_max": 0.061378, "scored": true, "final": false}, {"index": 2, "model_call": 63, "tokens": 3683922, "cost_weighted": 109157.8, "wall_clock_seconds": 5647.5, "raw": 0.031904, "normalized": 0.031904, "effective": 0.031904, "running_max": 0.061378, "scored": true, "final": false}, {"index": 3, "model_call": 75, "tokens": 5037008, "cost_weighted": 135842.6, "wall_clock_seconds": 9341.9, "raw": 0.000237, "normalized": 0.000237, "effective": 0.000237, "running_max": 0.061378, "scored": true, "final": false}, {"index": 4, "model_call": 80, "tokens": 5666122, "cost_weighted": 144382.9, "wall_clock_seconds": 9503.6, "raw": 0.031198, "normalized": 0.031198, "effective": 0.031198, "running_max": 0.061378, "scored": true, "final": false}, {"index": 5, "model_call": 83, "tokens": 6053017, "cost_weighted": 148679.7, "wall_clock_seconds": 12912.0, "raw": 0.049004, "normalized": 0.049004, "effective": 0.049004, "running_max": 0.061378, "scored": true, "final": false}, {"index": 6, "model_call": 103, "tokens": 7886265, "cost_weighted": 242476.8, "wall_clock_seconds": 16706.2, "raw": 0.046592, "normalized": 0.046592, "effective": 0.046592, "running_max": 0.061378, "scored": true, "final": false}, {"index": 7, "model_call": 112, "tokens": 8130773, "cost_weighted": 268100.8, "wall_clock_seconds": 16862.4, "raw": 0.044242, "normalized": 0.044242, "effective": 0.044242, "running_max": 0.061378, "scored": true, "final": false}, {"index": 8, "model_call": 125, "tokens": 8779657, "cost_weighted": 296098.4, "wall_clock_seconds": 17257.8, "raw": 0.039877, "normalized": 0.039877, "effective": 0.039877, "running_max": 0.061378, "scored": true, "final": false}, {"index": 9, "model_call": 139, "tokens": 9791535, "cost_weighted": 328021.7, "wall_clock_seconds": 17715.5, "raw": 0.039388, "normalized": 0.039388, "effective": 0.039388, "running_max": 0.061378, "scored": true, "final": false}, {"index": 10, "model_call": 163, "tokens": 12100034, "cost_weighted": 375319.3, "wall_clock_seconds": 19116.4, "raw": 0.039877, "normalized": 0.039877, "effective": 0.039877, "running_max": 0.061378, "scored": true, "final": false}, {"index": 11, "model_call": 185, "tokens": 14850461, "cost_weighted": 479297.0, "wall_clock_seconds": 22906.3, "raw": 0.041197, "normalized": 0.041197, "effective": 0.041197, "running_max": 0.061378, "scored": true, "final": false}, {"index": 12, "model_call": 213, "tokens": 16038339, "cost_weighted": 560662.6, "wall_clock_seconds": 23638.0, "raw": 0.039524, "normalized": 0.039524, "effective": 0.039524, "running_max": 0.061378, "scored": true, "final": false}, {"index": 13, "model_call": 238, "tokens": 18075069, "cost_weighted": 629597.3, "wall_clock_seconds": 24556.7, "raw": 0.045453, "normalized": 0.045453, "effective": 0.045453, "running_max": 0.061378, "scored": true, "final": false} ]}, {"sample_key": "ggjyU6K9mSoAUVPNPaQkkv:splash_generalize", "eval_id": "ggjyU6K9mSoAUVPNPaQkkv", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "deepseek-v4.1-flash-novita-high-opencode", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "2176de0", "source_log_sha256": "3dcdac639a0c321aa1ea996fa7a7ba91c2406ab993d51de42d5112e6d04b859e", "agent": "opencode", "agent_version": "1.18.30", "task_score": 0.105691, "area_term": 0.076419, "final_best": 0.222778, "best_raw": 0.501, "best_effective": 0.222778, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 32275204, "usage": {"input": 254975, "output": 244613, "cache_read": 31775616, "cache_write": 0}, "cost_usd": 0.5607, "cost_source": "list_price", "wall_clock_seconds": 7492.1, "stop_reason": "submissions_exhausted", "hints": [{"id": 2, "multiplier": 0.5, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 9781427, "wall_clock_seconds": 10.4, "raw": 0.464, "normalized": 0.404444, "discount": 0.5, "effective": 0.202222, "running_max": 0.202222, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 9.65}, {"index": 2, "tokens": 12134036, "wall_clock_seconds": 20.5, "raw": 0.465, "normalized": 0.405556, "discount": 0.5, "effective": 0.202778, "running_max": 0.202778, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 19.83}, {"index": 3, "tokens": 15510718, "wall_clock_seconds": 18.6, "raw": 0.49, "normalized": 0.433333, "discount": 0.5, "effective": 0.216667, "running_max": 0.216667, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 17.87}, {"index": 4, "tokens": 18000451, "wall_clock_seconds": 22.7, "raw": 0.497, "normalized": 0.441111, "discount": 0.5, "effective": 0.220556, "running_max": 0.220556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 22.11}, {"index": 5, "tokens": 21895062, "wall_clock_seconds": 32.4, "raw": 0.11, "normalized": 0.011111, "discount": 0.5, "effective": 0.005556, "running_max": 0.220556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 31.92}, {"index": 6, "tokens": 22662465, "wall_clock_seconds": 17.1, "raw": 0.496, "normalized": 0.44, "discount": 0.5, "effective": 0.22, "running_max": 0.220556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 16.57}, {"index": 7, "tokens": 27547278, "wall_clock_seconds": 20.8, "raw": 0.457, "normalized": 0.396667, "discount": 0.5, "effective": 0.198333, "running_max": 0.220556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 20.27}, {"index": 8, "tokens": 28993097, "wall_clock_seconds": 39.9, "raw": 0.483, "normalized": 0.425556, "discount": 0.5, "effective": 0.212778, "running_max": 0.220556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 38.85}, {"index": 9, "tokens": 30467609, "wall_clock_seconds": 48.7, "raw": 0.493, "normalized": 0.436667, "discount": 0.5, "effective": 0.218333, "running_max": 0.220556, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 48.08}, {"index": 10, "tokens": 32275204, "wall_clock_seconds": 30.5, "raw": 0.501, "normalized": 0.445556, "discount": 0.5, "effective": 0.222778, "running_max": 0.222778, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 29.74} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:mystery_box", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "mystery_box", "task": "mystery_box", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.001993, "area_term": 0.001012, "final_best": 0.005914, "best_raw": 0.060343, "best_effective": 0.005914, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 50099155, "usage": {"input": 395321, "output": 234931, "cache_read": 49468903, "cache_write": 48525445}, "cost_usd": 6.9112, "cost_source": "list_price", "wall_clock_seconds": 4832.2, "stop_reason": "token_limit", "hints": [{"id": 3, "multiplier": 0.25, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 12350681, "wall_clock_seconds": 10.9, "raw": 0.049987, "normalized": 0.011613, "discount": 0.25, "effective": 0.002903, "running_max": 0.002903, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 27328045, "wall_clock_seconds": 10.4, "raw": 0.050648, "normalized": 0.012382, "discount": 0.25, "effective": 0.003095, "running_max": 0.003095, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 38056793, "wall_clock_seconds": 9.1, "raw": 0.055642, "normalized": 0.018189, "discount": 0.25, "effective": 0.004547, "running_max": 0.004547, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 46164248, "wall_clock_seconds": 10.6, "raw": 0.056152, "normalized": 0.018782, "discount": 0.25, "effective": 0.004695, "running_max": 0.004695, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 46864936, "wall_clock_seconds": 8.4, "raw": 0.060343, "normalized": 0.023654, "discount": 0.25, "effective": 0.005914, "running_max": 0.005914, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 47571031, "wall_clock_seconds": 7.4, "raw": 0.059463, "normalized": 0.022632, "discount": 0.25, "effective": 0.005658, "running_max": 0.005914, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 48282447, "wall_clock_seconds": 9.2, "raw": 0.059857, "normalized": 0.02309, "discount": 0.25, "effective": 0.005772, "running_max": 0.005914, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 48999473, "wall_clock_seconds": 10.3, "raw": 0.060178, "normalized": 0.023462, "discount": 0.25, "effective": 0.005866, "running_max": 0.005914, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 49736871, "wall_clock_seconds": 8.2, "raw": 0.059271, "normalized": 0.022409, "discount": 0.25, "effective": 0.005602, "running_max": 0.005914, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:mystery_box--nohints", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "mystery_box--nohints", "task": "mystery_box", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.2", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.013783, "area_term": 0.010071, "final_best": 0.028631, "best_raw": 0.064623, "best_effective": 0.028631, "floor": 0.04, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 17394742, "usage": {"input": 157047, "output": 85732, "cache_read": 17151963, "cache_write": 17151963}, "cost_usd": 2.4409, "cost_source": "list_price", "wall_clock_seconds": 1779.6, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5710148, "wall_clock_seconds": 8.6, "raw": 0.04565, "normalized": 0.00657, "discount": 1.0, "effective": 0.00657, "running_max": 0.00657, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 7015189, "wall_clock_seconds": 8.5, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8863547, "wall_clock_seconds": 7.2, "raw": 0.04671, "normalized": 0.007802, "discount": 1.0, "effective": 0.007802, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 10259080, "wall_clock_seconds": 7.9, "raw": 0.038963, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.007802, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 11990830, "wall_clock_seconds": 6.0, "raw": 0.064227, "normalized": 0.028171, "discount": 1.0, "effective": 0.028171, "running_max": 0.028171, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 13029837, "wall_clock_seconds": 7.8, "raw": 0.064623, "normalized": 0.028631, "discount": 1.0, "effective": 0.028631, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 14106109, "wall_clock_seconds": 3.1, "raw": 0.063534, "normalized": 0.027365, "discount": 1.0, "effective": 0.027365, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 14792423, "wall_clock_seconds": 2.8, "raw": 0.061452, "normalized": 0.024944, "discount": 1.0, "effective": 0.024944, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 16360265, "wall_clock_seconds": 1.4, "raw": 0.037628, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 17394742, "wall_clock_seconds": 3.0, "raw": 0.036529, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.028631, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:night_school", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "night_school", "task": "night_school", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v2.5", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.181009, "area_term": 0.145106, "final_best": 0.324618, "best_raw": 0.5745, "best_effective": 0.324618, "floor": 0.15, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 14780436, "usage": {"input": 210349, "output": 129975, "cache_read": 14440112, "cache_write": 14440112}, "cost_usd": 2.3303, "cost_source": "list_price", "wall_clock_seconds": 8180.3, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.65, "before_submission": 2}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1171064, "wall_clock_seconds": 2.6, "raw": 0.197, "normalized": 0.055294, "discount": 1.0, "effective": 0.055294, "running_max": 0.055294, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 2961073, "wall_clock_seconds": 11.1, "raw": 0.2135, "normalized": 0.074706, "discount": 0.65, "effective": 0.048559, "running_max": 0.055294, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 8680155, "wall_clock_seconds": 11.1, "raw": 0.4815, "normalized": 0.39, "discount": 0.65, "effective": 0.2535, "running_max": 0.2535, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 9492066, "wall_clock_seconds": 10.9, "raw": 0.542, "normalized": 0.461176, "discount": 0.65, "effective": 0.299765, "running_max": 0.299765, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 11405062, "wall_clock_seconds": 11.0, "raw": 0.5745, "normalized": 0.499412, "discount": 0.65, "effective": 0.324618, "running_max": 0.324618, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:reaction_rates", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "reaction_rates", "task": "reaction_rates", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.017195, "area_term": 0.005896, "final_best": 0.062387, "best_raw": 0.249649, "best_effective": 0.062387, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 9, "max_submissions": 10, "tokens_total": 47128576, "usage": {"input": 362253, "output": 222128, "cache_read": 46544195, "cache_write": 45750276}, "cost_usd": 6.5033, "cost_source": "list_price", "wall_clock_seconds": 6633.0, "stop_reason": "agent_finished", "hints": [{"id": 1, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.7, "before_submission": 2}, {"id": 4, "multiplier": 0.8, "before_submission": 2}, {"id": 2, "multiplier": 0.75, "before_submission": 3}, {"id": 5, "multiplier": 0.85, "before_submission": 3}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2079031, "wall_clock_seconds": 3.8, "raw": 0.0, "normalized": 0.0, "discount": 0.7, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": 3.46}, {"index": 2, "tokens": 12704073, "wall_clock_seconds": 24.6, "raw": 0.0, "normalized": 0.0, "discount": 0.392, "effective": 0.0, "running_max": 0.0, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": 24.0}, {"index": 3, "tokens": 18964349, "wall_clock_seconds": 4.5, "raw": 0.0, "normalized": 0.0, "discount": 0.2499, "effective": 0.0, "running_max": 0.0, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 4.11}, {"index": 4, "tokens": 28634055, "wall_clock_seconds": 34.0, "raw": 0.089854, "normalized": 0.089854, "discount": 0.2499, "effective": 0.022455, "running_max": 0.022455, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 33.49}, {"index": 5, "tokens": 34630267, "wall_clock_seconds": 37.9, "raw": 0.249046, "normalized": 0.249046, "discount": 0.2499, "effective": 0.062237, "running_max": 0.062237, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 37.5}, {"index": 6, "tokens": 39308627, "wall_clock_seconds": 37.9, "raw": 0.249046, "normalized": 0.249046, "discount": 0.2499, "effective": 0.062237, "running_max": 0.062237, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 37.52}, {"index": 7, "tokens": 41697689, "wall_clock_seconds": 37.0, "raw": 0.249581, "normalized": 0.249581, "discount": 0.2499, "effective": 0.06237, "running_max": 0.06237, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 36.48}, {"index": 8, "tokens": 43865629, "wall_clock_seconds": 35.8, "raw": 0.210743, "normalized": 0.210743, "discount": 0.2499, "effective": 0.052665, "running_max": 0.06237, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 35.41}, {"index": 9, "tokens": 45806188, "wall_clock_seconds": 35.8, "raw": 0.249649, "normalized": 0.249649, "discount": 0.2499, "effective": 0.062387, "running_max": 0.062387, "hints_active": 5, "valid": true, "invalid_kind": null, "runtime_seconds": 35.39} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:reaction_rates--nohints", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "reaction_rates--nohints", "task": "reaction_rates", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.5", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.06238, "area_term": 0.047407, "final_best": 0.122271, "best_raw": 0.122271, "best_effective": 0.122271, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 41712294, "usage": {"input": 295203, "output": 188099, "cache_read": 41228992, "cache_write": 40762037}, "cost_usd": 5.7187, "cost_source": "list_price", "wall_clock_seconds": 3466.2, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 1907986, "wall_clock_seconds": 4.1, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 3.72}, {"index": 2, "tokens": 2114322, "wall_clock_seconds": 55.7, "raw": 0.049088, "normalized": 0.049088, "discount": 1.0, "effective": 0.049088, "running_max": 0.049088, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 55.18}, {"index": 3, "tokens": 8894814, "wall_clock_seconds": 36.6, "raw": 0.079122, "normalized": 0.079122, "discount": 1.0, "effective": 0.079122, "running_max": 0.079122, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 36.09}, {"index": 4, "tokens": 23962242, "wall_clock_seconds": 32.9, "raw": 0.054923, "normalized": 0.054923, "discount": 1.0, "effective": 0.054923, "running_max": 0.079122, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 31.84}, {"index": 5, "tokens": 25634749, "wall_clock_seconds": 26.9, "raw": 0.055521, "normalized": 0.055521, "discount": 1.0, "effective": 0.055521, "running_max": 0.079122, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.54}, {"index": 6, "tokens": 33138736, "wall_clock_seconds": 25.3, "raw": 0.055521, "normalized": 0.055521, "discount": 1.0, "effective": 0.055521, "running_max": 0.079122, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 25.0}, {"index": 7, "tokens": 36384620, "wall_clock_seconds": 34.2, "raw": 0.080591, "normalized": 0.080591, "discount": 1.0, "effective": 0.080591, "running_max": 0.080591, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 33.86}, {"index": 8, "tokens": 38465818, "wall_clock_seconds": 24.4, "raw": 0.118975, "normalized": 0.118975, "discount": 1.0, "effective": 0.118975, "running_max": 0.118975, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 24.07}, {"index": 9, "tokens": 40602784, "wall_clock_seconds": 31.0, "raw": 0.120837, "normalized": 0.120837, "discount": 1.0, "effective": 0.120837, "running_max": 0.120837, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 30.66}, {"index": 10, "tokens": 41712294, "wall_clock_seconds": 33.6, "raw": 0.122271, "normalized": 0.122271, "discount": 1.0, "effective": 0.122271, "running_max": 0.122271, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 33.26} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:scan_stitch", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "scan_stitch", "task": "scan_stitch", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.0, "area_term": 0.0, "final_best": 0.0, "best_raw": 0.09978, "best_effective": 0.0, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 47287649, "usage": {"input": 354811, "output": 205646, "cache_read": 46727192, "cache_write": 46021939}, "cost_usd": 6.4609, "cost_source": "list_price", "wall_clock_seconds": 3129.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 4, "multiplier": 0.35, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 3990541, "wall_clock_seconds": 0.3, "raw": 0.09978, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 21333251, "wall_clock_seconds": 0.3, "raw": 0.080417, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 25972665, "wall_clock_seconds": 0.2, "raw": 0.091076, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 29966501, "wall_clock_seconds": 0.3, "raw": 0.093957, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 32434396, "wall_clock_seconds": 0.3, "raw": 0.087561, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 36841357, "wall_clock_seconds": 0.3, "raw": 0.091703, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 39838983, "wall_clock_seconds": 0.3, "raw": 0.089962, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 42919683, "wall_clock_seconds": 0.3, "raw": 0.08737, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 46696615, "wall_clock_seconds": 0.3, "raw": 0.095009, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 47287649, "wall_clock_seconds": 0.2, "raw": 0.081794, "normalized": 0.0, "discount": 0.35, "effective": 0.0, "running_max": 0.0, "hints_active": 1, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:scan_stitch--nohints", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "scan_stitch--nohints", "task": "scan_stitch", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.1", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.000502, "area_term": 0.000239, "final_best": 0.001553, "best_raw": 0.101242, "best_effective": 0.001553, "floor": 0.1, "ceiling": 0.9, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 35498196, "usage": {"input": 237806, "output": 161179, "cache_read": 35099211, "cache_write": 35099211}, "cost_usd": 4.8789, "cost_source": "list_price", "wall_clock_seconds": 7059.1, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 8178432, "wall_clock_seconds": 0.3, "raw": 0.09917, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 17678988, "wall_clock_seconds": 0.3, "raw": 0.098225, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 18975525, "wall_clock_seconds": 0.3, "raw": 0.054514, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 20303357, "wall_clock_seconds": 0.3, "raw": 0.081556, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 23424463, "wall_clock_seconds": 0.3, "raw": 0.101049, "normalized": 0.001312, "discount": 1.0, "effective": 0.001312, "running_max": 0.001312, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 6, "tokens": 26913714, "wall_clock_seconds": 0.2, "raw": 0.069458, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.001312, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 7, "tokens": 30257755, "wall_clock_seconds": 0.2, "raw": 0.082371, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.001312, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 8, "tokens": 31893156, "wall_clock_seconds": 0.3, "raw": 0.101242, "normalized": 0.001553, "discount": 1.0, "effective": 0.001553, "running_max": 0.001553, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 9, "tokens": 34202408, "wall_clock_seconds": 0.3, "raw": 0.090841, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.001553, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 10, "tokens": 35498196, "wall_clock_seconds": 0.3, "raw": 0.097718, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.001553, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:shapes_generalize", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "shapes_generalize", "task": "shapes_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.3", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.039957, "area_term": 0.031571, "final_best": 0.0735, "best_raw": 0.368, "best_effective": 0.0735, "floor": 0.2, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 31363043, "usage": {"input": 273586, "output": 154233, "cache_read": 30935224, "cache_write": 30709112}, "cost_usd": 4.3843, "cost_source": "list_price", "wall_clock_seconds": 3209.8, "stop_reason": "submissions_exhausted", "hints": [{"id": 1, "multiplier": 0.5, "before_submission": 7}, {"id": 2, "multiplier": 0.35, "before_submission": 7}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 2437150, "wall_clock_seconds": 1.0, "raw": 0.233, "normalized": 0.04125, "discount": 1.0, "effective": 0.04125, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.29}, {"index": 2, "tokens": 2940576, "wall_clock_seconds": 0.8, "raw": 0.173, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.25}, {"index": 3, "tokens": 3217877, "wall_clock_seconds": 0.8, "raw": 0.145, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.27}, {"index": 4, "tokens": 3489703, "wall_clock_seconds": 0.8, "raw": 0.188, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.26}, {"index": 5, "tokens": 3766347, "wall_clock_seconds": 0.8, "raw": 0.2, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 6, "tokens": 4054677, "wall_clock_seconds": 0.8, "raw": 0.206, "normalized": 0.0075, "discount": 1.0, "effective": 0.0075, "running_max": 0.04125, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 0.21}, {"index": 7, "tokens": 15773648, "wall_clock_seconds": 51.6, "raw": 0.312, "normalized": 0.14, "discount": 0.35, "effective": 0.049, "running_max": 0.049, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 50.98}, {"index": 8, "tokens": 23430342, "wall_clock_seconds": 27.3, "raw": 0.308, "normalized": 0.135, "discount": 0.35, "effective": 0.04725, "running_max": 0.049, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 26.65}, {"index": 9, "tokens": 28000019, "wall_clock_seconds": 32.2, "raw": 0.32, "normalized": 0.15, "discount": 0.35, "effective": 0.0525, "running_max": 0.0525, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 31.71}, {"index": 10, "tokens": 31363043, "wall_clock_seconds": 42.4, "raw": 0.368, "normalized": 0.21, "discount": 0.35, "effective": 0.0735, "running_max": 0.0735, "hints_active": 2, "valid": true, "invalid_kind": null, "runtime_seconds": 41.88} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:shattered_prior", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "shattered_prior", "task": "shattered_prior", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.015714, "area_term": 0.012942, "final_best": 0.026802, "best_raw": 0.04873, "best_effective": 0.026802, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 5, "max_submissions": 10, "tokens_total": 24112230, "usage": {"input": 227577, "output": 122653, "cache_read": 23762000, "cache_write": 23762000}, "cost_usd": 3.4037, "cost_source": "list_price", "wall_clock_seconds": 2546.8, "stop_reason": "agent_finished", "hints": [{"id": 2, "multiplier": 0.7, "before_submission": 1}, {"id": 3, "multiplier": 0.65, "before_submission": 1}, {"id": 4, "multiplier": 0.55, "before_submission": 1}], "synthetic": false, "submissions": [ {"index": 1, "tokens": 5409863, "wall_clock_seconds": 1.8, "raw": 0.04873, "normalized": 0.04873, "discount": 0.55, "effective": 0.026802, "running_max": 0.026802, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 10828760, "wall_clock_seconds": 1.8, "raw": 0.002081, "normalized": 0.002081, "discount": 0.55, "effective": 0.001145, "running_max": 0.026802, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 15891823, "wall_clock_seconds": 1.7, "raw": 0.047588, "normalized": 0.047588, "discount": 0.55, "effective": 0.026174, "running_max": 0.026802, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 22367083, "wall_clock_seconds": 1.9, "raw": 0.0, "normalized": 0.0, "discount": 0.55, "effective": 0.0, "running_max": 0.026802, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 5, "tokens": 23231708, "wall_clock_seconds": 1.8, "raw": 0.040535, "normalized": 0.040535, "discount": 0.55, "effective": 0.022294, "running_max": 0.026802, "hints_active": 3, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:shattered_prior--nohints", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "shattered_prior--nohints", "task": "shattered_prior", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v3.3", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.04415, "area_term": 0.043005, "final_best": 0.04873, "best_raw": 0.04873, "best_effective": 0.04873, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 23143620, "usage": {"input": 187393, "output": 109100, "cache_read": 22847127, "cache_write": 22814640}, "cost_usd": 3.2146, "cost_source": "list_price", "wall_clock_seconds": 1898.1, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 858923, "wall_clock_seconds": 1.7, "raw": 0.04873, "normalized": 0.04873, "discount": 1.0, "effective": 0.04873, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 2, "tokens": 11178064, "wall_clock_seconds": 2.1, "raw": 0.002081, "normalized": 0.002081, "discount": 1.0, "effective": 0.002081, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 3, "tokens": 19159556, "wall_clock_seconds": 1.7, "raw": 0.047588, "normalized": 0.047588, "discount": 1.0, "effective": 0.047588, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null}, {"index": 4, "tokens": 19476703, "wall_clock_seconds": 1.7, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.04873, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": null} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:ship_detect", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "ship_detect", "task": "ship_detect", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v2.3", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.030013, "area_term": 0.026234, "final_best": 0.045129, "best_raw": 0.045129, "best_effective": 0.045129, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "cost_weighted", "start": 20000.0, "limit": 2000000.0, "final_best_weight": 0.2}, "submissions_used": 2, "max_submissions": null, "tokens_total": 7364308, "usage": {"input": 125572, "output": 41581, "cache_read": 7197155, "cache_write": 7083227}, "cost_usd": 1.0853, "cost_source": "list_price", "wall_clock_seconds": 1873.8, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "model_call": 69, "tokens": 3184756, "cost_weighted": 137543.9, "wall_clock_seconds": 1025.2, "raw": 0.045129, "normalized": 0.045129, "effective": 0.045129, "running_max": 0.045129, "scored": true, "final": false}, {"index": 2, "model_call": 80, "tokens": 4070693, "cost_weighted": 173448.3, "wall_clock_seconds": 1588.1, "raw": 0.04186, "normalized": 0.04186, "effective": 0.04186, "running_max": 0.045129, "scored": true, "final": false} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:ship_tune", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "ship_tune", "task": "ship_tune", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-07", "prompt_version": "v1.4", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.017532, "area_term": 0.011588, "final_best": 0.041307, "best_raw": 0.041307, "best_effective": 0.041307, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 6, "max_submissions": 10, "tokens_total": 45719899, "usage": {"input": 329567, "output": 167198, "cache_read": 45223134, "cache_write": 42800375}, "cost_usd": 6.0507, "cost_source": "list_price", "wall_clock_seconds": 3990.3, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 12371909, "wall_clock_seconds": 108.5, "raw": 0.037458, "normalized": 0.037458, "discount": 1.0, "effective": 0.037458, "running_max": 0.037458, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.99}, {"index": 2, "tokens": 35272824, "wall_clock_seconds": 123.3, "raw": 0.038274, "normalized": 0.038274, "discount": 1.0, "effective": 0.038274, "running_max": 0.038274, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 122.69}, {"index": 3, "tokens": 37483368, "wall_clock_seconds": 130.8, "raw": 0.040044, "normalized": 0.040044, "discount": 1.0, "effective": 0.040044, "running_max": 0.040044, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 130.3}, {"index": 4, "tokens": 40857558, "wall_clock_seconds": 122.3, "raw": 0.041307, "normalized": 0.041307, "discount": 1.0, "effective": 0.041307, "running_max": 0.041307, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 121.68}, {"index": 5, "tokens": 42528803, "wall_clock_seconds": 126.8, "raw": 0.038331, "normalized": 0.038331, "discount": 1.0, "effective": 0.038331, "running_max": 0.041307, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 126.28}, {"index": 6, "tokens": 44243476, "wall_clock_seconds": 125.6, "raw": 0.041156, "normalized": 0.041156, "discount": 1.0, "effective": 0.041156, "running_max": 0.041307, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 125.08} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:splash_generalize", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "splash_generalize", "task": "splash_generalize", "hints_enabled": true, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-08", "prompt_version": "v1.5", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.245162, "area_term": 0.184786, "final_best": 0.486667, "best_raw": 0.538, "best_effective": 0.486667, "floor": 0.1, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 10, "max_submissions": 10, "tokens_total": 21231387, "usage": {"input": 345338, "output": 205911, "cache_read": 20680138, "cache_write": 19004924}, "cost_usd": 3.3747, "cost_source": "list_price", "wall_clock_seconds": 3293.9, "stop_reason": "submissions_exhausted", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 8554763, "wall_clock_seconds": 53.0, "raw": 0.527, "normalized": 0.474444, "discount": 1.0, "effective": 0.474444, "running_max": 0.474444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 52.42}, {"index": 2, "tokens": 11848014, "wall_clock_seconds": 9.4, "raw": 0.522, "normalized": 0.468889, "discount": 1.0, "effective": 0.468889, "running_max": 0.474444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.73}, {"index": 3, "tokens": 12613869, "wall_clock_seconds": 12.4, "raw": 0.525, "normalized": 0.472222, "discount": 1.0, "effective": 0.472222, "running_max": 0.474444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 11.75}, {"index": 4, "tokens": 13245049, "wall_clock_seconds": 9.0, "raw": 0.348, "normalized": 0.275556, "discount": 1.0, "effective": 0.275556, "running_max": 0.474444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 8.37}, {"index": 5, "tokens": 15081670, "wall_clock_seconds": 26.9, "raw": 0.504, "normalized": 0.448889, "discount": 1.0, "effective": 0.448889, "running_max": 0.474444, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 26.19}, {"index": 6, "tokens": 15971985, "wall_clock_seconds": 8.6, "raw": 0.534, "normalized": 0.482222, "discount": 1.0, "effective": 0.482222, "running_max": 0.482222, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 7.94}, {"index": 7, "tokens": 17823791, "wall_clock_seconds": 13.1, "raw": 0.537, "normalized": 0.485556, "discount": 1.0, "effective": 0.485556, "running_max": 0.485556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 12.54}, {"index": 8, "tokens": 18981822, "wall_clock_seconds": 14.2, "raw": 0.535, "normalized": 0.483333, "discount": 1.0, "effective": 0.483333, "running_max": 0.485556, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 13.54}, {"index": 9, "tokens": 20195436, "wall_clock_seconds": 23.1, "raw": 0.538, "normalized": 0.486667, "discount": 1.0, "effective": 0.486667, "running_max": 0.486667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.39}, {"index": 10, "tokens": 21231387, "wall_clock_seconds": 32.1, "raw": 0.531, "normalized": 0.478889, "discount": 1.0, "effective": 0.478889, "running_max": 0.486667, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 31.51} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:tod_pipeline--nohints", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "tod_pipeline--nohints", "task": "tod_pipeline", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-12", "prompt_version": "v1.8", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.004121, "area_term": 0.002806, "final_best": 0.009381, "best_raw": 0.009381, "best_effective": 0.009381, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 27023194, "usage": {"input": 245903, "output": 118765, "cache_read": 26658526, "cache_write": 25966510}, "cost_usd": 3.712, "cost_source": "list_price", "wall_clock_seconds": 1879.2, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 7730493, "wall_clock_seconds": 15.3, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.0, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 15.0}, {"index": 2, "tokens": 12607434, "wall_clock_seconds": 12.3, "raw": 0.009381, "normalized": 0.009381, "discount": 1.0, "effective": 0.009381, "running_max": 0.009381, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 11.91}, {"index": 3, "tokens": 17594834, "wall_clock_seconds": 19.1, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.009381, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 18.83}, {"index": 4, "tokens": 25266025, "wall_clock_seconds": 22.4, "raw": 0.0, "normalized": 0.0, "discount": 1.0, "effective": 0.0, "running_max": 0.009381, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 22.1} ]}, {"sample_key": "jyXG6xP9E38k7LbpUticA2:weirdml_bonanza", "eval_id": "jyXG6xP9E38k7LbpUticA2", "sample_id": "weirdml_bonanza", "task": "weirdml_bonanza", "hints_enabled": false, "model_id": "gemini-3.8-flash-aistudio-high", "run_date": "2026-09-17", "run_kind": "official", "verdict": "valid", "score_era": "2026-09-10", "prompt_version": "v1.1", "source_commit": "2176de0", "source_log_sha256": "752d2427d733f096a38c09cd11798fc9ba0042aafc98f97a4ed59d70aee3b9a1", "agent": "gemini_cli", "agent_version": "0.59.0", "task_score": 0.210105, "area_term": 0.135918, "final_best": 0.506854, "best_raw": 0.506854, "best_effective": 0.506854, "floor": 0.0, "ceiling": 1.0, "axis": {"kind": "tokens", "start": 500000.0, "limit": 50000000.0, "final_best_weight": 0.2}, "submissions_used": 4, "max_submissions": 10, "tokens_total": 36164380, "usage": {"input": 378502, "output": 196389, "cache_read": 35589489, "cache_write": 33673893}, "cost_usd": 5.0937, "cost_source": "list_price", "wall_clock_seconds": 3040.6, "stop_reason": "agent_finished", "hints": [], "synthetic": false, "submissions": [ {"index": 1, "tokens": 8390185, "wall_clock_seconds": 5.9, "raw": 0.158394, "normalized": 0.158394, "discount": 1.0, "effective": 0.158394, "running_max": 0.158394, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 5.25}, {"index": 2, "tokens": 17444799, "wall_clock_seconds": 62.7, "raw": 0.458876, "normalized": 0.458876, "discount": 1.0, "effective": 0.458876, "running_max": 0.458876, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 61.98}, {"index": 3, "tokens": 28382666, "wall_clock_seconds": 107.7, "raw": 0.504118, "normalized": 0.504118, "discount": 1.0, "effective": 0.504118, "running_max": 0.504118, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 107.04}, {"index": 4, "tokens": 32514426, "wall_clock_seconds": 81.7, "raw": 0.506854, "normalized": 0.506854, "discount": 1.0, "effective": 0.506854, "running_max": 0.506854, "hints_active": 0, "valid": true, "invalid_kind": null, "runtime_seconds": 81.08} ]} ] }