{"task_set_hash":"664f8c7733fef6efa55c8c122d46434498b4d0ebd22aea2102c81dc5f4b2b4cf","model":{"slug":"anthropic/claude-sonnet-5","display_name":"Claude Sonnet 5","api_model_id":"claude-sonnet-5","family_slug":"claude","added_at":"2026-06-09","settings_suffix":"","max_input_tokens":1000000,"max_output_tokens":128000,"capabilities":["thinking","image","pdf","structured","batch"]},"aggregates":{"avg_score":72.120699,"tasks_attempted":985,"tasks_passed":556,"tasks_attempted_distinct":232,"tasks_passed_attempt_1":134.333333,"tasks_passed_attempt_2_only":51,"pass_at_n":0.798851,"pass_denominator":232,"avg_cost_usd":0.087239,"latency_p50_ms":0,"latency_p95_ms":0,"pass_rate_ci":{"lower":0.7425906737750474,"upper":0.845374582001146},"pass_hat_at_n":0.7112068965517241,"cost_per_pass_usd":0.108891,"run_count":3,"verified_runs":0},"settings":{"temperature":null,"thinking_budget":null,"tokens_avg_per_run":2921473,"consistency_pct":71.55},"history":[{"run_id":"a1d5ecf0-e656-4fbb-985a-2b3a1e3616e1","ts":"2026-09-09T22:38:04.562Z","score":71.7621,"cost_usd":19.856082,"tier":"claimed","status":"completed","completed_at":"2026-09-09T23:29:33.906Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":231,"tasks_passed":186,"duration_ms":3469143},{"run_id":"b447ecd3-d6b5-488a-a151-b4232c8b8c83","ts":"2026-09-09T22:14:54.947Z","score":72.972973,"cost_usd":20.299842,"tier":"claimed","status":"completed","completed_at":"2026-09-09T23:07:48.369Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":232,"tasks_passed":187,"duration_ms":3709862},{"run_id":"ae125a12-0b60-4149-9874-aa6a280b87d3","ts":"2026-09-09T19:57:19.487Z","score":71.608725,"cost_usd":20.387672,"tier":"claimed","status":"completed","completed_at":"2026-09-09T20:47:34.703Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":231,"tasks_passed":183,"duration_ms":3829610},{"run_id":"145a7708-c39e-4e5b-8940-f768b9aec3d8","ts":"2026-09-09T00:41:35.868Z","score":41.906983,"cost_usd":29.170322,"tier":"claimed","status":"completed","completed_at":"2026-09-09T05:11:17.979Z","excluded_at":"2026-09-09T06:14:12.147Z","excluded_reason":"host out of memory: 149 attempt-1 compiles failed in about 195 ms with zero compiler errors and no artifact","tasks_attempted":208,"tasks_passed":138,"duration_ms":2429703},{"run_id":"c7a94f8b-2bc5-4800-9f27-5905185a12cf","ts":"2026-09-09T00:28:20.451Z","score":54.237528,"cost_usd":27.927904,"tier":"claimed","status":"completed","completed_at":"2026-09-09T05:10:22.664Z","excluded_at":"2026-09-09T20:51:51.114Z","excluded_reason":"host out of memory: 82 compiles failed in 120-436 ms with zero compiler errors and empty output; the clean rerun of the same model and tasks has 0 of 325","tasks_attempted":232,"tasks_passed":169,"duration_ms":3131951},{"run_id":"4c095313-8a6f-482e-9def-e7d3ae395815","ts":"2026-09-09T00:08:43.940Z","score":64.180384,"cost_usd":20.200014,"tier":"claimed","status":"completed","completed_at":"2026-09-09T05:08:44.313Z","excluded_at":"2026-09-09T20:51:43.506Z","excluded_reason":"host out of memory: 31 compiles failed in 122-249 ms with zero compiler errors and empty output; the clean rerun of the same model and tasks has 0 of 325","tasks_attempted":232,"tasks_passed":165,"duration_ms":3071406}],"failure_modes":[{"code":"AL0104","count":188,"pct":0.200426,"example_message":"Syntax error, ')' expected"},{"code":"AL0000","count":181,"pct":0.192964,"example_message":"App generation failed"},{"code":"AL0132","count":98,"pct":0.104478,"example_message":"'FieldType' does not contain a definition for 'Enum'"},{"code":"AL0107","count":66,"pct":0.070362,"example_message":"Syntax error, identifier expected. Provide a valid name (letters, digits, and underscores only)."},{"code":"AL0111","count":57,"pct":0.060768,"example_message":"Semicolon expected. Add a semicolon (;) to terminate the statement."},{"code":"AL0105","count":56,"pct":0.059701,"example_message":"Syntax error, identifier expected; 'key' is a keyword"},{"code":"AL0126","count":38,"pct":0.040512,"example_message":"No overload for method 'Clear' takes 1 arguments. Candidates: 'Clear()' defined in Codeunit 'CG H054 Cache' by the extension CentralGauge_CG-AL-H054_2 by CentralGauge (1.0.0.0)"},{"code":"AL0135","count":27,"pct":0.028785,"example_message":"There is no argument given that corresponds to the required formal parameter 'ConflictMessage' of 'SyncContact(Code[20], Code[20], Text[100], Text[20], Text[100], Text[20], var Text)'"},{"code":"AL0198","count":25,"pct":0.026652,"example_message":"Expected one of the application object keywords (table, tableextension, page, pageextension, pagecustomization, profile, profileextension, codeunit, report, reportextension, xmlport, query, controladdin, dotnet, enum, enumextension, interface, permissionset, permissionsetextension, entitlement)"},{"code":"AL0118","count":23,"pct":0.02452,"example_message":"The name 'CreateSequentialGuid' does not exist in the current context."}],"recent_runs":[{"run_id":"a1d5ecf0-e656-4fbb-985a-2b3a1e3616e1","ts":"2026-09-09T22:38:04.562Z","score":71.7621,"cost_usd":19.856082,"tier":"claimed","status":"completed","completed_at":"2026-09-09T23:29:33.906Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":231,"tasks_passed":186,"duration_ms":3469143},{"run_id":"b447ecd3-d6b5-488a-a151-b4232c8b8c83","ts":"2026-09-09T22:14:54.947Z","score":72.972973,"cost_usd":20.299842,"tier":"claimed","status":"completed","completed_at":"2026-09-09T23:07:48.369Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":232,"tasks_passed":187,"duration_ms":3709862},{"run_id":"ae125a12-0b60-4149-9874-aa6a280b87d3","ts":"2026-09-09T19:57:19.487Z","score":71.608725,"cost_usd":20.387672,"tier":"claimed","status":"completed","completed_at":"2026-09-09T20:47:34.703Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":231,"tasks_passed":183,"duration_ms":3829610},{"run_id":"145a7708-c39e-4e5b-8940-f768b9aec3d8","ts":"2026-09-09T00:41:35.868Z","score":41.906983,"cost_usd":29.170322,"tier":"claimed","status":"completed","completed_at":"2026-09-09T05:11:17.979Z","excluded_at":"2026-09-09T06:14:12.147Z","excluded_reason":"host out of memory: 149 attempt-1 compiles failed in about 195 ms with zero compiler errors and no artifact","tasks_attempted":208,"tasks_passed":138,"duration_ms":2429703},{"run_id":"c7a94f8b-2bc5-4800-9f27-5905185a12cf","ts":"2026-09-09T00:28:20.451Z","score":54.237528,"cost_usd":27.927904,"tier":"claimed","status":"completed","completed_at":"2026-09-09T05:10:22.664Z","excluded_at":"2026-09-09T20:51:51.114Z","excluded_reason":"host out of memory: 82 compiles failed in 120-436 ms with zero compiler errors and empty output; the clean rerun of the same model and tasks has 0 of 325","tasks_attempted":232,"tasks_passed":169,"duration_ms":3131951},{"run_id":"4c095313-8a6f-482e-9def-e7d3ae395815","ts":"2026-09-09T00:08:43.940Z","score":64.180384,"cost_usd":20.200014,"tier":"claimed","status":"completed","completed_at":"2026-09-09T05:08:44.313Z","excluded_at":"2026-09-09T20:51:43.506Z","excluded_reason":"host out of memory: 31 compiles failed in 122-249 ms with zero compiler errors and empty output; the clean rerun of the same model and tasks has 0 of 325","tasks_attempted":232,"tasks_passed":165,"duration_ms":3071406}]}