{"task_set_hash":"664f8c7733fef6efa55c8c122d46434498b4d0ebd22aea2102c81dc5f4b2b4cf","model":{"slug":"openrouter/z-ai/glm-5.3","display_name":"GLM 5.3 (OpenRouter)","api_model_id":"z-ai/glm-5.3","family_slug":"glm","added_at":"2026-08-18","settings_suffix":"","max_input_tokens":1310720,"max_output_tokens":943718,"capabilities":["thinking","structured","batch"]},"aggregates":{"avg_score":61.271286,"tasks_attempted":659,"tasks_passed":338,"tasks_attempted_distinct":232,"tasks_passed_attempt_1":134.5,"tasks_passed_attempt_2_only":34.5,"pass_at_n":0.728448,"pass_denominator":232,"avg_cost_usd":0.194195,"latency_p50_ms":0,"latency_p95_ms":0,"pass_rate_ci":{"lower":0.6678414583971168,"upper":0.7816127515471847},"pass_hat_at_n":0.6637931034482759,"cost_per_pass_usd":0.266587,"run_count":2,"verified_runs":0},"settings":{"temperature":null,"thinking_budget":null,"tokens_avg_per_run":10631016,"consistency_pct":83.62},"history":[{"run_id":"4b623ade-d42d-4504-9488-96daefde9816","ts":"2026-09-11T12:19:00.003Z","score":49.006734,"cost_usd":26.491244,"tier":"claimed","status":"completed","completed_at":"2026-09-11T13:42:47.647Z","excluded_at":"2026-09-11T20:33:39.128Z","excluded_reason":"provider batch failure: the attempt-2 OpenRouter batch (batch-1789133297) returned failed for all 98 of 98 items, so every attempt-2 record carries zero tokens and no retry ever reached the model; pass@2 equals pass@1 at 134/232 where the two healthy GLM runs gained +33 and +36 on retry","tasks_attempted":232,"tasks_passed":134,"duration_ms":2378516},{"run_id":"ef24c680-60be-43c2-8d4f-fd199d5902b3","ts":"2026-09-11T11:58:43.210Z","score":60.713688,"cost_usd":44.58531,"tier":"claimed","status":"completed","completed_at":"2026-09-11T16:29:38.143Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":232,"tasks_passed":168,"duration_ms":3042042},{"run_id":"42894119-b128-46d7-9381-e1c5a0a70fa7","ts":"2026-09-11T11:38:58.111Z","score":61.837411,"cost_usd":45.521261,"tier":"claimed","status":"completed","completed_at":"2026-09-11T19:06:00.607Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":232,"tasks_passed":170,"duration_ms":2981346}],"failure_modes":[{"code":"AL0104","count":135,"pct":0.252809,"example_message":"Syntax error, ')' expected"},{"code":"AL0000","count":104,"pct":0.194757,"example_message":"App generation failed"},{"code":"AL0132","count":62,"pct":0.116105,"example_message":"'Guid' does not contain a definition for 'IsEmptyGuid'"},{"code":"AL0107","count":38,"pct":0.071161,"example_message":"Syntax error, identifier expected. Provide a valid name (letters, digits, and underscores only)."},{"code":"AL0111","count":35,"pct":0.065543,"example_message":"Semicolon expected. Add a semicolon (;) to terminate the statement."},{"code":"AL0105","count":28,"pct":0.052434,"example_message":"Syntax error, identifier expected; 'key' is a keyword"},{"code":"AL0198","count":22,"pct":0.041199,"example_message":"Expected one of the application object keywords (table, tableextension, page, pageextension, pagecustomization, profile, profileextension, codeunit, report, reportextension, xmlport, query, controladdin, dotnet, enum, enumextension, interface, permissionset, permissionsetextension, entitlement)"},{"code":"AL0360","count":21,"pct":0.039326,"example_message":"Text literal was not properly terminated. Use the character ' to terminate the literal."},{"code":"AL0118","count":10,"pct":0.018727,"example_message":"The name 'StrStartsWith' does not exist in the current context."},{"code":"AL0122","count":9,"pct":0.016854,"example_message":"Cannot implicitly convert type 'Text' to 'SecretText'. Use an explicit conversion or change the type."}],"recent_runs":[{"run_id":"4b623ade-d42d-4504-9488-96daefde9816","ts":"2026-09-11T12:19:00.003Z","score":49.006734,"cost_usd":26.491244,"tier":"claimed","status":"completed","completed_at":"2026-09-11T13:42:47.647Z","excluded_at":"2026-09-11T20:33:39.128Z","excluded_reason":"provider batch failure: the attempt-2 OpenRouter batch (batch-1789133297) returned failed for all 98 of 98 items, so every attempt-2 record carries zero tokens and no retry ever reached the model; pass@2 equals pass@1 at 134/232 where the two healthy GLM runs gained +33 and +36 on retry","tasks_attempted":232,"tasks_passed":134,"duration_ms":2378516},{"run_id":"ef24c680-60be-43c2-8d4f-fd199d5902b3","ts":"2026-09-11T11:58:43.210Z","score":60.713688,"cost_usd":44.58531,"tier":"claimed","status":"completed","completed_at":"2026-09-11T16:29:38.143Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":232,"tasks_passed":168,"duration_ms":3042042},{"run_id":"42894119-b128-46d7-9381-e1c5a0a70fa7","ts":"2026-09-11T11:38:58.111Z","score":61.837411,"cost_usd":45.521261,"tier":"claimed","status":"completed","completed_at":"2026-09-11T19:06:00.607Z","excluded_at":null,"excluded_reason":null,"tasks_attempted":232,"tasks_passed":170,"duration_ms":2981346}]}